Prometheus Error Checking and Context Inclusion

pkg/costmodel/costmodel.go

@@ -2617,18 +2617,28 @@ func QueryRange(cli prometheusClient.Client, query string, start, end time.Time,
 
 	resp, body, warnings, err := cli.Do(context.Background(), req)
 	for _, w := range warnings {
-		klog.V(3).Infof("[Warning] '%s' fetching query '%s'", w, query)
+		klog.V(3).Infof("Warning '%s' fetching query '%s'", w, query)
 	}
 	if err != nil {
-		return nil, fmt.Errorf("[Error] %s fetching query %s", err.Error(), query)
+		if resp == nil {
+			return nil, fmt.Errorf("Error: %s, Body: %s Query: %s", err.Error(), body, query)
+		}
+
+		return nil, fmt.Errorf("%d (%s) Headers: %s Error: %s Body: %s Query: %s", resp.StatusCode, http.StatusText(resp.StatusCode), util.HeaderString(resp.Header), body, err.Error(), query)
+	}
+
+	// Unsuccessful Status Code, log body and status
+	statusCode := resp.StatusCode
+	statusText := http.StatusText(statusCode)
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, fmt.Errorf("%d (%s) Headers: %s Body: %s Query: %s", statusCode, statusText, util.HeaderString(resp.Header), body, query)
 	}
 
 	var toReturn interface{}
 	err = json.Unmarshal(body, &toReturn)
 	if err != nil {
-		return nil, fmt.Errorf("[Error] %d %s fetching query %s", resp.StatusCode, err.Error(), query)
+		return nil, fmt.Errorf("%d (%s) Headers: %s Error: %s Body: %s Query: %s", statusCode, statusText, util.HeaderString(resp.Header), err.Error(), body, query)
 	}
-
 	return toReturn, nil
 }
 
@@ -2649,15 +2659,23 @@ func Query(cli prometheusClient.Client, query string) (interface{}, error) {
 	}
 	if err != nil {
 		if resp == nil {
-			return nil, fmt.Errorf("Error %s fetching query %s", err.Error(), query)
+			return nil, fmt.Errorf("Error: %s, Body: %s Query: %s", err.Error(), body, query)
 		}
 
-		return nil, fmt.Errorf("%d Error %s fetching query %s", resp.StatusCode, err.Error(), query)
+		return nil, fmt.Errorf("%d (%s) Headers: %s Error: %s Body: %s Query: %s", resp.StatusCode, http.StatusText(resp.StatusCode), util.HeaderString(resp.Header), body, err.Error(), query)
 	}
+
+	// Unsuccessful Status Code, log body and status
+	statusCode := resp.StatusCode
+	statusText := http.StatusText(statusCode)
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, fmt.Errorf("%d (%s) Headers: %s, Body: %s Query: %s", statusCode, statusText, util.HeaderString(resp.Header), body, query)
+	}
+
 	var toReturn interface{}
 	err = json.Unmarshal(body, &toReturn)
 	if err != nil {
-		return nil, fmt.Errorf("Error %s fetching query %s", err.Error(), query)
+		return nil, fmt.Errorf("%d (%s) Headers: %s Error: %s Body: %s Query: %s", statusCode, statusText, util.HeaderString(resp.Header), err.Error(), body, query)
 	}
 	return toReturn, nil
 }

pkg/costmodel/router.go

@@ -937,6 +937,7 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) {
 	}
 
 	queryConcurrency := env.GetMaxQueryConcurrency()
+	klog.Infof("Prometheus/Thanos Client Max Concurrency set to %d", queryConcurrency)
 
 	var LongTimeoutRoundTripper http.RoundTripper = &http.Transport{ // may be necessary for long prometheus queries. TODO: make this configurable
 		Proxy: http.ProxyFromEnvironment,

pkg/prom/query.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 
 	"github.com/kubecost/cost-model/pkg/errors"
+	"github.com/kubecost/cost-model/pkg/util"
 	prometheus "github.com/prometheus/client_golang/api"
 	"k8s.io/klog"
 )
@@ -92,15 +93,23 @@ func (ctx *Context) query(query string) (interface{}, error) {
 	}
 	if err != nil {
 		if resp == nil {
-			return nil, fmt.Errorf("Error %s fetching query %s", err.Error(), query)
+			return nil, fmt.Errorf("Error: %s, Body: %s Query: %s", err.Error(), body, query)
 		}
 
-		return nil, fmt.Errorf("%d Error %s fetching query %s", resp.StatusCode, err.Error(), query)
+		return nil, fmt.Errorf("%d (%s) Headers: %s Error: %s Body: %s Query: %s", resp.StatusCode, http.StatusText(resp.StatusCode), util.HeaderString(resp.Header), body, err.Error(), query)
 	}
+
+	// Unsuccessful Status Code, log body and status
+	statusCode := resp.StatusCode
+	statusText := http.StatusText(statusCode)
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, fmt.Errorf("%d (%s) Headers: %s, Body: %s Query: %s", statusCode, statusText, util.HeaderString(resp.Header), body, query)
+	}
+
 	var toReturn interface{}
 	err = json.Unmarshal(body, &toReturn)
 	if err != nil {
-		return nil, fmt.Errorf("Error %s fetching query %s", err.Error(), query)
+		return nil, fmt.Errorf("%d (%s) Headers: %s Error: %s Body: %s Query: %s", statusCode, statusText, util.HeaderString(resp.Header), err.Error(), body, query)
 	}
 	return toReturn, nil
 }

pkg/util/http.go

@@ -0,0 +1,33 @@
+package util
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+)
+
+// HeaderString write the request/response http.Header to a string.
+func HeaderString(h http.Header) string {
+	var sb strings.Builder
+	var first bool = true
+	sb.WriteString("{ ")
+
+	for k, vs := range h {
+		if first {
+			first = false
+		} else {
+			sb.WriteString(", ")
+		}
+		fmt.Fprintf(&sb, "%s: [ ", k)
+		for idx, v := range vs {
+			sb.WriteString(v)
+			if idx != len(vs)-1 {
+				sb.WriteString(", ")
+			}
+		}
+		sb.WriteString(" ]")
+	}
+	sb.WriteString(" }")
+
+	return sb.String()
+}

test/util_test.go

@@ -0,0 +1,48 @@
+package test
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/kubecost/cost-model/pkg/util"
+)
+
+func TestHeaderString(t *testing.T) {
+	h := make(http.Header)
+	h.Add("foo", "abc")
+	h.Add("foo", "123")
+	h.Add("bar", "foo")
+	h.Add("Content-Type", "application/octet-stream")
+
+	s := util.HeaderString(h)
+	if len(s) == 0 {
+		t.Errorf("Header String failed to produce a valid output")
+		return
+	}
+
+	t.Logf("Result: %s\n", s)
+}
+
+func TestEmptyHeader(t *testing.T) {
+	h := make(http.Header)
+
+	s := util.HeaderString(h)
+	if len(s) == 0 {
+		t.Errorf("Header String failed to produce a valid output")
+		return
+	}
+
+	t.Logf("Result: %s\n", s)
+}
+
+func TestNilHeader(t *testing.T) {
+	var h http.Header
+
+	s := util.HeaderString(h)
+	if len(s) == 0 {
+		t.Errorf("Header String failed to produce a valid output")
+		return
+	}
+
+	t.Logf("Result: %s\n", s)
+}