Add prometheus query offset as an environment variable with heavy code docs.

pkg/env/costmodelenv.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/kubecost/cost-model/pkg/log"
+	"github.com/kubecost/cost-model/pkg/util/timeutil"
 )
 
 const (
@@ -79,6 +80,7 @@ const (
 	KubecostConfigBucketEnvVar    = "KUBECOST_CONFIG_BUCKET"
 	ClusterInfoFileEnabledEnvVar  = "CLUSTER_INFO_FILE_ENABLED"
 	ClusterCacheFileEnabledEnvVar = "CLUSTER_CACHE_FILE_ENABLED"
+	PrometheusQueryOffsetEnvVar   = "PROMETHEUS_QUERY_OFFSET"
 )
 
 // GetKubecostConfigBucket returns a file location for a mounted bucket configuration which is used to store
@@ -99,6 +101,26 @@ func IsClusterCacheFileEnabled() bool {
 	return GetBool(ClusterCacheFileEnabledEnvVar, false)
 }
 
+// GetPrometheusQueryOffset returns the time.Duration to offset all prometheus queries by. NOTE: This env var is applied
+// to all non-range queries made via our query context. This should only be applied when there is a significant delay in
+// data arriving in the target prom db. For example, if supplying a thanos or cortex querier for the prometheus server, using
+// a 3h offset will ensure that current time = current time - 3h.
+//
+// This offset is NOT the same as the GetThanosOffset() option, as that is only applied to queries made specifically targetting
+// thanos. This offset is applied globally.
+func GetPrometheusQueryOffset() time.Duration {
+	offset := Get(PrometheusQueryOffsetEnvVar, "")
+	if offset == "" {
+		return 0
+	}
+
+	dur, err := timeutil.ParseDuration(offset)
+	if err != nil {
+		return 0
+	}
+	return dur
+}
+
 func GetPricingConfigmapName() string {
 	return Get(PricingConfigmapName, "pricing-configs")
 }

pkg/prom/query.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/kubecost/cost-model/pkg/env"
 	"github.com/kubecost/cost-model/pkg/errors"
 	"github.com/kubecost/cost-model/pkg/log"
 	"github.com/kubecost/cost-model/pkg/util/httputil"
@@ -21,6 +22,10 @@ const (
 	epQueryRange = apiPrefix + "/query_range"
 )
 
+// prometheus query offset to apply to each non-range query
+// package scope to prevent calling duration parse each use
+var promQueryOffset time.Duration = env.GetPrometheusQueryOffset()
+
 // Context wraps a Prometheus client and provides methods for querying and
 // parsing query responses and errors.
 type Context struct {
@@ -171,7 +176,14 @@ func (ctx *Context) RawQuery(query string) ([]byte, error) {
 	u := ctx.Client.URL(epQuery, nil)
 	q := u.Query()
 	q.Set("query", query)
-	q.Set("time", time.Now().UTC().Add(-3*time.Hour).Format(time.RFC3339))
+
+	// for non-range queries, we set the timestamp for the query to time-offset
+	// this is a special use case that's typically only used when our primary
+	// prom db has delayed insertion (thanos, cortex, etc...)
+	if promQueryOffset != 0 {
+		q.Set("time", time.Now().UTC().Add(-promQueryOffset).Format(time.RFC3339))
+	}
+
 	u.RawQuery = q.Encode()
 
 	req, err := http.NewRequest(http.MethodPost, u.String(), nil)