Separate MetricsQuerier, remove legacy code no longer relevant. Add some general API docs for newer implementations. Fix bugs related to LoadBalancers and node disks.

core/pkg/clusters/clusterinfo.go

@@ -1,5 +1,7 @@
 package clusters
 
+import "maps"
+
 // The following constants are used as keys into the cluster info map data structure
 const (
 	ClusterInfoIdKey               = "id"
@@ -73,3 +75,25 @@ type ClusterInfoProvider interface {
 	// GetClusterInfo returns a string map containing the local/remote connected cluster info
 	GetClusterInfo() map[string]string
 }
+
+// ClusterInfoDecorator is a ClusterInfoProvider that decorates another ClusterInfoProvider with additional info.
+type ClusterInfoDecorator struct {
+	provider       ClusterInfoProvider
+	additionalInfo map[string]string
+}
+
+// NewClusterInfoDecorator creates a new ClusterInfoDecorator which will append additional info to the cluster info
+// returned by the provider.
+func NewClusterInfoDecorator(provider ClusterInfoProvider, additionalInfo map[string]string) ClusterInfoProvider {
+	return &ClusterInfoDecorator{
+		provider:       provider,
+		additionalInfo: additionalInfo,
+	}
+}
+
+// GetClusterInfo returns a string map containing the local/remote connected cluster info
+func (cid *ClusterInfoDecorator) GetClusterInfo() map[string]string {
+	info := cid.provider.GetClusterInfo()
+	maps.Copy(info, cid.additionalInfo)
+	return info
+}

core/pkg/source/datasource.go

@@ -7,7 +7,7 @@ import (
 	"github.com/opencost/opencost/core/pkg/clusters"
 )
 
-type ClusterMetricsQuerier interface {
+type MetricsQuerier interface {
 	// Cluster Disks
 	QueryPVActiveMinutes(start, end time.Time) *Future[PVActiveMinutesResult]
 	QueryPVUsedAverage(start, end time.Time) *Future[PVUsedAvgResult]
@@ -20,8 +20,6 @@ type ClusterMetricsQuerier interface {
 	QueryLocalStorageUsedAvg(start, end time.Time) *Future[LocalStorageUsedAvgResult]
 	QueryLocalStorageUsedMax(start, end time.Time) *Future[LocalStorageUsedMaxResult]
 	QueryLocalStorageBytes(start, end time.Time) *Future[LocalStorageBytesResult]
-	QueryLocalStorageBytesByProvider(provider string, start, end time.Time) *Future[LocalStorageBytesByProviderResult]
-	QueryLocalStorageUsedByProvider(provider string, start, end time.Time) *Future[LocalStorageUsedByProviderResult]
 
 	// Nodes
 	QueryNodeActiveMinutes(start, end time.Time) *Future[NodeActiveMinutesResult]
@@ -32,7 +30,6 @@ type ClusterMetricsQuerier interface {
 	QueryNodeGPUCount(start, end time.Time) *Future[NodeGPUCountResult]
 	QueryNodeCPUModeTotal(start, end time.Time) *Future[NodeCPUModeTotalResult]
 	QueryNodeIsSpot(start, end time.Time) *Future[NodeIsSpotResult]
-	QueryNodeCPUModePercent(start, end time.Time) *Future[NodeCPUModePercentResult]
 	QueryNodeRAMSystemPercent(start, end time.Time) *Future[NodeRAMSystemPercentResult]
 	QueryNodeRAMUserPercent(start, end time.Time) *Future[NodeRAMUserPercentResult]
 
@@ -44,40 +41,25 @@ type ClusterMetricsQuerier interface {
 	QueryClusterManagementDuration(start, end time.Time) *Future[ClusterManagementDurationResult]
 	QueryClusterManagementPricePerHr(start, end time.Time) *Future[ClusterManagementPricePerHrResult]
 
-	// Cluster Costs
-	QueryDataCount(start, end time.Time) *Future[DataCountResult]
-	QueryTotalGPU(start, end time.Time) *Future[TotalGPUResult]
-	QueryTotalCPU(start, end time.Time) *Future[TotalCPUResult]
-	QueryTotalRAM(start, end time.Time) *Future[TotalRAMResult]
-	QueryTotalStorage(start, end time.Time) *Future[TotalStorageResult]
-
-	// Cluster Costs
-	QueryClusterCores(start, end time.Time, step time.Duration) *Future[ClusterCoresResult]
-	QueryClusterRAM(start, end time.Time, step time.Duration) *Future[ClusterRAMResult]
-	QueryClusterStorage(start, end time.Time, step time.Duration) *Future[ClusterStorageResult]
-	QueryClusterStorageByProvider(provider string, start, end time.Time, step time.Duration) *Future[ClusterStorageResult]
-	QueryClusterTotal(start, end time.Time, step time.Duration) *Future[ClusterTotalResult]
-	QueryClusterTotalByProvider(provider string, start, end time.Time, step time.Duration) *Future[ClusterTotalResult]
-	QueryClusterNodes(start, end time.Time, step time.Duration) *Future[ClusterNodesResult]
-	QueryClusterNodesByProvider(provider string, start, end time.Time, step time.Duration) *Future[ClusterNodesResult]
-}
-
-type AllocationMetricsQuerier interface {
+	// Pods
 	QueryPods(start, end time.Time) *Future[PodsResult]
 	QueryPodsUID(start, end time.Time) *Future[PodsResult]
 
+	// RAM
 	QueryRAMBytesAllocated(start, end time.Time) *Future[RAMBytesAllocatedResult]
 	QueryRAMRequests(start, end time.Time) *Future[RAMRequestsResult]
 	QueryRAMUsageAvg(start, end time.Time) *Future[RAMUsageAvgResult]
 	QueryRAMUsageMax(start, end time.Time) *Future[RAMUsageMaxResult]
 	QueryNodeRAMPricePerGiBHr(start, end time.Time) *Future[NodeRAMPricePerGiBHrResult]
 
+	// CPU
 	QueryCPUCoresAllocated(start, end time.Time) *Future[CPUCoresAllocatedResult]
 	QueryCPURequests(start, end time.Time) *Future[CPURequestsResult]
 	QueryCPUUsageAvg(start, end time.Time) *Future[CPUUsageAvgResult]
 	QueryCPUUsageMax(start, end time.Time) *Future[CPUUsageMaxResult]
 	QueryNodeCPUPricePerHr(start, end time.Time) *Future[NodeCPUPricePerHrResult]
 
+	// GPU
 	QueryGPUsAllocated(start, end time.Time) *Future[GPUsAllocatedResult]
 	QueryGPUsRequested(start, end time.Time) *Future[GPUsRequestedResult]
 	QueryGPUsUsageAvg(start, end time.Time) *Future[GPUsUsageAvgResult]
@@ -86,14 +68,17 @@ type AllocationMetricsQuerier interface {
 	QueryGPUInfo(start, end time.Time) *Future[GPUInfoResult]
 	QueryIsGPUShared(start, end time.Time) *Future[IsGPUSharedResult]
 
+	// PVC
 	QueryPodPVCAllocation(start, end time.Time) *Future[PodPVCAllocationResult]
 	QueryPVCBytesRequested(start, end time.Time) *Future[PVCBytesRequestedResult]
 	QueryPVCInfo(start, end time.Time) *Future[PVCInfoResult]
 
+	// PV
 	QueryPVBytes(start, end time.Time) *Future[PVBytesResult]
 	QueryPVPricePerGiBHour(start, end time.Time) *Future[PVPricePerGiBHourResult]
 	QueryPVInfo(start, end time.Time) *Future[PVInfoResult]
 
+	// Network
 	QueryNetZoneGiB(start, end time.Time) *Future[NetZoneGiBResult]
 	QueryNetZonePricePerGiB(start, end time.Time) *Future[NetZonePricePerGiBResult]
 	QueryNetRegionGiB(start, end time.Time) *Future[NetRegionGiBResult]
@@ -103,9 +88,11 @@ type AllocationMetricsQuerier interface {
 	QueryNetReceiveBytes(start, end time.Time) *Future[NetReceiveBytesResult]
 	QueryNetTransferBytes(start, end time.Time) *Future[NetTransferBytesResult]
 
+	// Annotations
 	QueryNamespaceAnnotations(start, end time.Time) *Future[NamespaceAnnotationsResult]
 	QueryPodAnnotations(start, end time.Time) *Future[PodAnnotationsResult]
 
+	// Labels
 	QueryNodeLabels(start, end time.Time) *Future[NodeLabelsResult]
 	QueryNamespaceLabels(start, end time.Time) *Future[NamespaceLabelsResult]
 	QueryPodLabels(start, end time.Time) *Future[PodLabelsResult]
@@ -115,22 +102,26 @@ type AllocationMetricsQuerier interface {
 	QueryDaemonSetLabels(start, end time.Time) *Future[DaemonSetLabelsResult]
 	QueryJobLabels(start, end time.Time) *Future[JobLabelsResult]
 
+	// ReplicaSet -> Controller mapping
 	QueryPodsWithReplicaSetOwner(start, end time.Time) *Future[PodsWithReplicaSetOwnerResult]
 	QueryReplicaSetsWithoutOwners(start, end time.Time) *Future[ReplicaSetsWithoutOwnersResult]
 	QueryReplicaSetsWithRollout(start, end time.Time) *Future[ReplicaSetsWithRolloutResult]
 
+	// Data Coverage Query
 	QueryDataCoverage(limitDays int) (time.Time, time.Time, error)
 }
 
 type OpenCostDataSource interface {
-	ClusterMetricsQuerier
-	AllocationMetricsQuerier
+	// RegisterEndPoints registers any custom endpoints that can be used for diagnostics or debug purposes.
+	RegisterEndPoints(router *httprouter.Router)
 
-	NewClusterMap(clusterInfoProvider clusters.ClusterInfoProvider) clusters.ClusterMap
+	// Metrics returns a MetricsQuerier that can be used to query historical metrics data from the data source.
+	Metrics() MetricsQuerier
 
-	RegisterEndPoints(router *httprouter.Router)
+	// ClusterMap returns a mapping of cluster identifier to ClusterInfo for all known clusters (local only for
+	// single cluster deployments).
+	ClusterMap() clusters.ClusterMap
 
 	BatchDuration() time.Duration
 	Resolution() time.Duration
-	MetaData() map[string]string
 }

core/pkg/source/decoders.go

@@ -79,6 +79,9 @@ type LocalStorageActiveMinutesResult struct {
 func DecodeLocalStorageActiveMinutesResult(result *QueryResult) *LocalStorageActiveMinutesResult {
 	cluster, _ := result.GetCluster()
 	node, _ := result.GetNode()
+	if node == "" {
+		node, _ = result.GetInstance()
+	}
 	providerId, _ := result.GetProviderID()
 
 	return &LocalStorageActiveMinutesResult{
@@ -190,18 +193,6 @@ func DecodeLocalStorageBytesResult(result *QueryResult) *LocalStorageBytesResult
 	}
 }
 
-type LocalStorageBytesByProviderResult = TotalStorageResult
-
-func DecodeLocalStorageBytesByProviderResult(result *QueryResult) *LocalStorageBytesByProviderResult {
-	return DecodeTotalStorageResult(result)
-}
-
-type LocalStorageUsedByProviderResult = TotalStorageResult
-
-func DecodeLocalStorageUsedByProviderResult(result *QueryResult) *LocalStorageUsedByProviderResult {
-	return DecodeTotalStorageResult(result)
-}
-
 type NodeActiveMinutesResult struct {
 	Cluster    string
 	Node       string
@@ -329,26 +320,6 @@ func DecodeNodeIsSpotResult(result *QueryResult) *NodeIsSpotResult {
 	}
 }
 
-type NodeCPUModePercentResult struct {
-	Cluster string
-	Node    string
-	Mode    string
-	Data    []*util.Vector
-}
-
-func DecodeNodeCPUModePercentResult(result *QueryResult) *NodeCPUModePercentResult {
-	cluster, _ := result.GetCluster()
-	node, _ := result.GetString("kubernetes_node")
-	mode, _ := result.GetString("mode")
-
-	return &NodeCPUModePercentResult{
-		Cluster: cluster,
-		Node:    node,
-		Mode:    mode,
-		Data:    result.Values,
-	}
-}
-
 type NodeRAMSystemPercentResult struct {
 	Cluster  string
 	Instance string
@@ -425,108 +396,6 @@ func DecodeClusterManagementPricePerHrResult(result *QueryResult) *ClusterManage
 	return DecodeClusterManagementDurationResult(result)
 }
 
-type DataCountResult struct {
-	Cluster string
-	Data    []*util.Vector
-}
-
-func DecodeDataCountResult(result *QueryResult) *DataCountResult {
-	cluster, _ := result.GetCluster()
-
-	return &DataCountResult{
-		Cluster: cluster,
-		Data:    result.Values,
-	}
-}
-
-type TotalResult struct {
-	Cluster string
-
-	Data []*util.Vector
-}
-
-func DecodeTotalResult(result *QueryResult) *TotalResult {
-	cluster, _ := result.GetCluster()
-	return &TotalResult{
-		Cluster: cluster,
-		Data:    result.Values,
-	}
-}
-
-type TotalCPUResult = TotalResult
-
-func DecodeTotalCPUResult(result *QueryResult) *TotalCPUResult {
-	return DecodeTotalResult(result)
-}
-
-type TotalRAMResult = TotalResult
-
-func DecodeTotalRAMResult(result *QueryResult) *TotalRAMResult {
-	return DecodeTotalResult(result)
-}
-
-type TotalGPUResult = TotalResult
-
-func DecodeTotalGPUResult(result *QueryResult) *TotalGPUResult {
-	return DecodeTotalResult(result)
-}
-
-type TotalStorageResult = TotalResult
-
-func DecodeTotalStorageResult(result *QueryResult) *TotalStorageResult {
-	return DecodeTotalResult(result)
-}
-
-type ClusterResult struct {
-	Data []*util.Vector
-}
-
-func DecodeClusterResult(result *QueryResult) *ClusterResult {
-	return &ClusterResult{
-		Data: result.Values,
-	}
-}
-
-type ClusterCoresResult = ClusterResult
-
-func DecodeClusterCoresResult(result *QueryResult) *ClusterCoresResult {
-	return DecodeClusterResult(result)
-}
-
-type ClusterRAMResult = ClusterResult
-
-func DecodeClusterRAMResult(result *QueryResult) *ClusterRAMResult {
-	return DecodeClusterResult(result)
-}
-
-type ClusterStorageResult = ClusterResult
-
-func DecodeClusterStorageResult(result *QueryResult) *ClusterStorageResult {
-	return DecodeClusterResult(result)
-}
-
-type ClusterTotalResult = ClusterResult
-
-func DecodeClusterTotalResult(result *QueryResult) *ClusterTotalResult {
-	return DecodeClusterResult(result)
-}
-
-type ClusterNodesResult = ClusterResult
-
-func DecodeClusterNodesResult(result *QueryResult) *ClusterNodesResult {
-	return DecodeClusterResult(result)
-}
-
-type ClusterNodesByProviderResult struct {
-	Data []*util.Vector
-}
-
-func DecodeClusterNodesByProviderResult(result *QueryResult) *ClusterNodesByProviderResult {
-	return &ClusterNodesByProviderResult{
-		Data: result.Values,
-	}
-}
-
 type PodsResult struct {
 	UID       string
 	Cluster   string

core/pkg/source/future.go

@@ -1,12 +1,17 @@
 package source
 
+// ResultDecoder[T] is a function that decodes a `QueryResult` into a `*T` type.
 type ResultDecoder[T any] func(*QueryResult) *T
 
+// Future[T] is a async future/promise/task that will resolve into a []*T return or
+// error when awaited. This type provides a type-safe way of interfacing with `QueryResultsChan`
+// via a `ResultDecoder[T]` function.
 type Future[T any] struct {
 	decoder     ResultDecoder[T]
 	resultsChan QueryResultsChan
 }
 
+// NewFuture Creates a new `Future[T]` with the given `ResultDecoder[T]` and `QueryResultsChan`.
 func NewFuture[T any](decoder ResultDecoder[T], resultsChan QueryResultsChan) *Future[T] {
 	return &Future[T]{
 		decoder:     decoder,
@@ -31,6 +36,8 @@ func (f *Future[T]) awaitWith(errorCollector *QueryErrorCollector) ([]*T, error)
 	return decoded, nil
 }
 
+// Await blocks and waits for the `Future` to resolve, and returns the results if successful, or an error
+// otherwise.
 func (f *Future[T]) Await() ([]*T, error) {
 	results, err := f.resultsChan.Await()
 	if err != nil {

core/pkg/source/querygroup.go

@@ -1,19 +1,44 @@
 package source
 
-type QueryGroup struct {
-	errorCollector *QueryErrorCollector
-}
-
+// QueryGroupAsyncResult is a representation of a single async query in a group.
 type QueryGroupAsyncResult struct {
 	errorCollector *QueryErrorCollector
 	resultsChan    QueryResultsChan
 }
 
+// newQueryGroupAsyncResult creates a new QueryGroupAsyncResult with the given error collector and results channel.
+func newQueryGroupAsyncResult(collector *QueryErrorCollector, resultsChan QueryResultsChan) *QueryGroupAsyncResult {
+	return &QueryGroupAsyncResult{
+		errorCollector: collector,
+		resultsChan:    resultsChan,
+	}
+}
+
+// Await blocks and waits for the `QueryGroupAsyncResult` to resolve, and returns a slice of generic `QueryResult`
+// instances if successful, or an error otherwise.
+func (qgar *QueryGroupAsyncResult) Await() ([]*QueryResult, error) {
+	defer close(qgar.resultsChan)
+	result := <-qgar.resultsChan
+
+	q := result.Query
+	err := result.Error
+
+	if err != nil {
+		qgar.errorCollector.AppendError(&QueryError{Query: q, Error: err})
+		return nil, err
+	}
+
+	return result.Results, nil
+}
+
+// QueryGroupFuture[T] is a representation of a single async query in a group with a typed result.
 type QueryGroupFuture[T any] struct {
 	errorCollector *QueryErrorCollector
 	future         *Future[T]
 }
 
+// WithGroup creates a new QueryGroupFuture[T] instance with the given QueryGroup and Future instances.
+// This is the specific way to add a typed `Future[T]` to a `QueryGroup`.
 func WithGroup[T any](g *QueryGroup, f *Future[T]) *QueryGroupFuture[T] {
 	return &QueryGroupFuture[T]{
 		errorCollector: g.errorCollector,
@@ -21,10 +46,33 @@ func WithGroup[T any](g *QueryGroup, f *Future[T]) *QueryGroupFuture[T] {
 	}
 }
 
+// Await blocks and waits for the `QueryGroupFuture[T]` to resolve, and returns a slice of `*T` instances if successful,
+// or an error otherwise.
 func (qgf *QueryGroupFuture[T]) Await() ([]*T, error) {
 	return qgf.future.awaitWith(qgf.errorCollector)
 }
 
+// QueryGroup is a representation of multiple async queries. It provides a shared error collector
+// for all queries in the group.
+//
+// Example:
+//
+//	grp := NewQueryGroup()
+//	q1 := WithGroup(grp, QueryFoo())
+//	q2 := WithGroup(grp, QueryBar())
+//
+//	results1, _ := q1.Await()
+//	results2, _ := q2.Await()
+//
+//	if grp.HasErrors() {
+//		return grp.Error() // <-- error return type
+//	}
+type QueryGroup struct {
+	errorCollector *QueryErrorCollector
+}
+
+// NewQueryGroup creates a new QueryGroup instance which can be used to group non-typed async queries with
+// the `With(QueryResultsChan)` instance method, or with the package function `WithGroup[T](*QueryGroup, *Future[T])`
 func NewQueryGroup() *QueryGroup {
 	var errorCollector QueryErrorCollector
 
@@ -33,40 +81,30 @@ func NewQueryGroup() *QueryGroup {
 	}
 }
 
+// With adds the given `QueryResultsChan` to the QueryGroup instance and returns a `QueryGroupAsyncResult` instance to be
+// awaited
 func (qg *QueryGroup) With(resultsChan QueryResultsChan) *QueryGroupAsyncResult {
 	return newQueryGroupAsyncResult(qg.errorCollector, resultsChan)
 }
 
+// HasErrors returns true if any of the async queries in the group have errored. Note that all results must be awaited
+// in order to be checked for errors.
 func (qg *QueryGroup) HasErrors() bool {
 	return qg.errorCollector.IsError()
 }
 
+// Error returns nil if there were no errors in the group. Otherwise, it returns all of the errors that occurred grouped
+// into a single error.
 func (qg *QueryGroup) Error() error {
+	if !qg.errorCollector.IsError() {
+		var err error
+		return err
+	}
+
 	return qg.errorCollector
 }
 
+// Errors returns all of individual errors that occurred in the group.
 func (qg *QueryGroup) Errors() []*QueryError {
 	return qg.errorCollector.Errors()
 }
-
-func newQueryGroupAsyncResult(collector *QueryErrorCollector, resultsChan QueryResultsChan) *QueryGroupAsyncResult {
-	return &QueryGroupAsyncResult{
-		errorCollector: collector,
-		resultsChan:    resultsChan,
-	}
-}
-
-func (qgar *QueryGroupAsyncResult) Await() ([]*QueryResult, error) {
-	defer close(qgar.resultsChan)
-	result := <-qgar.resultsChan
-
-	q := result.Query
-	err := result.Error
-
-	if err != nil {
-		qgar.errorCollector.AppendError(&QueryError{Query: q, Error: err})
-		return nil, err
-	}
-
-	return result.Results, nil
-}

core/pkg/source/queryresult.go

@@ -24,6 +24,19 @@ func (qrc QueryResultsChan) Await() ([]*QueryResult, error) {
 	return results.Results, nil
 }
 
+// ResultKeys is a "configuration" struct that contains the keys/labels used to resolve labeled query
+// results. ResultKeys can be defined with every QueryResults instance if necessary, and alter the keys
+// used to fetch results when calling the following methods on QueryResults:
+//
+//	GetCluster()
+//	GetNamespace()
+//	GetNode()
+//	GetInstance()
+//	GetInstanceType()
+//	GetContainer()
+//	GetPod()
+//	GetProviderID()
+//	GetDevice()
 type ResultKeys struct {
 	ClusterKey      string
 	NamespaceKey    string
@@ -36,6 +49,7 @@ type ResultKeys struct {
 	DeviceKey       string
 }
 
+// DefaultResultKeys returns a new ResultKeys instance with typical default values.
 func DefaultResultKeys() *ResultKeys {
 	return &ResultKeys{
 		ClusterKey:      "cluster_id",
@@ -50,6 +64,8 @@ func DefaultResultKeys() *ResultKeys {
 	}
 }
 
+// ClusterKeyWithDefaults returns a new ResultKeys instance with the provided cluster key and the
+// rest of the keys set to their default values.
 func ClusterKeyWithDefaults(clusterKey string) *ResultKeys {
 	keys := DefaultResultKeys()
 	keys.ClusterKey = clusterKey

go.mod

@@ -51,7 +51,6 @@ require (
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/viper v1.8.1
 	github.com/stretchr/testify v1.9.0
-	go.etcd.io/bbolt v1.3.5
 	go.opentelemetry.io/otel v1.24.0
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
 	golang.org/x/oauth2 v0.23.0
@@ -63,7 +62,6 @@ require (
 	k8s.io/api v0.32.0
 	k8s.io/apimachinery v0.32.0
 	k8s.io/client-go v0.32.0
-	sigs.k8s.io/yaml v1.4.0
 )
 
 require (
@@ -72,6 +70,7 @@ require (
 	github.com/sony/gobreaker v0.5.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
 )
 
 require (
@@ -125,7 +124,7 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/flatbuffers v23.5.26+incompatible // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect

go.sum

@@ -290,8 +290,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -562,8 +562,6 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
 github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
 github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
 github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
-go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
-go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=

modules/prometheus-source/pkg/prom/datasource.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/julienschmidt/httprouter"
@@ -140,12 +139,15 @@ type PrometheusDataSource struct {
 	thanosConfig   *OpenCostThanosConfig
 	thanosClient   prometheus.Client
 	thanosContexts *ContextFactory
+
+	metricsQuerier *PrometheusMetricsQuerier
+	clusterMap     clusters.ClusterMap
 }
 
 // NewDefaultPrometheusDataSource creates and initializes a new `PrometheusDataSource` with configuration
 // parsed from environment variables. This function will block until a connection to prometheus is established,
 // or fails. It is recommended to run this function in a goroutine on a retry cycle.
-func NewDefaultPrometheusDataSource() (*PrometheusDataSource, error) {
+func NewDefaultPrometheusDataSource(clusterInfoProvider clusters.ClusterInfoProvider) (*PrometheusDataSource, error) {
 	config, err := NewOpenCostPrometheusConfigFromEnv()
 	if err != nil {
 		return nil, fmt.Errorf("failed to create prometheus config from env: %w", err)
@@ -160,11 +162,11 @@ func NewDefaultPrometheusDataSource() (*PrometheusDataSource, error) {
 		}
 	}
 
-	return NewPrometheusDataSource(config, thanosConfig)
+	return NewPrometheusDataSource(clusterInfoProvider, config, thanosConfig)
 }
 
 // NewPrometheusDataSource initializes clients for Prometheus and Thanos, and returns a new PrometheusDataSource.
-func NewPrometheusDataSource(promConfig *OpenCostPrometheusConfig, thanosConfig *OpenCostThanosConfig) (*PrometheusDataSource, error) {
+func NewPrometheusDataSource(infoProvider clusters.ClusterInfoProvider, promConfig *OpenCostPrometheusConfig, thanosConfig *OpenCostThanosConfig) (*PrometheusDataSource, error) {
 	promClient, err := NewPrometheusClient(promConfig.ServerEndpoint, promConfig.ClientConfig)
 	if err != nil {
 		return nil, fmt.Errorf("failed to build prometheus client: %w", err)
@@ -233,6 +235,35 @@ func NewPrometheusDataSource(promConfig *OpenCostPrometheusConfig, thanosConfig
 		}
 	}
 
+	// metadata creation for cluster info
+	thanosEnabled := thanosClient != nil
+	metadata := map[string]string{
+		clusters.ClusterInfoThanosEnabledKey: fmt.Sprintf("%t", thanosEnabled),
+	}
+	if thanosEnabled {
+		metadata[clusters.ClusterInfoThanosOffsetKey] = thanosConfig.Offset
+	}
+
+	// cluster info provider
+	clusterInfoProvider := clusters.NewClusterInfoDecorator(infoProvider, metadata)
+
+	var clusterMap clusters.ClusterMap
+	if thanosEnabled {
+		clusterMap = newPrometheusClusterMap(thanosContexts, clusterInfoProvider, 10*time.Minute)
+	} else {
+		clusterMap = newPrometheusClusterMap(promContexts, clusterInfoProvider, 5*time.Minute)
+	}
+
+	// create metrics querier implementation for prometheus and thanos
+	metricsQuerier := newPrometheusMetricsQuerier(
+		promConfig,
+		promClient,
+		promContexts,
+		thanosConfig,
+		thanosClient,
+		thanosContexts,
+	)
+
 	return &PrometheusDataSource{
 		promConfig:     promConfig,
 		promClient:     promClient,
@@ -240,6 +271,8 @@ func NewPrometheusDataSource(promConfig *OpenCostPrometheusConfig, thanosConfig
 		thanosConfig:   thanosConfig,
 		thanosClient:   thanosClient,
 		thanosContexts: thanosContexts,
+		metricsQuerier: metricsQuerier,
+		clusterMap:     clusterMap,
 	}, nil
 }
 
@@ -542,14 +575,6 @@ func (pds *PrometheusDataSource) ThanosContexts() *ContextFactory {
 	return pds.thanosContexts
 }
 
-func (pds *PrometheusDataSource) NewClusterMap(clusterInfoProvider clusters.ClusterInfoProvider) clusters.ClusterMap {
-	if pds.thanosClient != nil {
-		return newPrometheusClusterMap(pds.thanosContexts, clusterInfoProvider, 10*time.Minute)
-	}
-
-	return newPrometheusClusterMap(pds.promContexts, clusterInfoProvider, 5*time.Minute)
-}
-
 func (pds *PrometheusDataSource) RegisterEndPoints(router *httprouter.Router) {
 	// endpoints migrated from server
 	router.GET("/validatePrometheus", pds.prometheusMetadata)
@@ -573,1633 +598,18 @@ func (pds *PrometheusDataSource) RefreshInterval() time.Duration {
 	return pds.promConfig.ScrapeInterval
 }
 
-func (pds *PrometheusDataSource) BatchDuration() time.Duration {
-	return pds.promConfig.MaxQueryDuration
-}
-
-func (pds *PrometheusDataSource) Resolution() time.Duration {
-	return pds.promConfig.DataResolution
-}
-
-func (pds *PrometheusDataSource) MetaData() map[string]string {
-	thanosEnabled := pds.thanosClient != nil
-
-	metadata := map[string]string{
-		clusters.ClusterInfoThanosEnabledKey: fmt.Sprintf("%t", thanosEnabled),
-	}
-
-	if thanosEnabled {
-		metadata[clusters.ClusterInfoThanosOffsetKey] = pds.thanosConfig.Offset
-	}
-
-	return metadata
-}
-
-//--------------------------------------------------------------------------
-//  InstantMetricsQuerier
-//--------------------------------------------------------------------------
-
-func (pds *PrometheusDataSource) QueryPVPricePerGiBHour(start, end time.Time) *source.Future[source.PVPricePerGiBHourResult] {
-	const pvCostQuery = `avg(avg_over_time(pv_hourly_cost{%s}[%s])) by (%s, persistentvolume, volumename, provider_id)`
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryPVCost")
-	}
-
-	queryPVCost := fmt.Sprintf(pvCostQuery, pds.promConfig.ClusterFilter, durStr, pds.promConfig.ClusterLabel)
-
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodePVPricePerGiBHourResult, ctx.QueryAtTime(queryPVCost, end))
-}
-
-func (pds *PrometheusDataSource) QueryPVUsedAverage(start, end time.Time) *source.Future[source.PVUsedAvgResult] {
-	// `avg(avg_over_time(kubelet_volume_stats_used_bytes{%s}[%s])) by (%s, persistentvolumeclaim, namespace)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	const pvUsedAverageQuery = `avg(avg_over_time(kubelet_volume_stats_used_bytes{%s}[%s])) by (%s, persistentvolumeclaim, namespace)`
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryPVUsedAverage")
-	}
-
-	queryPVUsedAvg := fmt.Sprintf(pvUsedAverageQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodePVUsedAvgResult, ctx.QueryAtTime(queryPVUsedAvg, end))
-}
-
-func (pds *PrometheusDataSource) QueryPVUsedMax(start, end time.Time) *source.Future[source.PVUsedMaxResult] {
-	// `max(max_over_time(kubelet_volume_stats_used_bytes{%s}[%s])) by (%s, persistentvolumeclaim, namespace)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	const pvUsedMaxQuery = `max(max_over_time(kubelet_volume_stats_used_bytes{%s}[%s])) by (%s, persistentvolumeclaim, namespace)`
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryPVUsedMax")
-	}
-
-	queryPVUsedMax := fmt.Sprintf(pvUsedMaxQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodePVUsedMaxResult, ctx.QueryAtTime(queryPVUsedMax, end))
-}
-
-func (pds *PrometheusDataSource) QueryPVCInfo(start, end time.Time) *source.Future[source.PVCInfoResult] {
-	const queryFmtPVCInfo = `avg(kube_persistentvolumeclaim_info{volumename != "", %s}) by (persistentvolumeclaim, storageclass, volumename, namespace, %s)[%s:%s]`
-	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, resStr)
-
-	cfg := pds.promConfig
-	resolution := cfg.DataResolution
-	resStr := timeutil.DurationString(resolution)
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryPVCInfo")
-	}
-
-	queryPVCInfo := fmt.Sprintf(queryFmtPVCInfo, cfg.ClusterFilter, cfg.ClusterLabel, durStr, resStr)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodePVCInfoResult, ctx.QueryAtTime(queryPVCInfo, end))
-}
-
-func (pds *PrometheusDataSource) QueryPVActiveMinutes(start, end time.Time) *source.Future[source.PVActiveMinutesResult] {
-	const pvActiveMinsQuery = `avg(kube_persistentvolume_capacity_bytes{%s}) by (%s, persistentvolume)[%s:%dm]`
-	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
-
-	cfg := pds.promConfig
-	minsPerResolution := cfg.DataResolutionMinutes
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryPVActiveMinutes")
-	}
-
-	queryPVActiveMins := fmt.Sprintf(pvActiveMinsQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodePVActiveMinutesResult, ctx.QueryAtTime(queryPVActiveMins, end))
-}
-
-func (pds *PrometheusDataSource) QueryLocalStorageCost(start, end time.Time) *source.Future[source.LocalStorageCostResult] {
-	// `sum_over_time(sum(container_fs_limit_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`
-	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
-
-	const localStorageCostQuery = `sum_over_time(sum(container_fs_limit_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`
-
-	cfg := pds.promConfig
-	resolution := cfg.DataResolution
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryLocalStorageCost")
-	}
-
-	//Ensuring if data resolution is less than 60s default it to 1m
-	var minsPerResolution int
-	if minsPerResolution = int(resolution.Minutes()); int(resolution.Minutes()) == 0 {
-		minsPerResolution = 1
-		log.DedupedWarningf(3, "QueryLocalStorageCost: Configured resolution (%d seconds) is below the 60 seconds threshold. Overriding with 1 minute.", int(resolution.Seconds()))
-	}
-
-	// hourlyToCumulative is a scaling factor that, when multiplied by an
-	// hourly value, converts it to a cumulative value; i.e. [$/hr] *
-	// [min/res]*[hr/min] = [$/res]
-	hourlyToCumulative := float64(minsPerResolution) * (1.0 / 60.0)
-	costPerGBHr := 0.04 / 730.0
-
-	queryLocalStorageCost := fmt.Sprintf(localStorageCostQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeLocalStorageCostResult, ctx.QueryAtTime(queryLocalStorageCost, end))
-}
-
-func (pds *PrometheusDataSource) QueryLocalStorageUsedCost(start, end time.Time) *source.Future[source.LocalStorageUsedCostResult] {
-	// `sum_over_time(sum(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`
-	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
-
-	const localStorageUsedCostQuery = `sum_over_time(sum(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`
-
-	cfg := pds.promConfig
-	minsPerResolution := cfg.DataResolutionMinutes
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryLocalStorageUsedCost")
-	}
-
-	// hourlyToCumulative is a scaling factor that, when multiplied by an
-	// hourly value, converts it to a cumulative value; i.e. [$/hr] *
-	// [min/res]*[hr/min] = [$/res]
-	hourlyToCumulative := float64(minsPerResolution) * (1.0 / 60.0)
-	costPerGBHr := 0.04 / 730.0
-
-	queryLocalStorageUsedCost := fmt.Sprintf(localStorageUsedCostQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeLocalStorageUsedCostResult, ctx.QueryAtTime(queryLocalStorageUsedCost, end))
-}
-
-func (pds *PrometheusDataSource) QueryLocalStorageUsedAvg(start, end time.Time) *source.Future[source.LocalStorageUsedAvgResult] {
-	// `avg(sum(avg_over_time(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}[%s])) by (instance, device, %s, job)) by (instance, device, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel(), env.GetPromClusterLabel())
-
-	const localStorageUsedAvgQuery = `avg(sum(avg_over_time(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}[%s])) by (instance, device, %s, job)) by (instance, device, %s)`
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryLocalStorageUsedAvg")
-	}
-
-	queryLocalStorageUsedAvg := fmt.Sprintf(localStorageUsedAvgQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeLocalStorageUsedAvgResult, ctx.QueryAtTime(queryLocalStorageUsedAvg, end))
-}
-
-func (pds *PrometheusDataSource) QueryLocalStorageUsedMax(start, end time.Time) *source.Future[source.LocalStorageUsedMaxResult] {
-	// `max(sum(max_over_time(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}[%s])) by (instance, device, %s, job)) by (instance, device, %s)`
-	//  env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel(), env.GetPromClusterLabel())
-	const localStorageUsedMaxQuery = `max(sum(max_over_time(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}[%s])) by (instance, device, %s, job)) by (instance, device, %s)`
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryLocalStorageUsedMax")
-	}
-
-	queryLocalStorageUsedMax := fmt.Sprintf(localStorageUsedMaxQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeLocalStorageUsedMaxResult, ctx.QueryAtTime(queryLocalStorageUsedMax, end))
-}
-
-func (pds *PrometheusDataSource) QueryLocalStorageBytes(start, end time.Time) *source.Future[source.LocalStorageBytesResult] {
-	// `avg_over_time(sum(container_fs_limit_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, %s)[%s:%dm])`
-	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
-
-	const localStorageBytesQuery = `avg_over_time(sum(container_fs_limit_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, %s)[%s:%dm])`
-
-	cfg := pds.promConfig
-	minsPerResolution := cfg.DataResolutionMinutes
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryLocalStorageBytes")
-	}
-
-	queryLocalStorageBytes := fmt.Sprintf(localStorageBytesQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeLocalStorageBytesResult, ctx.QueryAtTime(queryLocalStorageBytes, end))
-}
-
-func (pds *PrometheusDataSource) QueryLocalStorageActiveMinutes(start, end time.Time) *source.Future[source.LocalStorageActiveMinutesResult] {
-	// `count(node_total_hourly_cost{%s}) by (%s, node)[%s:%dm]`
-	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
-
-	const localStorageActiveMinutesQuery = `count(node_total_hourly_cost{%s}) by (%s, node)[%s:%dm]`
-
-	cfg := pds.promConfig
-	minsPerResolution := cfg.DataResolutionMinutes
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryLocalStorageActiveMinutes")
-	}
-
-	queryLocalStorageActiveMins := fmt.Sprintf(localStorageActiveMinutesQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeLocalStorageActiveMinutesResult, ctx.QueryAtTime(queryLocalStorageActiveMins, end))
-}
-
-func (pds *PrometheusDataSource) QueryLocalStorageBytesByProvider(provider string, start, end time.Time) *source.Future[source.LocalStorageBytesByProviderResult] {
-	var localStorageBytesQuery string
-
-	key := strings.ToLower(provider)
-	if f, ok := providerStorageQueries[key]; ok {
-		localStorageBytesQuery = f(pds.promConfig, start, end, false, false)
-	} else {
-		localStorageBytesQuery = ""
-	}
-
-	if localStorageBytesQuery == "" {
-		return newEmptyResult(source.DecodeLocalStorageBytesByProviderResult)
-	}
-
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeLocalStorageBytesByProviderResult, ctx.QueryAtTime(localStorageBytesQuery, end))
-}
-
-func (pds *PrometheusDataSource) QueryLocalStorageUsedByProvider(provider string, start, end time.Time) *source.Future[source.LocalStorageUsedByProviderResult] {
-	var localStorageUsedQuery string
-
-	key := strings.ToLower(provider)
-	if f, ok := providerStorageQueries[key]; ok {
-		localStorageUsedQuery = f(pds.promConfig, start, end, false, true)
-	} else {
-		localStorageUsedQuery = ""
-	}
-
-	if localStorageUsedQuery == "" {
-		return newEmptyResult(source.DecodeLocalStorageUsedByProviderResult)
-	}
-
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeLocalStorageUsedByProviderResult, ctx.QueryAtTime(localStorageUsedQuery, end))
-}
-
-func (pds *PrometheusDataSource) QueryNodeCPUCoresCapacity(start, end time.Time) *source.Future[source.NodeCPUCoresCapacityResult] {
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	const nodeCPUCoresCapacityQuery = `avg(avg_over_time(kube_node_status_capacity_cpu_cores{%s}[%s])) by (%s, node)`
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNodeCPUCoresCapacity")
-	}
-
-	queryNodeCPUCoresCapacity := fmt.Sprintf(nodeCPUCoresCapacityQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeNodeCPUCoresCapacityResult, ctx.QueryAtTime(queryNodeCPUCoresCapacity, end))
-}
-
-func (pds *PrometheusDataSource) QueryNodeCPUCoresAllocatable(start, end time.Time) *source.Future[source.NodeCPUCoresAllocatableResult] {
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	const nodeCPUCoresAllocatableQuery = `avg(avg_over_time(kube_node_status_allocatable_cpu_cores{%s}[%s])) by (%s, node)`
-	// `avg(avg_over_time(container_cpu_allocation{container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNodeCPUCoresAllocatable")
-	}
-
-	queryNodeCPUCoresAllocatable := fmt.Sprintf(nodeCPUCoresAllocatableQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeNodeCPUCoresAllocatableResult, ctx.QueryAtTime(queryNodeCPUCoresAllocatable, end))
-}
-
-func (pds *PrometheusDataSource) QueryNodeRAMBytesCapacity(start, end time.Time) *source.Future[source.NodeRAMBytesCapacityResult] {
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	const nodeRAMBytesCapacityQuery = `avg(avg_over_time(kube_node_status_capacity_memory_bytes{%s}[%s])) by (%s, node)`
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNodeRAMBytesCapacity")
-	}
-
-	queryNodeRAMBytesCapacity := fmt.Sprintf(nodeRAMBytesCapacityQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeNodeRAMBytesCapacityResult, ctx.QueryAtTime(queryNodeRAMBytesCapacity, end))
-}
-
-func (pds *PrometheusDataSource) QueryNodeRAMBytesAllocatable(start, end time.Time) *source.Future[source.NodeRAMBytesAllocatableResult] {
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	const nodeRAMBytesAllocatableQuery = `avg(avg_over_time(kube_node_status_allocatable_memory_bytes{%s}[%s])) by (%s, node)`
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNodeRAMBytesAllocatable")
-	}
-
-	queryNodeRAMBytesAllocatable := fmt.Sprintf(nodeRAMBytesAllocatableQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeNodeRAMBytesAllocatableResult, ctx.QueryAtTime(queryNodeRAMBytesAllocatable, end))
-}
-
-func (pds *PrometheusDataSource) QueryNodeGPUCount(start, end time.Time) *source.Future[source.NodeGPUCountResult] {
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	const nodeGPUCountQuery = `avg(avg_over_time(node_gpu_count{%s}[%s])) by (%s, node, provider_id)`
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNodeGPUCount")
-	}
-
-	queryNodeGPUCount := fmt.Sprintf(nodeGPUCountQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeNodeGPUCountResult, ctx.QueryAtTime(queryNodeGPUCount, end))
-}
-
-func (pds *PrometheusDataSource) QueryNodeLabels(start, end time.Time) *source.Future[source.NodeLabelsResult] {
-	const labelsQuery = `avg_over_time(kube_node_labels{%s}[%s])`
-	// env.GetPromClusterFilter(), durStr
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNodeLabels")
-	}
-
-	queryLabels := fmt.Sprintf(labelsQuery, cfg.ClusterFilter, durStr)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeNodeLabelsResult, ctx.QueryAtTime(queryLabels, end))
-}
-
-func (pds *PrometheusDataSource) QueryNodeActiveMinutes(start, end time.Time) *source.Future[source.NodeActiveMinutesResult] {
-	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
-
-	const activeMinsQuery = `avg(node_total_hourly_cost{%s}) by (node, %s, provider_id)[%s:%dm]`
-
-	cfg := pds.promConfig
-	minsPerResolution := cfg.DataResolutionMinutes
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNodeActiveMinutes")
-	}
-
-	queryActiveMins := fmt.Sprintf(activeMinsQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeNodeActiveMinutesResult, ctx.QueryAtTime(queryActiveMins, end))
-}
-
-func (pds *PrometheusDataSource) QueryNodeCPUModeTotal(start, end time.Time) *source.Future[source.NodeCPUModeTotalResult] {
-	// env.GetPromClusterFilter(), durStr, minsPerResolution, env.GetPromClusterLabel())
-
-	const nodeCPUModeTotalQuery = `sum(rate(node_cpu_seconds_total{%s}[%s:%dm])) by (kubernetes_node, %s, mode)`
-
-	cfg := pds.promConfig
-	minsPerResolution := cfg.DataResolutionMinutes
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNodeCPUModeTotal")
-	}
-
-	queryCPUModeTotal := fmt.Sprintf(nodeCPUModeTotalQuery, cfg.ClusterFilter, durStr, minsPerResolution, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeNodeCPUModeTotalResult, ctx.QueryAtTime(queryCPUModeTotal, end))
-}
-
-func (pds *PrometheusDataSource) QueryNodeCPUModePercent(start, end time.Time) *source.Future[source.NodeCPUModePercentResult] {
-	const fmtQueryCPUModePct = `
-		sum(rate(node_cpu_seconds_total{%s}[%s])) by (%s, mode) / ignoring(mode)
-		group_left sum(rate(node_cpu_seconds_total{%s}[%s])) by (%s)
-	`
-	// env.GetPromClusterFilter(), windowStr, env.GetPromClusterLabel(), env.GetPromClusterFilter(), windowStr, fmtOffset, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNodeCPUModePercent")
-	}
-
-	queryCPUModePct := fmt.Sprintf(fmtQueryCPUModePct, cfg.ClusterFilter, durStr, cfg.ClusterLabel, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeNodeCPUModePercentResult, ctx.QueryAtTime(queryCPUModePct, end))
-}
-
-func (pds *PrometheusDataSource) QueryNodeRAMSystemPercent(start, end time.Time) *source.Future[source.NodeRAMSystemPercentResult] {
-	// env.GetPromClusterFilter(), durStr, minsPerResolution, env.GetPromClusterLabel(), env.GetPromClusterFilter(), durStr, minsPerResolution, env.GetPromClusterLabel(), env.GetPromClusterLabel())
-
-	const nodeRAMSystemPctQuery = `sum(sum_over_time(container_memory_working_set_bytes{container_name!="POD",container_name!="",namespace="kube-system", %s}[%s:%dm])) by (instance, %s) / avg(label_replace(sum(sum_over_time(kube_node_status_capacity_memory_bytes{%s}[%s:%dm])) by (node, %s), "instance", "$1", "node", "(.*)")) by (instance, %s)`
-
-	cfg := pds.promConfig
-	minsPerResolution := cfg.DataResolutionMinutes
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNodeRAMSystemPercent")
-	}
-
-	queryRAMSystemPct := fmt.Sprintf(nodeRAMSystemPctQuery, cfg.ClusterFilter, durStr, minsPerResolution, cfg.ClusterLabel, cfg.ClusterFilter, durStr, minsPerResolution, cfg.ClusterLabel, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeNodeRAMSystemPercentResult, ctx.QueryAtTime(queryRAMSystemPct, end))
-}
-
-func (pds *PrometheusDataSource) QueryNodeRAMUserPercent(start, end time.Time) *source.Future[source.NodeRAMUserPercentResult] {
-	// env.GetPromClusterFilter(), durStr, minsPerResolution, env.GetPromClusterLabel(), env.GetPromClusterFilter(), durStr, minsPerResolution, env.GetPromClusterLabel(), env.GetPromClusterLabel())
-
-	const nodeRAMUserPctQuery = `sum(sum_over_time(container_memory_working_set_bytes{container_name!="POD",container_name!="",namespace!="kube-system", %s}[%s:%dm])) by (instance, %s) / avg(label_replace(sum(sum_over_time(kube_node_status_capacity_memory_bytes{%s}[%s:%dm])) by (node, %s), "instance", "$1", "node", "(.*)")) by (instance, %s)`
-
-	cfg := pds.promConfig
-	minsPerResolution := cfg.DataResolutionMinutes
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNodeRAMUserPercent")
-	}
-
-	queryRAMUserPct := fmt.Sprintf(nodeRAMUserPctQuery, cfg.ClusterFilter, durStr, minsPerResolution, cfg.ClusterLabel, cfg.ClusterFilter, durStr, minsPerResolution, cfg.ClusterLabel, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeNodeRAMUserPercentResult, ctx.QueryAtTime(queryRAMUserPct, end))
-}
-
-func (pds *PrometheusDataSource) QueryLBPricePerHr(start, end time.Time) *source.Future[source.LBPricePerHrResult] {
-	const queryFmtLBCostPerHr = `avg(avg_over_time(kubecost_load_balancer_cost{%s}[%s])) by (namespace, service_name, ingress_ip, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryLBPricePerHr")
-	}
-
-	queryLBCostPerHr := fmt.Sprintf(queryFmtLBCostPerHr, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeLBPricePerHrResult, ctx.QueryAtTime(queryLBCostPerHr, end))
-}
-
-func (pds *PrometheusDataSource) QueryLBActiveMinutes(start, end time.Time) *source.Future[source.LBActiveMinutesResult] {
-	const lbActiveMinutesQuery = `avg(kubecost_load_balancer_cost{%s}) by (namespace, service_name, %s, ingress_ip)[%s:%dm]`
-	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
-
-	cfg := pds.promConfig
-	minsPerResolution := cfg.DataResolutionMinutes
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryLBActiveMinutes")
-	}
-
-	queryLBActiveMins := fmt.Sprintf(lbActiveMinutesQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeLBActiveMinutesResult, ctx.QueryAtTime(queryLBActiveMins, end))
-}
-
-func (pds *PrometheusDataSource) QueryClusterManagementDuration(start, end time.Time) *source.Future[source.ClusterManagementDurationResult] {
-	const clusterManagementDurationQuery = `avg(kubecost_cluster_management_cost{%s}) by (%s, provisioner_name)[%s:%dm]`
-
-	cfg := pds.promConfig
-	minsPerResolution := cfg.DataResolutionMinutes
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryClusterManagementDuration")
-	}
-
-	queryClusterManagementDuration := fmt.Sprintf(clusterManagementDurationQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeClusterManagementDurationResult, ctx.QueryAtTime(queryClusterManagementDuration, end))
-}
-
-func (pds *PrometheusDataSource) QueryClusterManagementPricePerHr(start, end time.Time) *source.Future[source.ClusterManagementPricePerHrResult] {
-	const clusterManagementCostQuery = `avg(avg_over_time(kubecost_cluster_management_cost{%s}[%s])) by (%s, provisioner_name)`
-	// env.GetPromClusterFilter(), durationStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryClusterManagementCost")
-	}
-
-	queryClusterManagementCost := fmt.Sprintf(clusterManagementCostQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeClusterManagementPricePerHrResult, ctx.QueryAtTime(queryClusterManagementCost, end))
-}
-
-func (pds *PrometheusDataSource) QueryDataCount(start, end time.Time) *source.Future[source.DataCountResult] {
-	const fmtQueryDataCount = `
-		count_over_time(sum(kube_node_status_capacity_cpu_cores{%s}) by (%s)[%s:%dm]) * %d
-	`
-	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), windowStr, minsPerResolution, minsPerResolution)
-
-	cfg := pds.promConfig
-	minsPerResolution := cfg.DataResolutionMinutes
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryDataCount")
-	}
-
-	queryDataCount := fmt.Sprintf(fmtQueryDataCount, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution, minsPerResolution)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeDataCountResult, ctx.QueryAtTime(queryDataCount, end))
-}
-
-func (pds *PrometheusDataSource) QueryTotalGPU(start, end time.Time) *source.Future[source.TotalGPUResult] {
-	const fmtQueryTotalGPU = `
-		sum(
-			sum_over_time(node_gpu_hourly_cost{%s}[%s:%dm]) * %f
-		) by (%s)
-	`
-	// env.GetPromClusterFilter(), windowStr, minsPerResolution, fmtOffset, hourlyToCumulative, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-	minsPerResolution := cfg.DataResolutionMinutes
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryTotalGPU")
-	}
-
-	// hourlyToCumulative is a scaling factor that, when multiplied by an hourly
-	// value, converts it to a cumulative value; i.e.
-	// [$/hr] * [min/res]*[hr/min] = [$/res]
-	hourlyToCumulative := float64(minsPerResolution) * (1.0 / 60.0)
-
-	queryTotalGPU := fmt.Sprintf(fmtQueryTotalGPU, cfg.ClusterFilter, durStr, minsPerResolution, hourlyToCumulative, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeTotalGPUResult, ctx.QueryAtTime(queryTotalGPU, end))
-}
-
-func (pds *PrometheusDataSource) QueryTotalCPU(start, end time.Time) *source.Future[source.TotalCPUResult] {
-	const fmtQueryTotalCPU = `
-		sum(
-			sum_over_time(avg(kube_node_status_capacity_cpu_cores{%s}) by (node, %s)[%s:%dm]) *
-			avg(avg_over_time(node_cpu_hourly_cost{%s}[%s:%dm])) by (node, %s) * %f
-		) by (%s)
-	`
-	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), windowStr, minsPerResolution, fmtOffset, env.GetPromClusterFilter(), windowStr, minsPerResolution, fmtOffset, env.GetPromClusterLabel(), hourlyToCumulative, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-	minsPerResolution := cfg.DataResolutionMinutes
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryTotalCPU")
-	}
-
-	// hourlyToCumulative is a scaling factor that, when multiplied by an hourly
-	// value, converts it to a cumulative value; i.e.
-	// [$/hr] * [min/res]*[hr/min] = [$/res]
-	hourlyToCumulative := float64(minsPerResolution) * (1.0 / 60.0)
-
-	queryTotalCPU := fmt.Sprintf(fmtQueryTotalCPU, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution, cfg.ClusterFilter, durStr, minsPerResolution, cfg.ClusterLabel, hourlyToCumulative, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeTotalCPUResult, ctx.QueryAtTime(queryTotalCPU, end))
+func (pds *PrometheusDataSource) Metrics() source.MetricsQuerier {
+	return pds.metricsQuerier
 }
 
-func (pds *PrometheusDataSource) QueryTotalRAM(start, end time.Time) *source.Future[source.TotalRAMResult] {
-	const fmtQueryTotalRAM = `
-		sum(
-			sum_over_time(avg(kube_node_status_capacity_memory_bytes{%s}) by (node, %s)[%s:%dm]) / 1024 / 1024 / 1024 *
-			avg(avg_over_time(node_ram_hourly_cost{%s}[%s:%dm])) by (node, %s) * %f
-		) by (%s)
-	`
-	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), windowStr, minsPerResolution, env.GetPromClusterFilter(), windowStr, minsPerResolution, env.GetPromClusterLabel(), hourlyToCumulative, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-	minsPerResolution := cfg.DataResolutionMinutes
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryTotalRAM")
-	}
-
-	// hourlyToCumulative is a scaling factor that, when multiplied by an hourly
-	// value, converts it to a cumulative value; i.e.
-	// [$/hr] * [min/res]*[hr/min] = [$/res]
-	hourlyToCumulative := float64(minsPerResolution) * (1.0 / 60.0)
-
-	queryTotalRAM := fmt.Sprintf(fmtQueryTotalRAM, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution, cfg.ClusterFilter, durStr, minsPerResolution, cfg.ClusterLabel, hourlyToCumulative, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeTotalRAMResult, ctx.QueryAtTime(queryTotalRAM, end))
+func (pds *PrometheusDataSource) ClusterMap() clusters.ClusterMap {
+	return pds.clusterMap
 }
 
-func (pds *PrometheusDataSource) QueryTotalStorage(start, end time.Time) *source.Future[source.TotalStorageResult] {
-	const fmtQueryTotalStorage = `
-		sum(
-			sum_over_time(avg(kube_persistentvolume_capacity_bytes{%s}) by (persistentvolume, %s)[%s:%dm]) / 1024 / 1024 / 1024 *
-			avg(avg_over_time(pv_hourly_cost{%s}[%s:%dm])) by (persistentvolume, %s) * %f
-		) by (%s)
-	`
-	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), windowStr, minsPerResolution, env.GetPromClusterFilter(), windowStr, minsPerResolution, env.GetPromClusterLabel(), hourlyToCumulative, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-	minsPerResolution := cfg.DataResolutionMinutes
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryTotalStorage")
-	}
-
-	// hourlyToCumulative is a scaling factor that, when multiplied by an hourly
-	// value, converts it to a cumulative value; i.e.
-	// [$/hr] * [min/res]*[hr/min] = [$/res]
-	hourlyToCumulative := float64(minsPerResolution) * (1.0 / 60.0)
-
-	queryTotalStorage := fmt.Sprintf(fmtQueryTotalStorage, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution, cfg.ClusterFilter, durStr, minsPerResolution, cfg.ClusterLabel, hourlyToCumulative, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeTotalStorageResult, ctx.QueryAtTime(queryTotalStorage, end))
+func (pds *PrometheusDataSource) BatchDuration() time.Duration {
+	return pds.promConfig.MaxQueryDuration
 }
 
-func (pds *PrometheusDataSource) QueryClusterCores(start, end time.Time, step time.Duration) *source.Future[source.ClusterCoresResult] {
-	const queryClusterCores = `sum(
-		avg(avg_over_time(kube_node_status_capacity_cpu_cores{%s}[%s])) by (node, %s) * avg(avg_over_time(node_cpu_hourly_cost{%s}[%s])) by (node, %s) * 730 +
-		avg(avg_over_time(node_gpu_hourly_cost{%s}[%s])) by (node, %s) * 730
-	  ) by (%s)`
-	// env.GetPromClusterFilter(), fmtWindow, env.GetPromClusterLabel(), env.GetPromClusterFilter(), fmtWindow, env.GetPromClusterLabel(), env.GetPromClusterFilter(), fmtWindow,  env.GetPromClusterLabel(), env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-	durStr := timeutil.DurationString(step)
-
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryClusterCores")
-	}
-
-	clusterCoresQuery := fmt.Sprintf(queryClusterCores, cfg.ClusterFilter, durStr, cfg.ClusterLabel, cfg.ClusterFilter, durStr, cfg.ClusterLabel, cfg.ClusterFilter, durStr, cfg.ClusterLabel, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeClusterCoresResult, ctx.QueryRange(clusterCoresQuery, start, end, step))
-}
-
-func (pds *PrometheusDataSource) QueryClusterRAM(start, end time.Time, step time.Duration) *source.Future[source.ClusterRAMResult] {
-	const queryClusterRAM = `sum(
-		avg(avg_over_time(kube_node_status_capacity_memory_bytes{%s}[%s])) by (node, %s) / 1024 / 1024 / 1024 * avg(avg_over_time(node_ram_hourly_cost{%s}[%s])) by (node, %s) * 730
-	  ) by (%s)`
-	//  env.GetPromClusterFilter(), fmtWindow, env.GetPromClusterLabel(), env.GetPromClusterFilter(), fmtWindow, env.GetPromClusterLabel(), env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-	durStr := timeutil.DurationString(step)
-
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryClusterRAM")
-	}
-
-	clusterRAMQuery := fmt.Sprintf(queryClusterRAM, cfg.ClusterFilter, durStr, cfg.ClusterLabel, cfg.ClusterFilter, durStr, cfg.ClusterLabel, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeClusterRAMResult, ctx.QueryRange(clusterRAMQuery, start, end, step))
-}
-
-func (pds *PrometheusDataSource) QueryClusterStorage(start, end time.Time, step time.Duration) *source.Future[source.ClusterStorageResult] {
-	return pds.QueryClusterStorageByProvider("", start, end, step)
-}
-
-func (pds *PrometheusDataSource) QueryClusterStorageByProvider(provider string, start, end time.Time, step time.Duration) *source.Future[source.ClusterStorageResult] {
-	const queryStorage = `sum(
-		avg(avg_over_time(pv_hourly_cost{%s}[%s])) by (persistentvolume, %s) * 730
-		* avg(avg_over_time(kube_persistentvolume_capacity_bytes{%s}[%s])) by (persistentvolume, %s) / 1024 / 1024 / 1024
-	  ) by (%s) %s`
-	// env.GetPromClusterFilter(), fmtWindow, env.GetPromClusterLabel(), env.GetPromClusterFilter(), fmtWindow, env.GetPromClusterLabel(), env.GetPromClusterLabel(), localStorageQuery)
-
-	var localStorageQuery string
-	if provider != "" {
-		key := strings.ToLower(provider)
-		if f, ok := providerStorageQueries[key]; ok {
-			localStorageQuery = f(pds.promConfig, start, end, true, false)
-		} else {
-			localStorageQuery = ""
-		}
-	}
-
-	if localStorageQuery != "" {
-		localStorageQuery = fmt.Sprintf(" + %s", localStorageQuery)
-	}
-
-	cfg := pds.promConfig
-	durStr := timeutil.DurationString(step)
-
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryClusterCores")
-	}
-
-	clusterStorageQuery := fmt.Sprintf(queryStorage, cfg.ClusterFilter, durStr, cfg.ClusterLabel, cfg.ClusterFilter, durStr, cfg.ClusterLabel, cfg.ClusterLabel, localStorageQuery)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeClusterStorageResult, ctx.QueryRange(clusterStorageQuery, start, end, step))
-}
-
-func (pds *PrometheusDataSource) QueryClusterTotal(start, end time.Time, step time.Duration) *source.Future[source.ClusterTotalResult] {
-	return pds.QueryClusterTotalByProvider("", start, end, step)
-}
-
-func (pds *PrometheusDataSource) QueryClusterTotalByProvider(provider string, start, end time.Time, step time.Duration) *source.Future[source.ClusterTotalResult] {
-	const queryTotal = `sum(avg(node_total_hourly_cost{%s}) by (node, %s)) * 730 +
-	  sum(
-		avg(avg_over_time(pv_hourly_cost{%s}[1h])) by (persistentvolume, %s) * 730
-		* avg(avg_over_time(kube_persistentvolume_capacity_bytes{%s}[1h])) by (persistentvolume, %s) / 1024 / 1024 / 1024
-	  ) by (%s) %s`
-
-	var localStorageQuery string
-	if provider != "" {
-		key := strings.ToLower(provider)
-		if f, ok := providerStorageQueries[key]; ok {
-			localStorageQuery = f(pds.promConfig, start, end, true, false)
-		} else {
-			localStorageQuery = ""
-		}
-	}
-
-	if localStorageQuery != "" {
-		localStorageQuery = fmt.Sprintf(" + %s", localStorageQuery)
-	}
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(step)
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryClusterTotalByProvider")
-	}
-
-	clusterTotalQuery := fmt.Sprintf(queryTotal, cfg.ClusterFilter, cfg.ClusterLabel, cfg.ClusterFilter, cfg.ClusterLabel, cfg.ClusterFilter, cfg.ClusterLabel, cfg.ClusterLabel, localStorageQuery)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeClusterTotalResult, ctx.QueryRange(clusterTotalQuery, start, end, step))
-}
-
-func (pds *PrometheusDataSource) QueryClusterNodes(start, end time.Time, step time.Duration) *source.Future[source.ClusterNodesResult] {
-	return pds.QueryClusterNodesByProvider("", start, end, step)
-}
-
-func (pds *PrometheusDataSource) QueryClusterNodesByProvider(provider string, start, end time.Time, step time.Duration) *source.Future[source.ClusterNodesResult] {
-	const queryNodes = `sum(avg(node_total_hourly_cost{%s}) by (node, %s)) * 730 %s`
-	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), localStorageQuery)
-
-	var localStorageQuery string
-	if provider != "" {
-		key := strings.ToLower(provider)
-		if f, ok := providerStorageQueries[key]; ok {
-			localStorageQuery = f(pds.promConfig, start, end, true, false)
-		} else {
-			localStorageQuery = ""
-		}
-	}
-
-	if localStorageQuery != "" {
-		localStorageQuery = fmt.Sprintf(" + %s", localStorageQuery)
-	}
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(step)
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryClusterNodesByProvider")
-	}
-
-	clusterNodesCostQuery := fmt.Sprintf(queryNodes, cfg.ClusterFilter, cfg.ClusterLabel, localStorageQuery)
-	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
-	return source.NewFuture(source.DecodeClusterNodesResult, ctx.QueryRange(clusterNodesCostQuery, start, end, step))
-}
-
-// AllocationMetricQuerier
-
-func (pds *PrometheusDataSource) QueryPods(start, end time.Time) *source.Future[source.PodsResult] {
-	const queryFmtPods = `avg(kube_pod_container_status_running{%s} != 0) by (pod, namespace, %s)[%s:%s]`
-	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, resStr)
-
-	cfg := pds.promConfig
-	resolution := cfg.DataResolution
-	resStr := timeutil.DurationString(resolution)
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryPods")
-	}
-
-	queryPods := fmt.Sprintf(queryFmtPods, cfg.ClusterFilter, cfg.ClusterLabel, durStr, resStr)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodePodsResult, ctx.QueryAtTime(queryPods, end))
-}
-
-func (pds *PrometheusDataSource) QueryPodsUID(start, end time.Time) *source.Future[source.PodsResult] {
-	const queryFmtPodsUID = `avg(kube_pod_container_status_running{%s} != 0) by (pod, namespace, uid, %s)[%s:%s]`
-	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, resStr)
-
-	cfg := pds.promConfig
-	resolution := cfg.DataResolution
-	resStr := timeutil.DurationString(resolution)
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryPodsUID")
-	}
-
-	queryPodsUID := fmt.Sprintf(queryFmtPodsUID, cfg.ClusterFilter, cfg.ClusterLabel, durStr, resStr)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodePodsResult, ctx.QueryAtTime(queryPodsUID, end))
-}
-
-func (pds *PrometheusDataSource) QueryRAMBytesAllocated(start, end time.Time) *source.Future[source.RAMBytesAllocatedResult] {
-	const queryFmtRAMBytesAllocated = `avg(avg_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s, provider_id)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryRAMBytesAllocated")
-	}
-
-	queryRAMBytesAllocated := fmt.Sprintf(queryFmtRAMBytesAllocated, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeRAMBytesAllocatedResult, ctx.QueryAtTime(queryRAMBytesAllocated, end))
-}
-
-func (pds *PrometheusDataSource) QueryRAMRequests(start, end time.Time) *source.Future[source.RAMRequestsResult] {
-	const queryFmtRAMRequests = `avg(avg_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte", container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryRAMRequests")
-	}
-
-	queryRAMRequests := fmt.Sprintf(queryFmtRAMRequests, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeRAMRequestsResult, ctx.QueryAtTime(queryRAMRequests, end))
-}
-
-func (pds *PrometheusDataSource) QueryRAMUsageAvg(start, end time.Time) *source.Future[source.RAMUsageAvgResult] {
-	const queryFmtRAMUsageAvg = `avg(avg_over_time(container_memory_working_set_bytes{container!="", container_name!="POD", container!="POD", %s}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryRAMUsageAvg")
-	}
-
-	queryRAMUsageAvg := fmt.Sprintf(queryFmtRAMUsageAvg, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeRAMUsageAvgResult, ctx.QueryAtTime(queryRAMUsageAvg, end))
-}
-
-func (pds *PrometheusDataSource) QueryRAMUsageMax(start, end time.Time) *source.Future[source.RAMUsageMaxResult] {
-	const queryFmtRAMUsageMax = `max(max_over_time(container_memory_working_set_bytes{container!="", container_name!="POD", container!="POD", %s}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryRAMUsageMax")
-	}
-
-	queryRAMUsageMax := fmt.Sprintf(queryFmtRAMUsageMax, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeRAMUsageMaxResult, ctx.QueryAtTime(queryRAMUsageMax, end))
-}
-
-func (pds *PrometheusDataSource) QueryCPUCoresAllocated(start, end time.Time) *source.Future[source.CPUCoresAllocatedResult] {
-	const queryFmtCPUCoresAllocated = `avg(avg_over_time(container_cpu_allocation{container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryCPUCoresAllocated")
-	}
-
-	queryCPUCoresAllocated := fmt.Sprintf(queryFmtCPUCoresAllocated, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeCPUCoresAllocatedResult, ctx.QueryAtTime(queryCPUCoresAllocated, end))
-}
-
-func (pds *PrometheusDataSource) QueryCPURequests(start, end time.Time) *source.Future[source.CPURequestsResult] {
-	const queryFmtCPURequests = `avg(avg_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryCPURequests")
-	}
-
-	queryCPURequests := fmt.Sprintf(queryFmtCPURequests, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeCPURequestsResult, ctx.QueryAtTime(queryCPURequests, end))
-}
-
-func (pds *PrometheusDataSource) QueryCPUUsageAvg(start, end time.Time) *source.Future[source.CPUUsageAvgResult] {
-	const queryFmtCPUUsageAvg = `avg(rate(container_cpu_usage_seconds_total{container!="", container_name!="POD", container!="POD", %s}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryCPUUsageAvg")
-	}
-
-	queryCPUUsageAvg := fmt.Sprintf(queryFmtCPUUsageAvg, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeCPUUsageAvgResult, ctx.QueryAtTime(queryCPUUsageAvg, end))
-}
-
-func (pds *PrometheusDataSource) QueryCPUUsageMax(start, end time.Time) *source.Future[source.CPUUsageMaxResult] {
-	// Because we use container_cpu_usage_seconds_total to calculate CPU usage
-	// at any given "instant" of time, we need to use an irate or rate. To then
-	// calculate a max (or any aggregation) we have to perform an aggregation
-	// query on top of an instant-by-instant maximum. Prometheus supports this
-	// type of query with a "subquery" [1], however it is reportedly expensive
-	// to make such a query. By default, Kubecost's Prometheus config includes
-	// a recording rule that keeps track of the instant-by-instant irate for CPU
-	// usage. The metric in this query is created by that recording rule.
-	//
-	// [1] https://prometheus.io/blog/2019/01/28/subquery-support/
-	//
-	// If changing the name of the recording rule, make sure to update the
-	// corresponding diagnostic query to avoid confusion.
-	const queryFmtCPUUsageMaxRecordingRule = `max(max_over_time(kubecost_container_cpu_usage_irate{%s}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
-
-	// This is the subquery equivalent of the above recording rule query. It is
-	// more expensive, but does not require the recording rule. It should be
-	// used as a fallback query if the recording rule data does not exist.
-	//
-	// The parameter after the colon [:<thisone>] in the subquery affects the
-	// resolution of the subquery.
-	// The parameter after the metric ...{}[<thisone>] should be set to 2x
-	// the resolution, to make sure the irate always has two points to query
-	// in case the Prom scrape duration has been reduced to be equal to the
-	// ETL resolution.
-	const queryFmtCPUUsageMaxSubquery = `max(max_over_time(irate(container_cpu_usage_seconds_total{container!="POD", container!="", %s}[%s])[%s:%s])) by (container, pod_name, pod, namespace, instance, %s)`
-	// env.GetPromClusterFilter(), doubleResStr, durStr, resStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryCPUUsageMax")
-	}
-
-	queryCPUUsageMaxRecordingRule := fmt.Sprintf(queryFmtCPUUsageMaxRecordingRule, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	resCPUUsageMaxRR := ctx.QueryAtTime(queryCPUUsageMaxRecordingRule, end)
-	resCPUUsageMax, _ := resCPUUsageMaxRR.Await()
-
-	if len(resCPUUsageMax) > 0 {
-		return wrapResults(queryCPUUsageMaxRecordingRule, source.DecodeCPUUsageMaxResult, resCPUUsageMax)
-	}
-
-	resolution := cfg.DataResolution
-	resStr := timeutil.DurationString(resolution)
-	doubleResStr := timeutil.DurationString(2 * resolution)
-
-	queryCPUUsageMaxSubquery := fmt.Sprintf(queryFmtCPUUsageMaxSubquery, cfg.ClusterFilter, doubleResStr, durStr, resStr, cfg.ClusterLabel)
-	return source.NewFuture(source.DecodeCPUUsageMaxResult, ctx.QueryAtTime(queryCPUUsageMaxSubquery, end))
-}
-
-func (pds *PrometheusDataSource) QueryGPUsRequested(start, end time.Time) *source.Future[source.GPUsRequestedResult] {
-	const queryFmtGPUsRequested = `avg(avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryGPUsRequested")
-	}
-
-	queryGPUsRequested := fmt.Sprintf(queryFmtGPUsRequested, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeGPUsRequestedResult, ctx.QueryAtTime(queryGPUsRequested, end))
-}
-
-func (pds *PrometheusDataSource) QueryGPUsUsageAvg(start, end time.Time) *source.Future[source.GPUsUsageAvgResult] {
-	const queryFmtGPUsUsageAvg = `avg(avg_over_time(DCGM_FI_PROF_GR_ENGINE_ACTIVE{container!=""}[%s])) by (container, pod, namespace, %s)`
-	// durStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryGPUsUsageAvg")
-	}
-
-	queryGPUsUsageAvg := fmt.Sprintf(queryFmtGPUsUsageAvg, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeGPUsUsageAvgResult, ctx.QueryAtTime(queryGPUsUsageAvg, end))
-}
-
-func (pds *PrometheusDataSource) QueryGPUsUsageMax(start, end time.Time) *source.Future[source.GPUsUsageMaxResult] {
-	const queryFmtGPUsUsageMax = `max(max_over_time(DCGM_FI_PROF_GR_ENGINE_ACTIVE{container!=""}[%s])) by (container, pod, namespace, %s)`
-	// durStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryGPUsUsageMax")
-	}
-
-	queryGPUsUsageMax := fmt.Sprintf(queryFmtGPUsUsageMax, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeGPUsUsageMaxResult, ctx.QueryAtTime(queryGPUsUsageMax, end))
-}
-
-func (pds *PrometheusDataSource) QueryGPUsAllocated(start, end time.Time) *source.Future[source.GPUsAllocatedResult] {
-	const queryFmtGPUsAllocated = `avg(avg_over_time(container_gpu_allocation{container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryGPUsAllocated")
-	}
-
-	queryGPUsAllocated := fmt.Sprintf(queryFmtGPUsAllocated, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeGPUsAllocatedResult, ctx.QueryAtTime(queryGPUsAllocated, end))
-}
-
-func (pds *PrometheusDataSource) QueryIsGPUShared(start, end time.Time) *source.Future[source.IsGPUSharedResult] {
-	const queryFmtIsGPUShared = `avg(avg_over_time(kube_pod_container_resource_requests{container!="", node != "", pod != "", container!= "", unit = "integer",  %s}[%s])) by (container, pod, namespace, node, resource)`
-	// env.GetPromClusterFilter(), durStr
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryIsGPUShared")
-	}
-
-	queryIsGPUShared := fmt.Sprintf(queryFmtIsGPUShared, cfg.ClusterFilter, durStr)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeIsGPUSharedResult, ctx.QueryAtTime(queryIsGPUShared, end))
-}
-
-func (pds *PrometheusDataSource) QueryGPUInfo(start, end time.Time) *source.Future[source.GPUInfoResult] {
-	const queryFmtGetGPUInfo = `avg(avg_over_time(DCGM_FI_DEV_DEC_UTIL{container!="",%s}[%s])) by (container, pod, namespace, device, modelName, UUID)`
-	// env.GetPromClusterFilter(), durStr
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryGPUInfo")
-	}
-
-	queryGetGPUInfo := fmt.Sprintf(queryFmtGetGPUInfo, cfg.ClusterFilter, durStr)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeGPUInfoResult, ctx.QueryAtTime(queryGetGPUInfo, end))
-}
-
-func (pds *PrometheusDataSource) QueryNodeCPUPricePerHr(start, end time.Time) *source.Future[source.NodeCPUPricePerHrResult] {
-	const queryFmtNodeCostPerCPUHr = `avg(avg_over_time(node_cpu_hourly_cost{%s}[%s])) by (node, %s, instance_type, provider_id)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNodeCPUPricePerHr")
-	}
-
-	queryNodeCostPerCPUHr := fmt.Sprintf(queryFmtNodeCostPerCPUHr, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeNodeCPUPricePerHrResult, ctx.QueryAtTime(queryNodeCostPerCPUHr, end))
-}
-
-func (pds *PrometheusDataSource) QueryNodeRAMPricePerGiBHr(start, end time.Time) *source.Future[source.NodeRAMPricePerGiBHrResult] {
-	const queryFmtNodeCostPerRAMGiBHr = `avg(avg_over_time(node_ram_hourly_cost{%s}[%s])) by (node, %s, instance_type, provider_id)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNodeRAMPricePerGiBHr")
-	}
-
-	queryNodeCostPerRAMGiBHr := fmt.Sprintf(queryFmtNodeCostPerRAMGiBHr, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeNodeRAMPricePerGiBHrResult, ctx.QueryAtTime(queryNodeCostPerRAMGiBHr, end))
-}
-
-func (pds *PrometheusDataSource) QueryNodeGPUPricePerHr(start, end time.Time) *source.Future[source.NodeGPUPricePerHrResult] {
-	const queryFmtNodeCostPerGPUHr = `avg(avg_over_time(node_gpu_hourly_cost{%s}[%s])) by (node, %s, instance_type, provider_id)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNodeGPUPricePerHr")
-	}
-
-	queryNodeCostPerGPUHr := fmt.Sprintf(queryFmtNodeCostPerGPUHr, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeNodeGPUPricePerHrResult, ctx.QueryAtTime(queryNodeCostPerGPUHr, end))
-}
-
-func (pds *PrometheusDataSource) QueryNodeIsSpot(start, end time.Time) *source.Future[source.NodeIsSpotResult] {
-	const queryFmtNodeIsSpot = `avg_over_time(kubecost_node_is_spot{%s}[%s])`
-	//`avg_over_time(kubecost_node_is_spot{%s}[%s:%dm])`
-	// env.GetPromClusterFilter(), durStr)
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNodeIsSpot2")
-	}
-
-	queryNodeIsSpot := fmt.Sprintf(queryFmtNodeIsSpot, cfg.ClusterFilter, durStr)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeNodeIsSpotResult, ctx.QueryAtTime(queryNodeIsSpot, end))
-}
-
-func (pds *PrometheusDataSource) QueryPodPVCAllocation(start, end time.Time) *source.Future[source.PodPVCAllocationResult] {
-	const queryFmtPodPVCAllocation = `avg(avg_over_time(pod_pvc_allocation{%s}[%s])) by (persistentvolume, persistentvolumeclaim, pod, namespace, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryPodPVCAllocation")
-	}
-
-	queryPodPVCAllocation := fmt.Sprintf(queryFmtPodPVCAllocation, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodePodPVCAllocationResult, ctx.QueryAtTime(queryPodPVCAllocation, end))
-}
-
-func (pds *PrometheusDataSource) QueryPVCBytesRequested(start, end time.Time) *source.Future[source.PVCBytesRequestedResult] {
-	const queryFmtPVCBytesRequested = `avg(avg_over_time(kube_persistentvolumeclaim_resource_requests_storage_bytes{%s}[%s])) by (persistentvolumeclaim, namespace, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryPVCBytesRequested")
-	}
-
-	queryPVCBytesRequested := fmt.Sprintf(queryFmtPVCBytesRequested, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodePVCBytesRequestedResult, ctx.QueryAtTime(queryPVCBytesRequested, end))
-}
-
-func (pds *PrometheusDataSource) QueryPVBytes(start, end time.Time) *source.Future[source.PVBytesResult] {
-	const queryFmtPVBytes = `avg(avg_over_time(kube_persistentvolume_capacity_bytes{%s}[%s])) by (persistentvolume, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryPVBytes")
-	}
-
-	queryPVBytes := fmt.Sprintf(queryFmtPVBytes, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodePVBytesResult, ctx.QueryAtTime(queryPVBytes, end))
-}
-
-func (pds *PrometheusDataSource) QueryPVCostPerGiBHour(start, end time.Time) *source.Future[source.PVPricePerGiBHourResult] {
-	const queryFmtPVCostPerGiBHour = `avg(avg_over_time(pv_hourly_cost{%s}[%s])) by (volumename, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryPVCostPerGiBHour")
-	}
-
-	queryPVCostPerGiBHour := fmt.Sprintf(queryFmtPVCostPerGiBHour, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodePVPricePerGiBHourResult, ctx.QueryAtTime(queryPVCostPerGiBHour, end))
-}
-
-func (pds *PrometheusDataSource) QueryPVInfo(start, end time.Time) *source.Future[source.PVInfoResult] {
-	const queryFmtPVMeta = `avg(avg_over_time(kubecost_pv_info{%s}[%s])) by (%s, storageclass, persistentvolume, provider_id)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryPVMeta")
-	}
-
-	queryPVMeta := fmt.Sprintf(queryFmtPVMeta, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodePVInfoResult, ctx.QueryAtTime(queryPVMeta, end))
-}
-
-func (pds *PrometheusDataSource) QueryNetZoneGiB(start, end time.Time) *source.Future[source.NetZoneGiBResult] {
-	const queryFmtNetZoneGiB = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", same_zone="false", same_region="true", %s}[%s])) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNetZoneGiB")
-	}
-
-	queryNetZoneGiB := fmt.Sprintf(queryFmtNetZoneGiB, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeNetZoneGiBResult, ctx.QueryAtTime(queryNetZoneGiB, end))
-}
-
-func (pds *PrometheusDataSource) QueryNetZonePricePerGiB(start, end time.Time) *source.Future[source.NetZonePricePerGiBResult] {
-	const queryFmtNetZoneCostPerGiB = `avg(avg_over_time(kubecost_network_zone_egress_cost{%s}[%s])) by (%s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNetZonePricePerGiB")
-	}
-
-	queryNetZoneCostPerGiB := fmt.Sprintf(queryFmtNetZoneCostPerGiB, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeNetZonePricePerGiBResult, ctx.QueryAtTime(queryNetZoneCostPerGiB, end))
-}
-
-func (pds *PrometheusDataSource) QueryNetRegionGiB(start, end time.Time) *source.Future[source.NetRegionGiBResult] {
-	const queryFmtNetRegionGiB = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", same_zone="false", same_region="false", %s}[%s])) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNetRegionGiB")
-	}
-
-	queryNetRegionGiB := fmt.Sprintf(queryFmtNetRegionGiB, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeNetRegionGiBResult, ctx.QueryAtTime(queryNetRegionGiB, end))
-}
-
-func (pds *PrometheusDataSource) QueryNetRegionPricePerGiB(start, end time.Time) *source.Future[source.NetRegionPricePerGiBResult] {
-	const queryFmtNetRegionCostPerGiB = `avg(avg_over_time(kubecost_network_region_egress_cost{%s}[%s])) by (%s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNetRegionPricePerGiB")
-	}
-
-	queryNetRegionCostPerGiB := fmt.Sprintf(queryFmtNetRegionCostPerGiB, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeNetRegionPricePerGiBResult, ctx.QueryAtTime(queryNetRegionCostPerGiB, end))
-}
-
-func (pds *PrometheusDataSource) QueryNetInternetGiB(start, end time.Time) *source.Future[source.NetInternetGiBResult] {
-	const queryFmtNetInternetGiB = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="true", %s}[%s])) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNetInternetGiB")
-	}
-
-	queryNetInternetGiB := fmt.Sprintf(queryFmtNetInternetGiB, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeNetInternetGiBResult, ctx.QueryAtTime(queryNetInternetGiB, end))
-}
-
-func (pds *PrometheusDataSource) QueryNetInternetPricePerGiB(start, end time.Time) *source.Future[source.NetInternetPricePerGiBResult] {
-	const queryFmtNetInternetCostPerGiB = `avg(avg_over_time(kubecost_network_internet_egress_cost{%s}[%s])) by (%s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNetInternetPricePerGiB")
-	}
-
-	queryNetInternetCostPerGiB := fmt.Sprintf(queryFmtNetInternetCostPerGiB, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeNetInternetPricePerGiBResult, ctx.QueryAtTime(queryNetInternetCostPerGiB, end))
-}
-
-func (pds *PrometheusDataSource) QueryNetReceiveBytes(start, end time.Time) *source.Future[source.NetReceiveBytesResult] {
-	const queryFmtNetReceiveBytes = `sum(increase(container_network_receive_bytes_total{pod!="", %s}[%s])) by (pod_name, pod, namespace, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNetReceiveBytes")
-	}
-
-	queryNetReceiveBytes := fmt.Sprintf(queryFmtNetReceiveBytes, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeNetReceiveBytesResult, ctx.QueryAtTime(queryNetReceiveBytes, end))
-}
-
-func (pds *PrometheusDataSource) QueryNetTransferBytes(start, end time.Time) *source.Future[source.NetTransferBytesResult] {
-	const queryFmtNetTransferBytes = `sum(increase(container_network_transmit_bytes_total{pod!="", %s}[%s])) by (pod_name, pod, namespace, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNetTransferBytes")
-	}
-
-	queryNetTransferBytes := fmt.Sprintf(queryFmtNetTransferBytes, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeNetTransferBytesResult, ctx.QueryAtTime(queryNetTransferBytes, end))
-}
-
-func (pds *PrometheusDataSource) QueryNamespaceLabels(start, end time.Time) *source.Future[source.NamespaceLabelsResult] {
-	const queryFmtNamespaceLabels = `avg_over_time(kube_namespace_labels{%s}[%s])`
-	// env.GetPromClusterFilter(), durStr
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNamespaceLabels")
-	}
-
-	queryNamespaceLabels := fmt.Sprintf(queryFmtNamespaceLabels, cfg.ClusterFilter, durStr)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeNamespaceLabelsResult, ctx.QueryAtTime(queryNamespaceLabels, end))
-}
-
-func (pds *PrometheusDataSource) QueryNamespaceAnnotations(start, end time.Time) *source.Future[source.NamespaceAnnotationsResult] {
-	const queryFmtNamespaceAnnotations = `avg_over_time(kube_namespace_annotations{%s}[%s])`
-	// env.GetPromClusterFilter(), durStr
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNamespaceAnnotations")
-	}
-
-	queryNamespaceAnnotations := fmt.Sprintf(queryFmtNamespaceAnnotations, cfg.ClusterFilter, durStr)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeNamespaceAnnotationsResult, ctx.QueryAtTime(queryNamespaceAnnotations, end))
-}
-
-func (pds *PrometheusDataSource) QueryPodLabels(start, end time.Time) *source.Future[source.PodLabelsResult] {
-	const queryFmtPodLabels = `avg_over_time(kube_pod_labels{%s}[%s])`
-	// env.GetPromClusterFilter(), durStr
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryPodLabels")
-	}
-
-	queryPodLabels := fmt.Sprintf(queryFmtPodLabels, cfg.ClusterFilter, durStr)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodePodLabelsResult, ctx.QueryAtTime(queryPodLabels, end))
-}
-
-func (pds *PrometheusDataSource) QueryPodAnnotations(start, end time.Time) *source.Future[source.PodAnnotationsResult] {
-	const queryFmtPodAnnotations = `avg_over_time(kube_pod_annotations{%s}[%s])`
-	// env.GetPromClusterFilter(), durStr
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryPodAnnotations")
-	}
-
-	queryPodAnnotations := fmt.Sprintf(queryFmtPodAnnotations, cfg.ClusterFilter, durStr)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodePodAnnotationsResult, ctx.QueryAtTime(queryPodAnnotations, end))
-}
-
-func (pds *PrometheusDataSource) QueryServiceLabels(start, end time.Time) *source.Future[source.ServiceLabelsResult] {
-	const queryFmtServiceLabels = `avg_over_time(service_selector_labels{%s}[%s])`
-	// env.GetPromClusterFilter(), durStr
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryServiceLabels")
-	}
-
-	queryServiceLabels := fmt.Sprintf(queryFmtServiceLabels, cfg.ClusterFilter, durStr)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeServiceLabelsResult, ctx.QueryAtTime(queryServiceLabels, end))
-}
-
-func (pds *PrometheusDataSource) QueryDeploymentLabels(start, end time.Time) *source.Future[source.DeploymentLabelsResult] {
-	const queryFmtDeploymentLabels = `avg_over_time(deployment_match_labels{%s}[%s])`
-	// env.GetPromClusterFilter(), durStr
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryNamespaceAnnotations")
-	}
-
-	queryDeploymentLabels := fmt.Sprintf(queryFmtDeploymentLabels, cfg.ClusterFilter, durStr)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeDeploymentLabelsResult, ctx.QueryAtTime(queryDeploymentLabels, end))
-}
-
-func (pds *PrometheusDataSource) QueryStatefulSetLabels(start, end time.Time) *source.Future[source.StatefulSetLabelsResult] {
-	const queryFmtStatefulSetLabels = `avg_over_time(statefulSet_match_labels{%s}[%s])`
-	// env.GetPromClusterFilter(), durStr
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryStatefulSetLabels")
-	}
-
-	queryStatefulSetLabels := fmt.Sprintf(queryFmtStatefulSetLabels, cfg.ClusterFilter, durStr)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeStatefulSetLabelsResult, ctx.QueryAtTime(queryStatefulSetLabels, end))
-}
-
-func (pds *PrometheusDataSource) QueryDaemonSetLabels(start, end time.Time) *source.Future[source.DaemonSetLabelsResult] {
-	const queryFmtDaemonSetLabels = `sum(avg_over_time(kube_pod_owner{owner_kind="DaemonSet", %s}[%s])) by (pod, owner_name, namespace, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryDaemonSetLabels")
-	}
-
-	queryDaemonSetLabels := fmt.Sprintf(queryFmtDaemonSetLabels, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeDaemonSetLabelsResult, ctx.QueryAtTime(queryDaemonSetLabels, end))
-}
-
-func (pds *PrometheusDataSource) QueryJobLabels(start, end time.Time) *source.Future[source.JobLabelsResult] {
-	const queryFmtJobLabels = `sum(avg_over_time(kube_pod_owner{owner_kind="Job", %s}[%s])) by (pod, owner_name, namespace ,%s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryJobLabels")
-	}
-
-	queryJobLabels := fmt.Sprintf(queryFmtJobLabels, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeJobLabelsResult, ctx.QueryAtTime(queryJobLabels, end))
-}
-
-func (pds *PrometheusDataSource) QueryPodsWithReplicaSetOwner(start, end time.Time) *source.Future[source.PodsWithReplicaSetOwnerResult] {
-	const queryFmtPodsWithReplicaSetOwner = `sum(avg_over_time(kube_pod_owner{owner_kind="ReplicaSet", %s}[%s])) by (pod, owner_name, namespace ,%s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryPodsWithReplicaSetOwner")
-	}
-
-	queryPodsWithReplicaSetOwner := fmt.Sprintf(queryFmtPodsWithReplicaSetOwner, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodePodsWithReplicaSetOwnerResult, ctx.QueryAtTime(queryPodsWithReplicaSetOwner, end))
-}
-
-func (pds *PrometheusDataSource) QueryReplicaSetsWithoutOwners(start, end time.Time) *source.Future[source.ReplicaSetsWithoutOwnersResult] {
-	const queryFmtReplicaSetsWithoutOwners = `avg(avg_over_time(kube_replicaset_owner{owner_kind="<none>", owner_name="<none>", %s}[%s])) by (replicaset, namespace, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryReplicaSetsWithoutOwners")
-	}
-
-	queryReplicaSetsWithoutOwners := fmt.Sprintf(queryFmtReplicaSetsWithoutOwners, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeReplicaSetsWithoutOwnersResult, ctx.QueryAtTime(queryReplicaSetsWithoutOwners, end))
-}
-
-func (pds *PrometheusDataSource) QueryReplicaSetsWithRollout(start, end time.Time) *source.Future[source.ReplicaSetsWithRolloutResult] {
-	const queryFmtReplicaSetsWithRolloutOwner = `avg(avg_over_time(kube_replicaset_owner{owner_kind="Rollout", %s}[%s])) by (replicaset, namespace, owner_kind, owner_name, %s)`
-	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-
-	cfg := pds.promConfig
-
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		panic("failed to parse duration string passed to QueryReplicaSetsWithRollout")
-	}
-
-	queryReplicaSetsWithRolloutOwner := fmt.Sprintf(queryFmtReplicaSetsWithRolloutOwner, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	return source.NewFuture(source.DecodeReplicaSetsWithRolloutResult, ctx.QueryAtTime(queryReplicaSetsWithRolloutOwner, end))
-}
-
-func (pds *PrometheusDataSource) QueryDataCoverage(limitDays int) (time.Time, time.Time, error) {
-	const (
-		queryFmtOldestSample = `min_over_time(timestamp(group(node_cpu_hourly_cost{%s}))[%s:%s])`
-		queryFmtNewestSample = `max_over_time(timestamp(group(node_cpu_hourly_cost{%s}))[%s:%s])`
-	)
-
-	cfg := pds.promConfig
-	now := time.Now()
-	durStr := fmt.Sprintf("%dd", limitDays)
-
-	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
-	queryOldest := fmt.Sprintf(queryFmtOldestSample, cfg.ClusterFilter, durStr, "1h")
-	resOldestFut := ctx.QueryAtTime(queryOldest, now)
-
-	resOldest, err := resOldestFut.Await()
-	if err != nil {
-		return time.Time{}, time.Time{}, fmt.Errorf("querying oldest sample: %w", err)
-	}
-	if len(resOldest) == 0 || len(resOldest[0].Values) == 0 {
-		return time.Time{}, time.Time{}, fmt.Errorf("querying oldest sample: %w", err)
-	}
-
-	oldest := time.Unix(int64(resOldest[0].Values[0].Value), 0)
-
-	queryNewest := fmt.Sprintf(queryFmtNewestSample, cfg.ClusterFilter, durStr, "1h")
-	resNewestFut := ctx.QueryAtTime(queryNewest, now)
-
-	resNewest, err := resNewestFut.Await()
-	if err != nil {
-		return time.Time{}, time.Time{}, fmt.Errorf("querying newest sample: %w", err)
-	}
-	if len(resNewest) == 0 || len(resNewest[0].Values) == 0 {
-		return time.Time{}, time.Time{}, fmt.Errorf("querying newest sample: %w", err)
-	}
-
-	newest := time.Unix(int64(resNewest[0].Values[0].Value), 0)
-
-	return oldest, newest, nil
-}
-
-func newEmptyResult[T any](decoder source.ResultDecoder[T]) *source.Future[T] {
-	ch := make(source.QueryResultsChan)
-	go func() {
-		results := source.NewQueryResults("")
-		ch <- results
-	}()
-
-	return source.NewFuture(decoder, ch)
-}
-
-func wrapResults[T any](query string, decoder source.ResultDecoder[T], results []*source.QueryResult) *source.Future[T] {
-	ch := make(source.QueryResultsChan)
-
-	go func() {
-		r := source.NewQueryResults(query)
-		r.Results = results
-		ch <- r
-	}()
-
-	return source.NewFuture(decoder, ch)
+func (pds *PrometheusDataSource) Resolution() time.Duration {
+	return pds.promConfig.DataResolution
 }

modules/prometheus-source/pkg/prom/metricsquerier.go

@@ -0,0 +1,1322 @@
+package prom
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/opencost/opencost/core/pkg/log"
+	"github.com/opencost/opencost/core/pkg/source"
+	"github.com/opencost/opencost/core/pkg/util/timeutil"
+	prometheus "github.com/prometheus/client_golang/api"
+)
+
+//--------------------------------------------------------------------------
+//  PrometheusMetricsQuerier
+//--------------------------------------------------------------------------
+
+// PrometheusMetricsQuerier is the implementation of the data source's MetricsQuerier interface for Prometheus.
+type PrometheusMetricsQuerier struct {
+	promConfig   *OpenCostPrometheusConfig
+	promClient   prometheus.Client
+	promContexts *ContextFactory
+
+	thanosConfig   *OpenCostThanosConfig
+	thanosClient   prometheus.Client
+	thanosContexts *ContextFactory
+}
+
+func newPrometheusMetricsQuerier(
+	promConfig *OpenCostPrometheusConfig,
+	promClient prometheus.Client,
+	promContexts *ContextFactory,
+	thanosConfig *OpenCostThanosConfig,
+	thanosClient prometheus.Client,
+	thanosContexts *ContextFactory,
+) *PrometheusMetricsQuerier {
+	return &PrometheusMetricsQuerier{
+		promConfig:     promConfig,
+		promClient:     promClient,
+		promContexts:   promContexts,
+		thanosConfig:   thanosConfig,
+		thanosClient:   thanosClient,
+		thanosContexts: thanosContexts,
+	}
+}
+
+func (pds *PrometheusMetricsQuerier) QueryPVPricePerGiBHour(start, end time.Time) *source.Future[source.PVPricePerGiBHourResult] {
+	const pvCostQuery = `avg(avg_over_time(pv_hourly_cost{%s}[%s])) by (%s, persistentvolume, volumename, provider_id)`
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryPVCost")
+	}
+
+	queryPVCost := fmt.Sprintf(pvCostQuery, pds.promConfig.ClusterFilter, durStr, pds.promConfig.ClusterLabel)
+
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodePVPricePerGiBHourResult, ctx.QueryAtTime(queryPVCost, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryPVUsedAverage(start, end time.Time) *source.Future[source.PVUsedAvgResult] {
+	// `avg(avg_over_time(kubelet_volume_stats_used_bytes{%s}[%s])) by (%s, persistentvolumeclaim, namespace)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	const pvUsedAverageQuery = `avg(avg_over_time(kubelet_volume_stats_used_bytes{%s}[%s])) by (%s, persistentvolumeclaim, namespace)`
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryPVUsedAverage")
+	}
+
+	queryPVUsedAvg := fmt.Sprintf(pvUsedAverageQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodePVUsedAvgResult, ctx.QueryAtTime(queryPVUsedAvg, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryPVUsedMax(start, end time.Time) *source.Future[source.PVUsedMaxResult] {
+	// `max(max_over_time(kubelet_volume_stats_used_bytes{%s}[%s])) by (%s, persistentvolumeclaim, namespace)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	const pvUsedMaxQuery = `max(max_over_time(kubelet_volume_stats_used_bytes{%s}[%s])) by (%s, persistentvolumeclaim, namespace)`
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryPVUsedMax")
+	}
+
+	queryPVUsedMax := fmt.Sprintf(pvUsedMaxQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodePVUsedMaxResult, ctx.QueryAtTime(queryPVUsedMax, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryPVCInfo(start, end time.Time) *source.Future[source.PVCInfoResult] {
+	const queryFmtPVCInfo = `avg(kube_persistentvolumeclaim_info{volumename != "", %s}) by (persistentvolumeclaim, storageclass, volumename, namespace, %s)[%s:%s]`
+	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, resStr)
+
+	cfg := pds.promConfig
+	resolution := cfg.DataResolution
+	resStr := timeutil.DurationString(resolution)
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryPVCInfo")
+	}
+
+	queryPVCInfo := fmt.Sprintf(queryFmtPVCInfo, cfg.ClusterFilter, cfg.ClusterLabel, durStr, resStr)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodePVCInfoResult, ctx.QueryAtTime(queryPVCInfo, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryPVActiveMinutes(start, end time.Time) *source.Future[source.PVActiveMinutesResult] {
+	const pvActiveMinsQuery = `avg(kube_persistentvolume_capacity_bytes{%s}) by (%s, persistentvolume)[%s:%dm]`
+	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
+
+	cfg := pds.promConfig
+	minsPerResolution := cfg.DataResolutionMinutes
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryPVActiveMinutes")
+	}
+
+	queryPVActiveMins := fmt.Sprintf(pvActiveMinsQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodePVActiveMinutesResult, ctx.QueryAtTime(queryPVActiveMins, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryLocalStorageCost(start, end time.Time) *source.Future[source.LocalStorageCostResult] {
+	// `sum_over_time(sum(container_fs_limit_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`
+	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
+
+	const localStorageCostQuery = `sum_over_time(sum(container_fs_limit_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`
+
+	cfg := pds.promConfig
+	resolution := cfg.DataResolution
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryLocalStorageCost")
+	}
+
+	//Ensuring if data resolution is less than 60s default it to 1m
+	var minsPerResolution int
+	if minsPerResolution = int(resolution.Minutes()); int(resolution.Minutes()) == 0 {
+		minsPerResolution = 1
+		log.DedupedWarningf(3, "QueryLocalStorageCost: Configured resolution (%d seconds) is below the 60 seconds threshold. Overriding with 1 minute.", int(resolution.Seconds()))
+	}
+
+	// hourlyToCumulative is a scaling factor that, when multiplied by an
+	// hourly value, converts it to a cumulative value; i.e. [$/hr] *
+	// [min/res]*[hr/min] = [$/res]
+	hourlyToCumulative := float64(minsPerResolution) * (1.0 / 60.0)
+	costPerGBHr := 0.04 / 730.0
+
+	queryLocalStorageCost := fmt.Sprintf(localStorageCostQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeLocalStorageCostResult, ctx.QueryAtTime(queryLocalStorageCost, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryLocalStorageUsedCost(start, end time.Time) *source.Future[source.LocalStorageUsedCostResult] {
+	// `sum_over_time(sum(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`
+	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
+
+	const localStorageUsedCostQuery = `sum_over_time(sum(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`
+
+	cfg := pds.promConfig
+	minsPerResolution := cfg.DataResolutionMinutes
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryLocalStorageUsedCost")
+	}
+
+	// hourlyToCumulative is a scaling factor that, when multiplied by an
+	// hourly value, converts it to a cumulative value; i.e. [$/hr] *
+	// [min/res]*[hr/min] = [$/res]
+	hourlyToCumulative := float64(minsPerResolution) * (1.0 / 60.0)
+	costPerGBHr := 0.04 / 730.0
+
+	queryLocalStorageUsedCost := fmt.Sprintf(localStorageUsedCostQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeLocalStorageUsedCostResult, ctx.QueryAtTime(queryLocalStorageUsedCost, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryLocalStorageUsedAvg(start, end time.Time) *source.Future[source.LocalStorageUsedAvgResult] {
+	// `avg(sum(avg_over_time(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}[%s])) by (instance, device, %s, job)) by (instance, device, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel(), env.GetPromClusterLabel())
+
+	const localStorageUsedAvgQuery = `avg(sum(avg_over_time(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}[%s])) by (instance, device, %s, job)) by (instance, device, %s)`
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryLocalStorageUsedAvg")
+	}
+
+	queryLocalStorageUsedAvg := fmt.Sprintf(localStorageUsedAvgQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeLocalStorageUsedAvgResult, ctx.QueryAtTime(queryLocalStorageUsedAvg, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryLocalStorageUsedMax(start, end time.Time) *source.Future[source.LocalStorageUsedMaxResult] {
+	// `max(sum(max_over_time(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}[%s])) by (instance, device, %s, job)) by (instance, device, %s)`
+	//  env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel(), env.GetPromClusterLabel())
+	const localStorageUsedMaxQuery = `max(sum(max_over_time(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}[%s])) by (instance, device, %s, job)) by (instance, device, %s)`
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryLocalStorageUsedMax")
+	}
+
+	queryLocalStorageUsedMax := fmt.Sprintf(localStorageUsedMaxQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeLocalStorageUsedMaxResult, ctx.QueryAtTime(queryLocalStorageUsedMax, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryLocalStorageBytes(start, end time.Time) *source.Future[source.LocalStorageBytesResult] {
+	// `avg_over_time(sum(container_fs_limit_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, %s)[%s:%dm])`
+	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
+
+	const localStorageBytesQuery = `avg_over_time(sum(container_fs_limit_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, %s)[%s:%dm])`
+
+	cfg := pds.promConfig
+	minsPerResolution := cfg.DataResolutionMinutes
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryLocalStorageBytes")
+	}
+
+	queryLocalStorageBytes := fmt.Sprintf(localStorageBytesQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeLocalStorageBytesResult, ctx.QueryAtTime(queryLocalStorageBytes, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryLocalStorageActiveMinutes(start, end time.Time) *source.Future[source.LocalStorageActiveMinutesResult] {
+	// `count(node_total_hourly_cost{%s}) by (%s, node)[%s:%dm]`
+	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
+
+	const localStorageActiveMinutesQuery = `count(node_total_hourly_cost{%s}) by (%s, node, instance, provider_id)[%s:%dm]`
+
+	cfg := pds.promConfig
+	minsPerResolution := cfg.DataResolutionMinutes
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryLocalStorageActiveMinutes")
+	}
+
+	queryLocalStorageActiveMins := fmt.Sprintf(localStorageActiveMinutesQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeLocalStorageActiveMinutesResult, ctx.QueryAtTime(queryLocalStorageActiveMins, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNodeCPUCoresCapacity(start, end time.Time) *source.Future[source.NodeCPUCoresCapacityResult] {
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	const nodeCPUCoresCapacityQuery = `avg(avg_over_time(kube_node_status_capacity_cpu_cores{%s}[%s])) by (%s, node)`
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNodeCPUCoresCapacity")
+	}
+
+	queryNodeCPUCoresCapacity := fmt.Sprintf(nodeCPUCoresCapacityQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeNodeCPUCoresCapacityResult, ctx.QueryAtTime(queryNodeCPUCoresCapacity, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNodeCPUCoresAllocatable(start, end time.Time) *source.Future[source.NodeCPUCoresAllocatableResult] {
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	const nodeCPUCoresAllocatableQuery = `avg(avg_over_time(kube_node_status_allocatable_cpu_cores{%s}[%s])) by (%s, node)`
+	// `avg(avg_over_time(container_cpu_allocation{container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNodeCPUCoresAllocatable")
+	}
+
+	queryNodeCPUCoresAllocatable := fmt.Sprintf(nodeCPUCoresAllocatableQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeNodeCPUCoresAllocatableResult, ctx.QueryAtTime(queryNodeCPUCoresAllocatable, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNodeRAMBytesCapacity(start, end time.Time) *source.Future[source.NodeRAMBytesCapacityResult] {
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	const nodeRAMBytesCapacityQuery = `avg(avg_over_time(kube_node_status_capacity_memory_bytes{%s}[%s])) by (%s, node)`
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNodeRAMBytesCapacity")
+	}
+
+	queryNodeRAMBytesCapacity := fmt.Sprintf(nodeRAMBytesCapacityQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeNodeRAMBytesCapacityResult, ctx.QueryAtTime(queryNodeRAMBytesCapacity, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNodeRAMBytesAllocatable(start, end time.Time) *source.Future[source.NodeRAMBytesAllocatableResult] {
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	const nodeRAMBytesAllocatableQuery = `avg(avg_over_time(kube_node_status_allocatable_memory_bytes{%s}[%s])) by (%s, node)`
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNodeRAMBytesAllocatable")
+	}
+
+	queryNodeRAMBytesAllocatable := fmt.Sprintf(nodeRAMBytesAllocatableQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeNodeRAMBytesAllocatableResult, ctx.QueryAtTime(queryNodeRAMBytesAllocatable, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNodeGPUCount(start, end time.Time) *source.Future[source.NodeGPUCountResult] {
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	const nodeGPUCountQuery = `avg(avg_over_time(node_gpu_count{%s}[%s])) by (%s, node, provider_id)`
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNodeGPUCount")
+	}
+
+	queryNodeGPUCount := fmt.Sprintf(nodeGPUCountQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeNodeGPUCountResult, ctx.QueryAtTime(queryNodeGPUCount, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNodeLabels(start, end time.Time) *source.Future[source.NodeLabelsResult] {
+	const labelsQuery = `avg_over_time(kube_node_labels{%s}[%s])`
+	// env.GetPromClusterFilter(), durStr
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNodeLabels")
+	}
+
+	queryLabels := fmt.Sprintf(labelsQuery, cfg.ClusterFilter, durStr)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeNodeLabelsResult, ctx.QueryAtTime(queryLabels, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNodeActiveMinutes(start, end time.Time) *source.Future[source.NodeActiveMinutesResult] {
+	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
+
+	const activeMinsQuery = `avg(node_total_hourly_cost{%s}) by (node, %s, provider_id)[%s:%dm]`
+
+	cfg := pds.promConfig
+	minsPerResolution := cfg.DataResolutionMinutes
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNodeActiveMinutes")
+	}
+
+	queryActiveMins := fmt.Sprintf(activeMinsQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeNodeActiveMinutesResult, ctx.QueryAtTime(queryActiveMins, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNodeCPUModeTotal(start, end time.Time) *source.Future[source.NodeCPUModeTotalResult] {
+	// env.GetPromClusterFilter(), durStr, minsPerResolution, env.GetPromClusterLabel())
+
+	const nodeCPUModeTotalQuery = `sum(rate(node_cpu_seconds_total{%s}[%s:%dm])) by (kubernetes_node, %s, mode)`
+
+	cfg := pds.promConfig
+	minsPerResolution := cfg.DataResolutionMinutes
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNodeCPUModeTotal")
+	}
+
+	queryCPUModeTotal := fmt.Sprintf(nodeCPUModeTotalQuery, cfg.ClusterFilter, durStr, minsPerResolution, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeNodeCPUModeTotalResult, ctx.QueryAtTime(queryCPUModeTotal, end))
+}
+func (pds *PrometheusMetricsQuerier) QueryNodeRAMSystemPercent(start, end time.Time) *source.Future[source.NodeRAMSystemPercentResult] {
+	// env.GetPromClusterFilter(), durStr, minsPerResolution, env.GetPromClusterLabel(), env.GetPromClusterFilter(), durStr, minsPerResolution, env.GetPromClusterLabel(), env.GetPromClusterLabel())
+
+	const nodeRAMSystemPctQuery = `sum(sum_over_time(container_memory_working_set_bytes{container_name!="POD",container_name!="",namespace="kube-system", %s}[%s:%dm])) by (instance, %s) / avg(label_replace(sum(sum_over_time(kube_node_status_capacity_memory_bytes{%s}[%s:%dm])) by (node, %s), "instance", "$1", "node", "(.*)")) by (instance, %s)`
+
+	cfg := pds.promConfig
+	minsPerResolution := cfg.DataResolutionMinutes
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNodeRAMSystemPercent")
+	}
+
+	queryRAMSystemPct := fmt.Sprintf(nodeRAMSystemPctQuery, cfg.ClusterFilter, durStr, minsPerResolution, cfg.ClusterLabel, cfg.ClusterFilter, durStr, minsPerResolution, cfg.ClusterLabel, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeNodeRAMSystemPercentResult, ctx.QueryAtTime(queryRAMSystemPct, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNodeRAMUserPercent(start, end time.Time) *source.Future[source.NodeRAMUserPercentResult] {
+	// env.GetPromClusterFilter(), durStr, minsPerResolution, env.GetPromClusterLabel(), env.GetPromClusterFilter(), durStr, minsPerResolution, env.GetPromClusterLabel(), env.GetPromClusterLabel())
+
+	const nodeRAMUserPctQuery = `sum(sum_over_time(container_memory_working_set_bytes{container_name!="POD",container_name!="",namespace!="kube-system", %s}[%s:%dm])) by (instance, %s) / avg(label_replace(sum(sum_over_time(kube_node_status_capacity_memory_bytes{%s}[%s:%dm])) by (node, %s), "instance", "$1", "node", "(.*)")) by (instance, %s)`
+
+	cfg := pds.promConfig
+	minsPerResolution := cfg.DataResolutionMinutes
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNodeRAMUserPercent")
+	}
+
+	queryRAMUserPct := fmt.Sprintf(nodeRAMUserPctQuery, cfg.ClusterFilter, durStr, minsPerResolution, cfg.ClusterLabel, cfg.ClusterFilter, durStr, minsPerResolution, cfg.ClusterLabel, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeNodeRAMUserPercentResult, ctx.QueryAtTime(queryRAMUserPct, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryLBPricePerHr(start, end time.Time) *source.Future[source.LBPricePerHrResult] {
+	const queryFmtLBCostPerHr = `avg(avg_over_time(kubecost_load_balancer_cost{%s}[%s])) by (namespace, service_name, ingress_ip, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryLBPricePerHr")
+	}
+
+	queryLBCostPerHr := fmt.Sprintf(queryFmtLBCostPerHr, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeLBPricePerHrResult, ctx.QueryAtTime(queryLBCostPerHr, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryLBActiveMinutes(start, end time.Time) *source.Future[source.LBActiveMinutesResult] {
+	const lbActiveMinutesQuery = `avg(kubecost_load_balancer_cost{%s}) by (namespace, service_name, %s, ingress_ip)[%s:%dm]`
+	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
+
+	cfg := pds.promConfig
+	minsPerResolution := cfg.DataResolutionMinutes
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryLBActiveMinutes")
+	}
+
+	queryLBActiveMins := fmt.Sprintf(lbActiveMinutesQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeLBActiveMinutesResult, ctx.QueryAtTime(queryLBActiveMins, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryClusterManagementDuration(start, end time.Time) *source.Future[source.ClusterManagementDurationResult] {
+	const clusterManagementDurationQuery = `avg(kubecost_cluster_management_cost{%s}) by (%s, provisioner_name)[%s:%dm]`
+
+	cfg := pds.promConfig
+	minsPerResolution := cfg.DataResolutionMinutes
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryClusterManagementDuration")
+	}
+
+	queryClusterManagementDuration := fmt.Sprintf(clusterManagementDurationQuery, cfg.ClusterFilter, cfg.ClusterLabel, durStr, minsPerResolution)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeClusterManagementDurationResult, ctx.QueryAtTime(queryClusterManagementDuration, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryClusterManagementPricePerHr(start, end time.Time) *source.Future[source.ClusterManagementPricePerHrResult] {
+	const clusterManagementCostQuery = `avg(avg_over_time(kubecost_cluster_management_cost{%s}[%s])) by (%s, provisioner_name)`
+	// env.GetPromClusterFilter(), durationStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryClusterManagementCost")
+	}
+
+	queryClusterManagementCost := fmt.Sprintf(clusterManagementCostQuery, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(ClusterContextName)
+	return source.NewFuture(source.DecodeClusterManagementPricePerHrResult, ctx.QueryAtTime(queryClusterManagementCost, end))
+}
+
+// AllocationMetricQuerier
+
+func (pds *PrometheusMetricsQuerier) QueryPods(start, end time.Time) *source.Future[source.PodsResult] {
+	const queryFmtPods = `avg(kube_pod_container_status_running{%s} != 0) by (pod, namespace, %s)[%s:%s]`
+	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, resStr)
+
+	cfg := pds.promConfig
+	resolution := cfg.DataResolution
+	resStr := timeutil.DurationString(resolution)
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryPods")
+	}
+
+	queryPods := fmt.Sprintf(queryFmtPods, cfg.ClusterFilter, cfg.ClusterLabel, durStr, resStr)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodePodsResult, ctx.QueryAtTime(queryPods, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryPodsUID(start, end time.Time) *source.Future[source.PodsResult] {
+	const queryFmtPodsUID = `avg(kube_pod_container_status_running{%s} != 0) by (pod, namespace, uid, %s)[%s:%s]`
+	// env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, resStr)
+
+	cfg := pds.promConfig
+	resolution := cfg.DataResolution
+	resStr := timeutil.DurationString(resolution)
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryPodsUID")
+	}
+
+	queryPodsUID := fmt.Sprintf(queryFmtPodsUID, cfg.ClusterFilter, cfg.ClusterLabel, durStr, resStr)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodePodsResult, ctx.QueryAtTime(queryPodsUID, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryRAMBytesAllocated(start, end time.Time) *source.Future[source.RAMBytesAllocatedResult] {
+	const queryFmtRAMBytesAllocated = `avg(avg_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s, provider_id)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryRAMBytesAllocated")
+	}
+
+	queryRAMBytesAllocated := fmt.Sprintf(queryFmtRAMBytesAllocated, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeRAMBytesAllocatedResult, ctx.QueryAtTime(queryRAMBytesAllocated, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryRAMRequests(start, end time.Time) *source.Future[source.RAMRequestsResult] {
+	const queryFmtRAMRequests = `avg(avg_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte", container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryRAMRequests")
+	}
+
+	queryRAMRequests := fmt.Sprintf(queryFmtRAMRequests, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeRAMRequestsResult, ctx.QueryAtTime(queryRAMRequests, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryRAMUsageAvg(start, end time.Time) *source.Future[source.RAMUsageAvgResult] {
+	const queryFmtRAMUsageAvg = `avg(avg_over_time(container_memory_working_set_bytes{container!="", container_name!="POD", container!="POD", %s}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryRAMUsageAvg")
+	}
+
+	queryRAMUsageAvg := fmt.Sprintf(queryFmtRAMUsageAvg, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeRAMUsageAvgResult, ctx.QueryAtTime(queryRAMUsageAvg, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryRAMUsageMax(start, end time.Time) *source.Future[source.RAMUsageMaxResult] {
+	const queryFmtRAMUsageMax = `max(max_over_time(container_memory_working_set_bytes{container!="", container_name!="POD", container!="POD", %s}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryRAMUsageMax")
+	}
+
+	queryRAMUsageMax := fmt.Sprintf(queryFmtRAMUsageMax, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeRAMUsageMaxResult, ctx.QueryAtTime(queryRAMUsageMax, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryCPUCoresAllocated(start, end time.Time) *source.Future[source.CPUCoresAllocatedResult] {
+	const queryFmtCPUCoresAllocated = `avg(avg_over_time(container_cpu_allocation{container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryCPUCoresAllocated")
+	}
+
+	queryCPUCoresAllocated := fmt.Sprintf(queryFmtCPUCoresAllocated, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeCPUCoresAllocatedResult, ctx.QueryAtTime(queryCPUCoresAllocated, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryCPURequests(start, end time.Time) *source.Future[source.CPURequestsResult] {
+	const queryFmtCPURequests = `avg(avg_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryCPURequests")
+	}
+
+	queryCPURequests := fmt.Sprintf(queryFmtCPURequests, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeCPURequestsResult, ctx.QueryAtTime(queryCPURequests, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryCPUUsageAvg(start, end time.Time) *source.Future[source.CPUUsageAvgResult] {
+	const queryFmtCPUUsageAvg = `avg(rate(container_cpu_usage_seconds_total{container!="", container_name!="POD", container!="POD", %s}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryCPUUsageAvg")
+	}
+
+	queryCPUUsageAvg := fmt.Sprintf(queryFmtCPUUsageAvg, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeCPUUsageAvgResult, ctx.QueryAtTime(queryCPUUsageAvg, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryCPUUsageMax(start, end time.Time) *source.Future[source.CPUUsageMaxResult] {
+	// Because we use container_cpu_usage_seconds_total to calculate CPU usage
+	// at any given "instant" of time, we need to use an irate or rate. To then
+	// calculate a max (or any aggregation) we have to perform an aggregation
+	// query on top of an instant-by-instant maximum. Prometheus supports this
+	// type of query with a "subquery" [1], however it is reportedly expensive
+	// to make such a query. By default, Kubecost's Prometheus config includes
+	// a recording rule that keeps track of the instant-by-instant irate for CPU
+	// usage. The metric in this query is created by that recording rule.
+	//
+	// [1] https://prometheus.io/blog/2019/01/28/subquery-support/
+	//
+	// If changing the name of the recording rule, make sure to update the
+	// corresponding diagnostic query to avoid confusion.
+	const queryFmtCPUUsageMaxRecordingRule = `max(max_over_time(kubecost_container_cpu_usage_irate{%s}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
+
+	// This is the subquery equivalent of the above recording rule query. It is
+	// more expensive, but does not require the recording rule. It should be
+	// used as a fallback query if the recording rule data does not exist.
+	//
+	// The parameter after the colon [:<thisone>] in the subquery affects the
+	// resolution of the subquery.
+	// The parameter after the metric ...{}[<thisone>] should be set to 2x
+	// the resolution, to make sure the irate always has two points to query
+	// in case the Prom scrape duration has been reduced to be equal to the
+	// ETL resolution.
+	const queryFmtCPUUsageMaxSubquery = `max(max_over_time(irate(container_cpu_usage_seconds_total{container!="POD", container!="", %s}[%s])[%s:%s])) by (container, pod_name, pod, namespace, instance, %s)`
+	// env.GetPromClusterFilter(), doubleResStr, durStr, resStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryCPUUsageMax")
+	}
+
+	queryCPUUsageMaxRecordingRule := fmt.Sprintf(queryFmtCPUUsageMaxRecordingRule, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	resCPUUsageMaxRR := ctx.QueryAtTime(queryCPUUsageMaxRecordingRule, end)
+	resCPUUsageMax, _ := resCPUUsageMaxRR.Await()
+
+	if len(resCPUUsageMax) > 0 {
+		return wrapResults(queryCPUUsageMaxRecordingRule, source.DecodeCPUUsageMaxResult, resCPUUsageMax)
+	}
+
+	resolution := cfg.DataResolution
+	resStr := timeutil.DurationString(resolution)
+	doubleResStr := timeutil.DurationString(2 * resolution)
+
+	queryCPUUsageMaxSubquery := fmt.Sprintf(queryFmtCPUUsageMaxSubquery, cfg.ClusterFilter, doubleResStr, durStr, resStr, cfg.ClusterLabel)
+	return source.NewFuture(source.DecodeCPUUsageMaxResult, ctx.QueryAtTime(queryCPUUsageMaxSubquery, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryGPUsRequested(start, end time.Time) *source.Future[source.GPUsRequestedResult] {
+	const queryFmtGPUsRequested = `avg(avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryGPUsRequested")
+	}
+
+	queryGPUsRequested := fmt.Sprintf(queryFmtGPUsRequested, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeGPUsRequestedResult, ctx.QueryAtTime(queryGPUsRequested, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryGPUsUsageAvg(start, end time.Time) *source.Future[source.GPUsUsageAvgResult] {
+	const queryFmtGPUsUsageAvg = `avg(avg_over_time(DCGM_FI_PROF_GR_ENGINE_ACTIVE{container!=""}[%s])) by (container, pod, namespace, %s)`
+	// durStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryGPUsUsageAvg")
+	}
+
+	queryGPUsUsageAvg := fmt.Sprintf(queryFmtGPUsUsageAvg, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeGPUsUsageAvgResult, ctx.QueryAtTime(queryGPUsUsageAvg, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryGPUsUsageMax(start, end time.Time) *source.Future[source.GPUsUsageMaxResult] {
+	const queryFmtGPUsUsageMax = `max(max_over_time(DCGM_FI_PROF_GR_ENGINE_ACTIVE{container!=""}[%s])) by (container, pod, namespace, %s)`
+	// durStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryGPUsUsageMax")
+	}
+
+	queryGPUsUsageMax := fmt.Sprintf(queryFmtGPUsUsageMax, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeGPUsUsageMaxResult, ctx.QueryAtTime(queryGPUsUsageMax, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryGPUsAllocated(start, end time.Time) *source.Future[source.GPUsAllocatedResult] {
+	const queryFmtGPUsAllocated = `avg(avg_over_time(container_gpu_allocation{container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryGPUsAllocated")
+	}
+
+	queryGPUsAllocated := fmt.Sprintf(queryFmtGPUsAllocated, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeGPUsAllocatedResult, ctx.QueryAtTime(queryGPUsAllocated, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryIsGPUShared(start, end time.Time) *source.Future[source.IsGPUSharedResult] {
+	const queryFmtIsGPUShared = `avg(avg_over_time(kube_pod_container_resource_requests{container!="", node != "", pod != "", container!= "", unit = "integer",  %s}[%s])) by (container, pod, namespace, node, resource)`
+	// env.GetPromClusterFilter(), durStr
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryIsGPUShared")
+	}
+
+	queryIsGPUShared := fmt.Sprintf(queryFmtIsGPUShared, cfg.ClusterFilter, durStr)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeIsGPUSharedResult, ctx.QueryAtTime(queryIsGPUShared, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryGPUInfo(start, end time.Time) *source.Future[source.GPUInfoResult] {
+	const queryFmtGetGPUInfo = `avg(avg_over_time(DCGM_FI_DEV_DEC_UTIL{container!="",%s}[%s])) by (container, pod, namespace, device, modelName, UUID)`
+	// env.GetPromClusterFilter(), durStr
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryGPUInfo")
+	}
+
+	queryGetGPUInfo := fmt.Sprintf(queryFmtGetGPUInfo, cfg.ClusterFilter, durStr)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeGPUInfoResult, ctx.QueryAtTime(queryGetGPUInfo, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNodeCPUPricePerHr(start, end time.Time) *source.Future[source.NodeCPUPricePerHrResult] {
+	const queryFmtNodeCostPerCPUHr = `avg(avg_over_time(node_cpu_hourly_cost{%s}[%s])) by (node, %s, instance_type, provider_id)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNodeCPUPricePerHr")
+	}
+
+	queryNodeCostPerCPUHr := fmt.Sprintf(queryFmtNodeCostPerCPUHr, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeNodeCPUPricePerHrResult, ctx.QueryAtTime(queryNodeCostPerCPUHr, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNodeRAMPricePerGiBHr(start, end time.Time) *source.Future[source.NodeRAMPricePerGiBHrResult] {
+	const queryFmtNodeCostPerRAMGiBHr = `avg(avg_over_time(node_ram_hourly_cost{%s}[%s])) by (node, %s, instance_type, provider_id)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNodeRAMPricePerGiBHr")
+	}
+
+	queryNodeCostPerRAMGiBHr := fmt.Sprintf(queryFmtNodeCostPerRAMGiBHr, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeNodeRAMPricePerGiBHrResult, ctx.QueryAtTime(queryNodeCostPerRAMGiBHr, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNodeGPUPricePerHr(start, end time.Time) *source.Future[source.NodeGPUPricePerHrResult] {
+	const queryFmtNodeCostPerGPUHr = `avg(avg_over_time(node_gpu_hourly_cost{%s}[%s])) by (node, %s, instance_type, provider_id)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNodeGPUPricePerHr")
+	}
+
+	queryNodeCostPerGPUHr := fmt.Sprintf(queryFmtNodeCostPerGPUHr, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeNodeGPUPricePerHrResult, ctx.QueryAtTime(queryNodeCostPerGPUHr, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNodeIsSpot(start, end time.Time) *source.Future[source.NodeIsSpotResult] {
+	const queryFmtNodeIsSpot = `avg_over_time(kubecost_node_is_spot{%s}[%s])`
+	//`avg_over_time(kubecost_node_is_spot{%s}[%s:%dm])`
+	// env.GetPromClusterFilter(), durStr)
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNodeIsSpot2")
+	}
+
+	queryNodeIsSpot := fmt.Sprintf(queryFmtNodeIsSpot, cfg.ClusterFilter, durStr)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeNodeIsSpotResult, ctx.QueryAtTime(queryNodeIsSpot, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryPodPVCAllocation(start, end time.Time) *source.Future[source.PodPVCAllocationResult] {
+	const queryFmtPodPVCAllocation = `avg(avg_over_time(pod_pvc_allocation{%s}[%s])) by (persistentvolume, persistentvolumeclaim, pod, namespace, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryPodPVCAllocation")
+	}
+
+	queryPodPVCAllocation := fmt.Sprintf(queryFmtPodPVCAllocation, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodePodPVCAllocationResult, ctx.QueryAtTime(queryPodPVCAllocation, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryPVCBytesRequested(start, end time.Time) *source.Future[source.PVCBytesRequestedResult] {
+	const queryFmtPVCBytesRequested = `avg(avg_over_time(kube_persistentvolumeclaim_resource_requests_storage_bytes{%s}[%s])) by (persistentvolumeclaim, namespace, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryPVCBytesRequested")
+	}
+
+	queryPVCBytesRequested := fmt.Sprintf(queryFmtPVCBytesRequested, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodePVCBytesRequestedResult, ctx.QueryAtTime(queryPVCBytesRequested, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryPVBytes(start, end time.Time) *source.Future[source.PVBytesResult] {
+	const queryFmtPVBytes = `avg(avg_over_time(kube_persistentvolume_capacity_bytes{%s}[%s])) by (persistentvolume, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryPVBytes")
+	}
+
+	queryPVBytes := fmt.Sprintf(queryFmtPVBytes, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodePVBytesResult, ctx.QueryAtTime(queryPVBytes, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryPVCostPerGiBHour(start, end time.Time) *source.Future[source.PVPricePerGiBHourResult] {
+	const queryFmtPVCostPerGiBHour = `avg(avg_over_time(pv_hourly_cost{%s}[%s])) by (volumename, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryPVCostPerGiBHour")
+	}
+
+	queryPVCostPerGiBHour := fmt.Sprintf(queryFmtPVCostPerGiBHour, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodePVPricePerGiBHourResult, ctx.QueryAtTime(queryPVCostPerGiBHour, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryPVInfo(start, end time.Time) *source.Future[source.PVInfoResult] {
+	const queryFmtPVMeta = `avg(avg_over_time(kubecost_pv_info{%s}[%s])) by (%s, storageclass, persistentvolume, provider_id)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryPVMeta")
+	}
+
+	queryPVMeta := fmt.Sprintf(queryFmtPVMeta, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodePVInfoResult, ctx.QueryAtTime(queryPVMeta, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNetZoneGiB(start, end time.Time) *source.Future[source.NetZoneGiBResult] {
+	const queryFmtNetZoneGiB = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", same_zone="false", same_region="true", %s}[%s])) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNetZoneGiB")
+	}
+
+	queryNetZoneGiB := fmt.Sprintf(queryFmtNetZoneGiB, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeNetZoneGiBResult, ctx.QueryAtTime(queryNetZoneGiB, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNetZonePricePerGiB(start, end time.Time) *source.Future[source.NetZonePricePerGiBResult] {
+	const queryFmtNetZoneCostPerGiB = `avg(avg_over_time(kubecost_network_zone_egress_cost{%s}[%s])) by (%s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNetZonePricePerGiB")
+	}
+
+	queryNetZoneCostPerGiB := fmt.Sprintf(queryFmtNetZoneCostPerGiB, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeNetZonePricePerGiBResult, ctx.QueryAtTime(queryNetZoneCostPerGiB, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNetRegionGiB(start, end time.Time) *source.Future[source.NetRegionGiBResult] {
+	const queryFmtNetRegionGiB = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", same_zone="false", same_region="false", %s}[%s])) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNetRegionGiB")
+	}
+
+	queryNetRegionGiB := fmt.Sprintf(queryFmtNetRegionGiB, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeNetRegionGiBResult, ctx.QueryAtTime(queryNetRegionGiB, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNetRegionPricePerGiB(start, end time.Time) *source.Future[source.NetRegionPricePerGiBResult] {
+	const queryFmtNetRegionCostPerGiB = `avg(avg_over_time(kubecost_network_region_egress_cost{%s}[%s])) by (%s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNetRegionPricePerGiB")
+	}
+
+	queryNetRegionCostPerGiB := fmt.Sprintf(queryFmtNetRegionCostPerGiB, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeNetRegionPricePerGiBResult, ctx.QueryAtTime(queryNetRegionCostPerGiB, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNetInternetGiB(start, end time.Time) *source.Future[source.NetInternetGiBResult] {
+	const queryFmtNetInternetGiB = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="true", %s}[%s])) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNetInternetGiB")
+	}
+
+	queryNetInternetGiB := fmt.Sprintf(queryFmtNetInternetGiB, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeNetInternetGiBResult, ctx.QueryAtTime(queryNetInternetGiB, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNetInternetPricePerGiB(start, end time.Time) *source.Future[source.NetInternetPricePerGiBResult] {
+	const queryFmtNetInternetCostPerGiB = `avg(avg_over_time(kubecost_network_internet_egress_cost{%s}[%s])) by (%s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel()
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNetInternetPricePerGiB")
+	}
+
+	queryNetInternetCostPerGiB := fmt.Sprintf(queryFmtNetInternetCostPerGiB, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeNetInternetPricePerGiBResult, ctx.QueryAtTime(queryNetInternetCostPerGiB, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNetReceiveBytes(start, end time.Time) *source.Future[source.NetReceiveBytesResult] {
+	const queryFmtNetReceiveBytes = `sum(increase(container_network_receive_bytes_total{pod!="", %s}[%s])) by (pod_name, pod, namespace, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNetReceiveBytes")
+	}
+
+	queryNetReceiveBytes := fmt.Sprintf(queryFmtNetReceiveBytes, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeNetReceiveBytesResult, ctx.QueryAtTime(queryNetReceiveBytes, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNetTransferBytes(start, end time.Time) *source.Future[source.NetTransferBytesResult] {
+	const queryFmtNetTransferBytes = `sum(increase(container_network_transmit_bytes_total{pod!="", %s}[%s])) by (pod_name, pod, namespace, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNetTransferBytes")
+	}
+
+	queryNetTransferBytes := fmt.Sprintf(queryFmtNetTransferBytes, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeNetTransferBytesResult, ctx.QueryAtTime(queryNetTransferBytes, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNamespaceLabels(start, end time.Time) *source.Future[source.NamespaceLabelsResult] {
+	const queryFmtNamespaceLabels = `avg_over_time(kube_namespace_labels{%s}[%s])`
+	// env.GetPromClusterFilter(), durStr
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNamespaceLabels")
+	}
+
+	queryNamespaceLabels := fmt.Sprintf(queryFmtNamespaceLabels, cfg.ClusterFilter, durStr)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeNamespaceLabelsResult, ctx.QueryAtTime(queryNamespaceLabels, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryNamespaceAnnotations(start, end time.Time) *source.Future[source.NamespaceAnnotationsResult] {
+	const queryFmtNamespaceAnnotations = `avg_over_time(kube_namespace_annotations{%s}[%s])`
+	// env.GetPromClusterFilter(), durStr
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNamespaceAnnotations")
+	}
+
+	queryNamespaceAnnotations := fmt.Sprintf(queryFmtNamespaceAnnotations, cfg.ClusterFilter, durStr)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeNamespaceAnnotationsResult, ctx.QueryAtTime(queryNamespaceAnnotations, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryPodLabels(start, end time.Time) *source.Future[source.PodLabelsResult] {
+	const queryFmtPodLabels = `avg_over_time(kube_pod_labels{%s}[%s])`
+	// env.GetPromClusterFilter(), durStr
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryPodLabels")
+	}
+
+	queryPodLabels := fmt.Sprintf(queryFmtPodLabels, cfg.ClusterFilter, durStr)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodePodLabelsResult, ctx.QueryAtTime(queryPodLabels, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryPodAnnotations(start, end time.Time) *source.Future[source.PodAnnotationsResult] {
+	const queryFmtPodAnnotations = `avg_over_time(kube_pod_annotations{%s}[%s])`
+	// env.GetPromClusterFilter(), durStr
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryPodAnnotations")
+	}
+
+	queryPodAnnotations := fmt.Sprintf(queryFmtPodAnnotations, cfg.ClusterFilter, durStr)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodePodAnnotationsResult, ctx.QueryAtTime(queryPodAnnotations, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryServiceLabels(start, end time.Time) *source.Future[source.ServiceLabelsResult] {
+	const queryFmtServiceLabels = `avg_over_time(service_selector_labels{%s}[%s])`
+	// env.GetPromClusterFilter(), durStr
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryServiceLabels")
+	}
+
+	queryServiceLabels := fmt.Sprintf(queryFmtServiceLabels, cfg.ClusterFilter, durStr)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeServiceLabelsResult, ctx.QueryAtTime(queryServiceLabels, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryDeploymentLabels(start, end time.Time) *source.Future[source.DeploymentLabelsResult] {
+	const queryFmtDeploymentLabels = `avg_over_time(deployment_match_labels{%s}[%s])`
+	// env.GetPromClusterFilter(), durStr
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryNamespaceAnnotations")
+	}
+
+	queryDeploymentLabels := fmt.Sprintf(queryFmtDeploymentLabels, cfg.ClusterFilter, durStr)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeDeploymentLabelsResult, ctx.QueryAtTime(queryDeploymentLabels, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryStatefulSetLabels(start, end time.Time) *source.Future[source.StatefulSetLabelsResult] {
+	const queryFmtStatefulSetLabels = `avg_over_time(statefulSet_match_labels{%s}[%s])`
+	// env.GetPromClusterFilter(), durStr
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryStatefulSetLabels")
+	}
+
+	queryStatefulSetLabels := fmt.Sprintf(queryFmtStatefulSetLabels, cfg.ClusterFilter, durStr)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeStatefulSetLabelsResult, ctx.QueryAtTime(queryStatefulSetLabels, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryDaemonSetLabels(start, end time.Time) *source.Future[source.DaemonSetLabelsResult] {
+	const queryFmtDaemonSetLabels = `sum(avg_over_time(kube_pod_owner{owner_kind="DaemonSet", %s}[%s])) by (pod, owner_name, namespace, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryDaemonSetLabels")
+	}
+
+	queryDaemonSetLabels := fmt.Sprintf(queryFmtDaemonSetLabels, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeDaemonSetLabelsResult, ctx.QueryAtTime(queryDaemonSetLabels, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryJobLabels(start, end time.Time) *source.Future[source.JobLabelsResult] {
+	const queryFmtJobLabels = `sum(avg_over_time(kube_pod_owner{owner_kind="Job", %s}[%s])) by (pod, owner_name, namespace ,%s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryJobLabels")
+	}
+
+	queryJobLabels := fmt.Sprintf(queryFmtJobLabels, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeJobLabelsResult, ctx.QueryAtTime(queryJobLabels, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryPodsWithReplicaSetOwner(start, end time.Time) *source.Future[source.PodsWithReplicaSetOwnerResult] {
+	const queryFmtPodsWithReplicaSetOwner = `sum(avg_over_time(kube_pod_owner{owner_kind="ReplicaSet", %s}[%s])) by (pod, owner_name, namespace ,%s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryPodsWithReplicaSetOwner")
+	}
+
+	queryPodsWithReplicaSetOwner := fmt.Sprintf(queryFmtPodsWithReplicaSetOwner, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodePodsWithReplicaSetOwnerResult, ctx.QueryAtTime(queryPodsWithReplicaSetOwner, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryReplicaSetsWithoutOwners(start, end time.Time) *source.Future[source.ReplicaSetsWithoutOwnersResult] {
+	const queryFmtReplicaSetsWithoutOwners = `avg(avg_over_time(kube_replicaset_owner{owner_kind="<none>", owner_name="<none>", %s}[%s])) by (replicaset, namespace, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryReplicaSetsWithoutOwners")
+	}
+
+	queryReplicaSetsWithoutOwners := fmt.Sprintf(queryFmtReplicaSetsWithoutOwners, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeReplicaSetsWithoutOwnersResult, ctx.QueryAtTime(queryReplicaSetsWithoutOwners, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryReplicaSetsWithRollout(start, end time.Time) *source.Future[source.ReplicaSetsWithRolloutResult] {
+	const queryFmtReplicaSetsWithRolloutOwner = `avg(avg_over_time(kube_replicaset_owner{owner_kind="Rollout", %s}[%s])) by (replicaset, namespace, owner_kind, owner_name, %s)`
+	// env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+
+	cfg := pds.promConfig
+
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		panic("failed to parse duration string passed to QueryReplicaSetsWithRollout")
+	}
+
+	queryReplicaSetsWithRolloutOwner := fmt.Sprintf(queryFmtReplicaSetsWithRolloutOwner, cfg.ClusterFilter, durStr, cfg.ClusterLabel)
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	return source.NewFuture(source.DecodeReplicaSetsWithRolloutResult, ctx.QueryAtTime(queryReplicaSetsWithRolloutOwner, end))
+}
+
+func (pds *PrometheusMetricsQuerier) QueryDataCoverage(limitDays int) (time.Time, time.Time, error) {
+	const (
+		queryFmtOldestSample = `min_over_time(timestamp(group(node_cpu_hourly_cost{%s}))[%s:%s])`
+		queryFmtNewestSample = `max_over_time(timestamp(group(node_cpu_hourly_cost{%s}))[%s:%s])`
+	)
+
+	cfg := pds.promConfig
+	now := time.Now()
+	durStr := fmt.Sprintf("%dd", limitDays)
+
+	ctx := pds.promContexts.NewNamedContext(AllocationContextName)
+	queryOldest := fmt.Sprintf(queryFmtOldestSample, cfg.ClusterFilter, durStr, "1h")
+	resOldestFut := ctx.QueryAtTime(queryOldest, now)
+
+	resOldest, err := resOldestFut.Await()
+	if err != nil {
+		return time.Time{}, time.Time{}, fmt.Errorf("querying oldest sample: %w", err)
+	}
+	if len(resOldest) == 0 || len(resOldest[0].Values) == 0 {
+		return time.Time{}, time.Time{}, fmt.Errorf("querying oldest sample: %w", err)
+	}
+
+	oldest := time.Unix(int64(resOldest[0].Values[0].Value), 0)
+
+	queryNewest := fmt.Sprintf(queryFmtNewestSample, cfg.ClusterFilter, durStr, "1h")
+	resNewestFut := ctx.QueryAtTime(queryNewest, now)
+
+	resNewest, err := resNewestFut.Await()
+	if err != nil {
+		return time.Time{}, time.Time{}, fmt.Errorf("querying newest sample: %w", err)
+	}
+	if len(resNewest) == 0 || len(resNewest[0].Values) == 0 {
+		return time.Time{}, time.Time{}, fmt.Errorf("querying newest sample: %w", err)
+	}
+
+	newest := time.Unix(int64(resNewest[0].Values[0].Value), 0)
+
+	return oldest, newest, nil
+}
+
+func newEmptyResult[T any](decoder source.ResultDecoder[T]) *source.Future[T] {
+	ch := make(source.QueryResultsChan)
+	go func() {
+		results := source.NewQueryResults("")
+		ch <- results
+	}()
+
+	return source.NewFuture(decoder, ch)
+}
+
+func wrapResults[T any](query string, decoder source.ResultDecoder[T], results []*source.QueryResult) *source.Future[T] {
+	ch := make(source.QueryResultsChan)
+
+	go func() {
+		r := source.NewQueryResults(query)
+		r.Results = results
+		ch <- r
+	}()
+
+	return source.NewFuture(decoder, ch)
+}

pkg/cmd/agent/agent.go

@@ -67,6 +67,37 @@ func newKubernetesClusterCache() (kubernetes.Interface, clustercache.ClusterCach
 func Execute(opts *AgentOpts) error {
 	log.Infof("Starting Kubecost Agent version %s", version.FriendlyVersion())
 
+	// initialize kubernetes client and cluster cache
+	k8sClient, clusterCache, err := newKubernetesClusterCache()
+	if err != nil {
+		panic(err.Error())
+	}
+
+	// Create ConfigFileManager for synchronization of shared configuration
+	confManager := config.NewConfigFileManager(&config.ConfigFileManagerOpts{
+		BucketStoreConfig: env.GetKubecostConfigBucket(),
+		LocalConfigPath:   "/",
+	})
+
+	configPrefix := env.GetConfigPathWithDefault(env.DefaultConfigMountPath)
+
+	cloudProviderKey := env.GetCloudProviderAPIKey()
+	cloudProvider, err := provider.NewProvider(clusterCache, cloudProviderKey, confManager)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	// ClusterInfo Provider to provide the cluster map with local and remote cluster data
+	localClusterInfo := costmodel.NewLocalClusterInfoProvider(k8sClient, cloudProvider)
+
+	var clusterInfoProvider clusters.ClusterInfoProvider
+	if env.IsExportClusterInfoEnabled() {
+		clusterInfoConf := confManager.ConfigFileAt(path.Join(configPrefix, "cluster-info.json"))
+		clusterInfoProvider = costmodel.NewClusterInfoWriteOnRequest(localClusterInfo, clusterInfoConf)
+	} else {
+		clusterInfoProvider = localClusterInfo
+	}
+
 	const maxRetries = 10
 	const retryInterval = 10 * time.Second
 
@@ -76,7 +107,7 @@ func Execute(opts *AgentOpts) error {
 	dataSource, err := retry.Retry(
 		ctx,
 		func() (source.OpenCostDataSource, error) {
-			ds, e := prom.NewDefaultPrometheusDataSource()
+			ds, e := prom.NewDefaultPrometheusDataSource(clusterInfoProvider)
 			if e != nil {
 				if source.IsRetryable(e) {
 					return nil, e
@@ -96,32 +127,12 @@ func Execute(opts *AgentOpts) error {
 		panic(fatalErr)
 	}
 
-	// initialize kubernetes client and cluster cache
-	k8sClient, clusterCache, err := newKubernetesClusterCache()
-	if err != nil {
-		panic(err.Error())
-	}
-
-	// Create ConfigFileManager for synchronization of shared configuration
-	confManager := config.NewConfigFileManager(&config.ConfigFileManagerOpts{
-		BucketStoreConfig: env.GetKubecostConfigBucket(),
-		LocalConfigPath:   "/",
-	})
-
-	cloudProviderKey := env.GetCloudProviderAPIKey()
-	cloudProvider, err := provider.NewProvider(clusterCache, cloudProviderKey, confManager)
-	if err != nil {
-		panic(err.Error())
-	}
-
 	// Append the pricing config watcher
 	kubecostNamespace := env.GetKubecostNamespace()
 	configWatchers := watcher.NewConfigMapWatchers(k8sClient, kubecostNamespace)
 	configWatchers.AddWatcher(provider.ConfigWatcherFor(cloudProvider))
 	configWatchers.Watch()
 
-	configPrefix := env.GetConfigPathWithDefault(env.DefaultConfigMountPath)
-
 	// Initialize cluster exporting if it's enabled
 	if env.IsExportClusterCacheEnabled() {
 		cacheLocation := confManager.ConfigFileAt(path.Join(configPrefix, "cluster-cache.json"))
@@ -129,19 +140,8 @@ func Execute(opts *AgentOpts) error {
 		clusterExporter.Run()
 	}
 
-	// ClusterInfo Provider to provide the cluster map with local and remote cluster data
-	localClusterInfo := costmodel.NewLocalClusterInfoProvider(k8sClient, dataSource, cloudProvider)
-
-	var clusterInfoProvider clusters.ClusterInfoProvider
-	if env.IsExportClusterInfoEnabled() {
-		clusterInfoConf := confManager.ConfigFileAt(path.Join(configPrefix, "cluster-info.json"))
-		clusterInfoProvider = costmodel.NewClusterInfoWriteOnRequest(localClusterInfo, clusterInfoConf)
-	} else {
-		clusterInfoProvider = localClusterInfo
-	}
-
 	// Initialize ClusterMap for maintaining ClusterInfo by ClusterID
-	clusterMap := dataSource.NewClusterMap(clusterInfoProvider)
+	clusterMap := dataSource.ClusterMap()
 
 	costModel := costmodel.NewCostModel(dataSource, cloudProvider, clusterCache, clusterMap, dataSource.BatchDuration())
 

pkg/costmodel/aggregation.go

@@ -4,17 +4,12 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
-	"time"
 
 	"github.com/julienschmidt/httprouter"
-	"github.com/opencost/opencost/pkg/errors"
 
-	"github.com/opencost/opencost/core/pkg/log"
 	"github.com/opencost/opencost/core/pkg/opencost"
-	"github.com/opencost/opencost/core/pkg/util"
 	"github.com/opencost/opencost/core/pkg/util/httputil"
 	"github.com/opencost/opencost/core/pkg/util/json"
-	"github.com/opencost/opencost/core/pkg/util/timeutil"
 	"github.com/opencost/opencost/pkg/env"
 )
 
@@ -62,124 +57,6 @@ func ParseAggregationProperties(aggregations []string) ([]string, error) {
 	return aggregateBy, nil
 }
 
-func (a *Accesses) warmAggregateCostModelCache() {
-	const clusterCostsCacheMinutes = 5.0
-
-	// Only allow one concurrent cache-warming operation
-	sem := util.NewSemaphore(1)
-
-	// Set default values, pulling them from application settings where applicable, and warm the cache
-	// for the given duration. Cache is intentionally set to expire (i.e. noExpireCache=false) so that
-	// if the default parameters change, the old cached defaults with eventually expire. Thus, the
-	// timing of the cache expiry/refresh is the only mechanism ensuring 100% cache warmth.
-	warmFunc := func(duration, offset time.Duration, cacheEfficiencyData bool) error {
-		fmtDuration, fmtOffset := timeutil.DurationOffsetStrings(duration, offset)
-		durationHrs, _ := timeutil.FormatDurationStringDaysToHours(fmtDuration)
-
-		windowStr := fmt.Sprintf("%s offset %s", fmtDuration, fmtOffset)
-		window, err := opencost.ParseWindowUTC(windowStr)
-		if err != nil {
-			return fmt.Errorf("invalid window from window string: %s", windowStr)
-		}
-
-		key := fmt.Sprintf("%s:%s", durationHrs, fmtOffset)
-
-		totals, err := a.ComputeClusterCosts(a.DataSource, a.CloudProvider, duration, offset, cacheEfficiencyData)
-		if err != nil {
-			log.Infof("Error building cluster costs cache %s", key)
-		}
-		maxMinutesWithData := 0.0
-		for _, cluster := range totals {
-			if cluster.DataMinutes > maxMinutesWithData {
-				maxMinutesWithData = cluster.DataMinutes
-			}
-		}
-		if len(totals) > 0 && maxMinutesWithData > clusterCostsCacheMinutes {
-			a.ClusterCostsCache.Set(key, totals, a.GetCacheExpiration(window.Duration()))
-			log.Infof("caching %s cluster costs for %s", fmtDuration, a.GetCacheExpiration(window.Duration()))
-		} else {
-			log.Warnf("not caching %s cluster costs: no data or less than %f minutes data ", fmtDuration, clusterCostsCacheMinutes)
-		}
-		return err
-	}
-
-	// 1 day
-	go func(sem *util.Semaphore) {
-		defer errors.HandlePanic()
-
-		offset := time.Minute
-		duration := 24 * time.Hour
-
-		for {
-			sem.Acquire()
-			warmFunc(duration, offset, true)
-			sem.Return()
-
-			log.Infof("aggregation: warm cache: %s", timeutil.DurationString(duration))
-			time.Sleep(a.GetCacheRefresh(duration))
-		}
-	}(sem)
-
-	if !env.IsETLEnabled() {
-		// 2 day
-		go func(sem *util.Semaphore) {
-			defer errors.HandlePanic()
-
-			offset := time.Minute
-			duration := 2 * 24 * time.Hour
-
-			for {
-				sem.Acquire()
-				warmFunc(duration, offset, false)
-				sem.Return()
-
-				log.Infof("aggregation: warm cache: %s", timeutil.DurationString(duration))
-				time.Sleep(a.GetCacheRefresh(duration))
-			}
-		}(sem)
-
-		// 7 day
-		go func(sem *util.Semaphore) {
-			defer errors.HandlePanic()
-
-			offset := time.Minute
-			duration := 7 * 24 * time.Hour
-
-			for {
-				sem.Acquire()
-				err := warmFunc(duration, offset, false)
-				sem.Return()
-
-				log.Infof("aggregation: warm cache: %s", timeutil.DurationString(duration))
-				if err == nil {
-					time.Sleep(a.GetCacheRefresh(duration))
-				} else {
-					time.Sleep(5 * time.Minute)
-				}
-			}
-		}(sem)
-
-		// 30 day
-		go func(sem *util.Semaphore) {
-			defer errors.HandlePanic()
-
-			for {
-				offset := time.Minute
-				duration := 30 * 24 * time.Hour
-
-				sem.Acquire()
-				err := warmFunc(duration, offset, false)
-				sem.Return()
-				if err == nil {
-					time.Sleep(a.GetCacheRefresh(duration))
-				} else {
-					time.Sleep(5 * time.Minute)
-				}
-			}
-		}(sem)
-	}
-}
-
 func (a *Accesses) ComputeAllocationHandlerSummary(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")
 

pkg/costmodel/allocation.go

@@ -213,7 +213,7 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 // DateRange checks the data (up to 90 days in the past), and returns the oldest and newest sample timestamp from opencost scraping metric
 // it supposed to be a good indicator of available allocation data
 func (cm *CostModel) DateRange(limitDays int) (time.Time, time.Time, error) {
-	return cm.DataSource.QueryDataCoverage(limitDays)
+	return cm.DataSource.Metrics().QueryDataCoverage(limitDays)
 }
 
 func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Duration) (*opencost.AllocationSet, map[nodeKey]*nodePricing, error) {
@@ -277,7 +277,7 @@ func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Dur
 	}
 
 	grp := source.NewQueryGroup()
-	ds := cm.DataSource
+	ds := cm.DataSource.Metrics()
 
 	resChRAMBytesAllocated := source.WithGroup(grp, ds.QueryRAMBytesAllocated(start, end))
 	resChRAMRequests := source.WithGroup(grp, ds.QueryRAMRequests(start, end))

pkg/costmodel/allocation_helpers.go

@@ -43,7 +43,8 @@ func (cm *CostModel) buildPodMap(window opencost.Window, maxBatchSize time.Durat
 	start, end := *window.Start(), *window.End()
 
 	grp := source.NewQueryGroup()
-	ds := cm.DataSource
+	ds := cm.DataSource.Metrics()
+	resolution := cm.DataSource.Resolution()
 
 	// Query for (start, end) by (pod, namespace, cluster) over the given
 	// window, using the given resolution, and if necessary in batches no
@@ -120,7 +121,7 @@ func (cm *CostModel) buildPodMap(window opencost.Window, maxBatchSize time.Durat
 			}
 		}
 
-		applyPodResults(window, ds.Resolution(), podMap, clusterStart, clusterEnd, resPods, ingestPodUID, podUIDKeyMap)
+		applyPodResults(window, resolution, podMap, clusterStart, clusterEnd, resPods, ingestPodUID, podUIDKeyMap)
 
 		coverage = coverage.ExpandEnd(batchEnd)
 		numQuery++

pkg/costmodel/allocation_incubating.go

@@ -244,7 +244,7 @@ type extendedNodeQueryResults struct {
 
 // queryExtendedNodeData makes additional prometheus queries for node data to append on
 // the AllocationNodePricing struct.
-func queryExtendedNodeData(grp *source.QueryGroup, ds source.OpenCostDataSource, start, end time.Time) (*extendedNodeQueryResults, error) {
+func queryExtendedNodeData(grp *source.QueryGroup, ds source.MetricsQuerier, start, end time.Time) (*extendedNodeQueryResults, error) {
 	resChQueryNodeCPUCores := grp.With(ds.QueryNodeCPUCoresCapacity(start, end))
 	resChQueryNodeRAMBytes := grp.With(ds.QueryNodeRAMBytesCapacity(start, end))
 	resChQueryNodeGPUCount := grp.With(ds.QueryNodeGPUCount(start, end))

pkg/costmodel/allocation_types.go

@@ -202,11 +202,6 @@ func (p *pv) minutes() float64 {
 	return p.End.Sub(p.Start).Minutes()
 }
 
-// key returns the pvKey for the calling pvc
-func (p *pv) key() pvKey {
-	return newPVKey(p.Cluster, p.Name)
-}
-
 // lbCost describes the start and end time of a Load Balancer along with cost
 type lbCost struct {
 	TotalCost float64

pkg/costmodel/allocationnode.go

@@ -22,7 +22,7 @@ import (
 type extendedNodeQueryResults struct{}
 
 // queryExtendedNodeData is a place holder function for the incubating feature
-func queryExtendedNodeData(_ *source.QueryGroup, _ source.OpenCostDataSource, _, _ time.Time) (*extendedNodeQueryResults, error) {
+func queryExtendedNodeData(_ *source.QueryGroup, _ source.MetricsQuerier, _, _ time.Time) (*extendedNodeQueryResults, error) {
 	return &extendedNodeQueryResults{}, nil
 }
 

pkg/costmodel/assets.go

@@ -91,6 +91,7 @@ func (cm *CostModel) ComputeAssets(start, end time.Time) (*opencost.AssetSet, er
 		loadBalancer := opencost.NewLoadBalancer(lb.Name, lb.Cluster, lb.ProviderID, s, e, opencost.NewWindow(&start, &end), lb.Private, lb.Ip)
 		cm.PropertiesFromCluster(loadBalancer.Properties)
 		loadBalancer.Cost = lb.Cost
+
 		assetSet.Insert(loadBalancer, nil)
 	}
 

pkg/costmodel/cluster.go

@@ -131,14 +131,15 @@ func ClusterDisks(dataSource source.OpenCostDataSource, cp models.Provider, star
 	resolution := env.GetETLResolution()
 
 	grp := source.NewQueryGroup()
+	mq := dataSource.Metrics()
 
-	resChPVCost := source.WithGroup(grp, dataSource.QueryPVPricePerGiBHour(start, end))
-	resChPVSize := source.WithGroup(grp, dataSource.QueryPVBytes(start, end))
-	resChActiveMins := source.WithGroup(grp, dataSource.QueryPVActiveMinutes(start, end))
-	resChPVStorageClass := source.WithGroup(grp, dataSource.QueryPVInfo(start, end))
-	resChPVUsedAvg := source.WithGroup(grp, dataSource.QueryPVUsedAverage(start, end))
-	resChPVUsedMax := source.WithGroup(grp, dataSource.QueryPVUsedMax(start, end))
-	resChPVCInfo := source.WithGroup(grp, dataSource.QueryPVCInfo(start, end))
+	resChPVCost := source.WithGroup(grp, mq.QueryPVPricePerGiBHour(start, end))
+	resChPVSize := source.WithGroup(grp, mq.QueryPVBytes(start, end))
+	resChActiveMins := source.WithGroup(grp, mq.QueryPVActiveMinutes(start, end))
+	resChPVStorageClass := source.WithGroup(grp, mq.QueryPVInfo(start, end))
+	resChPVUsedAvg := source.WithGroup(grp, mq.QueryPVUsedAverage(start, end))
+	resChPVUsedMax := source.WithGroup(grp, mq.QueryPVUsedMax(start, end))
+	resChPVCInfo := source.WithGroup(grp, mq.QueryPVCInfo(start, end))
 
 	resPVCost, _ := resChPVCost.Await()
 	resPVSize, _ := resChPVSize.Await()
@@ -165,12 +166,12 @@ func ClusterDisks(dataSource source.OpenCostDataSource, cp models.Provider, star
 	resLocalActiveMins := []*source.LocalStorageActiveMinutesResult{}
 
 	if env.GetAssetIncludeLocalDiskCost() {
-		resChLocalStorageCost := source.WithGroup(grp, dataSource.QueryLocalStorageCost(start, end))
-		resChLocalStorageUsedCost := source.WithGroup(grp, dataSource.QueryLocalStorageUsedCost(start, end))
-		resChLocalStoreageUsedAvg := source.WithGroup(grp, dataSource.QueryLocalStorageUsedAvg(start, end))
-		resChLocalStoreageUsedMax := source.WithGroup(grp, dataSource.QueryLocalStorageUsedMax(start, end))
-		resChLocalStorageBytes := source.WithGroup(grp, dataSource.QueryLocalStorageBytes(start, end))
-		resChLocalActiveMins := source.WithGroup(grp, dataSource.QueryLocalStorageActiveMinutes(start, end))
+		resChLocalStorageCost := source.WithGroup(grp, mq.QueryLocalStorageCost(start, end))
+		resChLocalStorageUsedCost := source.WithGroup(grp, mq.QueryLocalStorageUsedCost(start, end))
+		resChLocalStoreageUsedAvg := source.WithGroup(grp, mq.QueryLocalStorageUsedAvg(start, end))
+		resChLocalStoreageUsedMax := source.WithGroup(grp, mq.QueryLocalStorageUsedMax(start, end))
+		resChLocalStorageBytes := source.WithGroup(grp, mq.QueryLocalStorageBytes(start, end))
+		resChLocalActiveMins := source.WithGroup(grp, mq.QueryLocalStorageActiveMinutes(start, end))
 
 		resLocalStorageCost, _ = resChLocalStorageCost.Await()
 		resLocalStorageUsedCost, _ = resChLocalStorageUsedCost.Await()
@@ -362,7 +363,7 @@ func ClusterDisks(dataSource source.OpenCostDataSource, cp models.Provider, star
 
 		providerID := result.ProviderID
 		if providerID == "" {
-			log.DedupedWarningf(5, "ClusterDisks: local active mins data missing instance")
+			log.DedupedWarningf(5, "ClusterDisks: local active mins data missing provider_id")
 			continue
 		}
 
@@ -534,28 +535,29 @@ func costTimesMinute[T comparable](activeDataMap map[T]activeData, costMap map[T
 }
 
 func ClusterNodes(dataSource source.OpenCostDataSource, cp models.Provider, start, end time.Time) (map[NodeIdentifier]*Node, error) {
+	mq := dataSource.Metrics()
 	resolution := env.GetETLResolution()
 
 	requiredGrp := source.NewQueryGroup()
 	optionalGrp := source.NewQueryGroup()
 
 	// return errors if these fail
-	resChNodeCPUHourlyCost := source.WithGroup(requiredGrp, dataSource.QueryNodeCPUPricePerHr(start, end))
-	resChNodeCPUCoresCapacity := source.WithGroup(requiredGrp, dataSource.QueryNodeCPUCoresCapacity(start, end))
-	resChNodeCPUCoresAllocatable := source.WithGroup(requiredGrp, dataSource.QueryNodeCPUCoresAllocatable(start, end))
-	resChNodeRAMHourlyCost := source.WithGroup(requiredGrp, dataSource.QueryNodeRAMPricePerGiBHr(start, end))
-	resChNodeRAMBytesCapacity := source.WithGroup(requiredGrp, dataSource.QueryNodeRAMBytesCapacity(start, end))
-	resChNodeRAMBytesAllocatable := source.WithGroup(requiredGrp, dataSource.QueryNodeRAMBytesAllocatable(start, end))
-	resChNodeGPUCount := source.WithGroup(requiredGrp, dataSource.QueryNodeGPUCount(start, end))
-	resChNodeGPUHourlyPrice := source.WithGroup(requiredGrp, dataSource.QueryNodeGPUPricePerHr(start, end))
-	resChActiveMins := source.WithGroup(requiredGrp, dataSource.QueryNodeActiveMinutes(start, end))
-	resChIsSpot := source.WithGroup(requiredGrp, dataSource.QueryNodeIsSpot(start, end))
+	resChNodeCPUHourlyCost := source.WithGroup(requiredGrp, mq.QueryNodeCPUPricePerHr(start, end))
+	resChNodeCPUCoresCapacity := source.WithGroup(requiredGrp, mq.QueryNodeCPUCoresCapacity(start, end))
+	resChNodeCPUCoresAllocatable := source.WithGroup(requiredGrp, mq.QueryNodeCPUCoresAllocatable(start, end))
+	resChNodeRAMHourlyCost := source.WithGroup(requiredGrp, mq.QueryNodeRAMPricePerGiBHr(start, end))
+	resChNodeRAMBytesCapacity := source.WithGroup(requiredGrp, mq.QueryNodeRAMBytesCapacity(start, end))
+	resChNodeRAMBytesAllocatable := source.WithGroup(requiredGrp, mq.QueryNodeRAMBytesAllocatable(start, end))
+	resChNodeGPUCount := source.WithGroup(requiredGrp, mq.QueryNodeGPUCount(start, end))
+	resChNodeGPUHourlyPrice := source.WithGroup(requiredGrp, mq.QueryNodeGPUPricePerHr(start, end))
+	resChActiveMins := source.WithGroup(requiredGrp, mq.QueryNodeActiveMinutes(start, end))
+	resChIsSpot := source.WithGroup(requiredGrp, mq.QueryNodeIsSpot(start, end))
 
 	// Do not return errors if these fail, but log warnings
-	resChNodeCPUModeTotal := source.WithGroup(optionalGrp, dataSource.QueryNodeCPUModeTotal(start, end))
-	resChNodeRAMSystemPct := source.WithGroup(optionalGrp, dataSource.QueryNodeRAMSystemPercent(start, end))
-	resChNodeRAMUserPct := source.WithGroup(optionalGrp, dataSource.QueryNodeRAMUserPercent(start, end))
-	resChLabels := source.WithGroup(optionalGrp, dataSource.QueryNodeLabels(start, end))
+	resChNodeCPUModeTotal := source.WithGroup(optionalGrp, mq.QueryNodeCPUModeTotal(start, end))
+	resChNodeRAMSystemPct := source.WithGroup(optionalGrp, mq.QueryNodeRAMSystemPercent(start, end))
+	resChNodeRAMUserPct := source.WithGroup(optionalGrp, mq.QueryNodeRAMUserPercent(start, end))
+	resChLabels := source.WithGroup(optionalGrp, mq.QueryNodeLabels(start, end))
 
 	resNodeCPUHourlyCost, _ := resChNodeCPUHourlyCost.Await()
 	resNodeCPUCoresCapacity, _ := resChNodeCPUCoresCapacity.Await()
@@ -678,9 +680,10 @@ func ClusterLoadBalancers(dataSource source.OpenCostDataSource, start, end time.
 	resolution := env.GetETLResolution()
 
 	grp := source.NewQueryGroup()
+	mq := dataSource.Metrics()
 
-	resChLBCost := source.WithGroup(grp, dataSource.QueryLBPricePerHr(start, end))
-	resChActiveMins := source.WithGroup(grp, dataSource.QueryLBActiveMinutes(start, end))
+	resChLBCost := source.WithGroup(grp, mq.QueryLBPricePerHr(start, end))
+	resChActiveMins := source.WithGroup(grp, mq.QueryLBActiveMinutes(start, end))
 
 	resLBCost, _ := resChLBCost.Await()
 	resActiveMins, _ := resChActiveMins.Await()
@@ -701,12 +704,13 @@ func ClusterLoadBalancers(dataSource source.OpenCostDataSource, start, end time.
 		lbPricePerHr := result.Data[0].Value
 
 		lb := &LoadBalancer{
-			Cluster:   key.Cluster,
-			Namespace: key.Namespace,
-			Name:      key.Name,
-			Cost:      lbPricePerHr, // default to hourly cost, overwrite if active entry exists
-			Ip:        key.IngressIP,
-			Private:   privateIPCheck(key.IngressIP),
+			Cluster:    key.Cluster,
+			Namespace:  key.Namespace,
+			Name:       key.Name,
+			Cost:       lbPricePerHr, // default to hourly cost, overwrite if active entry exists
+			Ip:         key.IngressIP,
+			Private:    privateIPCheck(key.IngressIP),
+			ProviderID: provider.ParseLBID(key.IngressIP),
 		}
 
 		if active, ok := activeMap[key]; ok {
@@ -731,9 +735,10 @@ func ClusterManagement(dataSource source.OpenCostDataSource, start, end time.Tim
 	resolution := env.GetETLResolution()
 
 	grp := source.NewQueryGroup()
+	mq := dataSource.Metrics()
 
-	resChCMPrice := source.WithGroup(grp, dataSource.QueryClusterManagementPricePerHr(start, end))
-	resChCMDur := source.WithGroup(grp, dataSource.QueryClusterManagementDuration(start, end))
+	resChCMPrice := source.WithGroup(grp, mq.QueryClusterManagementPricePerHr(start, end))
+	resChCMDur := source.WithGroup(grp, mq.QueryClusterManagementDuration(start, end))
 
 	resCMPrice, _ := resChCMPrice.Await()
 	resCMDur, _ := resChCMDur.Await()
@@ -778,340 +783,6 @@ func privateIPCheck(ip string) bool {
 	return ipAddress.IsPrivate()
 }
 
-// ComputeClusterCosts gives the cumulative and monthly-rate cluster costs over a window of time for all clusters.
-func (a *Accesses) ComputeClusterCosts(dataSource source.OpenCostDataSource, provider models.Provider, window, offset time.Duration, withBreakdown bool) (map[string]*ClusterCosts, error) {
-	if window < 10*time.Minute {
-		return nil, fmt.Errorf("minimum window of 10m required; got %s", window)
-	}
-
-	// Compute number of minutes in the full interval, for use interpolating missed scrapes or scaling missing data
-	start, end := timeutil.ParseTimeRange(window, offset)
-	mins := end.Sub(start).Minutes()
-
-	providerName := ""
-
-	if clusterInfo, err := provider.ClusterInfo(); err != nil {
-		providerName = clusterInfo["provider"]
-	}
-
-	grp := source.NewQueryGroup()
-
-	queryDataCount := source.WithGroup(grp, dataSource.QueryDataCount(start, end))
-	queryTotalGPU := source.WithGroup(grp, dataSource.QueryTotalGPU(start, end))
-	queryTotalCPU := source.WithGroup(grp, dataSource.QueryTotalCPU(start, end))
-	queryTotalRAM := source.WithGroup(grp, dataSource.QueryTotalRAM(start, end))
-	queryTotalStorage := source.WithGroup(grp, dataSource.QueryTotalStorage(start, end))
-	queryTotalLocalStorage := source.WithGroup(grp, dataSource.QueryLocalStorageBytesByProvider(providerName, start, end))
-
-	var queryCPUModePct *source.QueryGroupFuture[source.NodeCPUModePercentResult]
-	var queryRAMSystemPct *source.QueryGroupFuture[source.NodeRAMSystemPercentResult]
-	var queryRAMUserPct *source.QueryGroupFuture[source.NodeRAMUserPercentResult]
-	var queryUsedLocalStorage *source.QueryGroupFuture[source.LocalStorageUsedByProviderResult]
-
-	if withBreakdown {
-		queryCPUModePct = source.WithGroup(grp, dataSource.QueryNodeCPUModePercent(start, end))
-		queryRAMSystemPct = source.WithGroup(grp, dataSource.QueryNodeRAMSystemPercent(start, end))
-		queryRAMUserPct = source.WithGroup(grp, dataSource.QueryNodeRAMUserPercent(start, end))
-		queryUsedLocalStorage = source.WithGroup(grp, dataSource.QueryLocalStorageUsedByProvider(providerName, start, end))
-	}
-
-	resDataCount, _ := queryDataCount.Await()
-	resTotalGPU, _ := queryTotalGPU.Await()
-	resTotalCPU, _ := queryTotalCPU.Await()
-	resTotalRAM, _ := queryTotalRAM.Await()
-	resTotalStorage, _ := queryTotalStorage.Await()
-
-	if grp.HasErrors() {
-		return nil, grp.Error()
-	}
-
-	defaultClusterID := env.GetClusterID()
-
-	dataMinsByCluster := map[string]float64{}
-	for _, result := range resDataCount {
-		clusterID := result.Cluster
-		if clusterID == "" {
-			clusterID = defaultClusterID
-		}
-
-		dataMins := mins
-		if len(result.Data) > 0 {
-			dataMins = result.Data[0].Value
-		} else {
-			log.Warnf("Cluster cost data count returned no results for cluster %s", clusterID)
-		}
-		dataMinsByCluster[clusterID] = dataMins
-	}
-
-	// Determine combined discount
-	discount, customDiscount := 0.0, 0.0
-	c, err := a.CloudProvider.GetConfig()
-	if err == nil {
-		discount, err = ParsePercentString(c.Discount)
-		if err != nil {
-			discount = 0.0
-		}
-		customDiscount, err = ParsePercentString(c.NegotiatedDiscount)
-		if err != nil {
-			customDiscount = 0.0
-		}
-	}
-
-	// Intermediate structure storing mapping of [clusterID][type ∈ {cpu, ram, storage, total}]=cost
-	costData := make(map[string]map[string]float64)
-
-	// Helper function to iterate over Prom query results, parsing the raw values into
-	// the intermediate costData structure.
-	setCostsFromResults := func(costData map[string]map[string]float64, results []*source.TotalResult, name string, discount float64, customDiscount float64) {
-		for _, result := range results {
-			clusterID := result.Cluster
-			if clusterID == "" {
-				clusterID = defaultClusterID
-			}
-
-			if _, ok := costData[clusterID]; !ok {
-				costData[clusterID] = map[string]float64{}
-			}
-
-			if len(result.Data) > 0 {
-				costData[clusterID][name] += result.Data[0].Value * (1.0 - discount) * (1.0 - customDiscount)
-				costData[clusterID]["total"] += result.Data[0].Value * (1.0 - discount) * (1.0 - customDiscount)
-			}
-		}
-	}
-	// Apply both sustained use and custom discounts to RAM and CPU
-	setCostsFromResults(costData, resTotalCPU, "cpu", discount, customDiscount)
-	setCostsFromResults(costData, resTotalRAM, "ram", discount, customDiscount)
-	// Apply only custom discount to GPU and storage
-	setCostsFromResults(costData, resTotalGPU, "gpu", 0.0, customDiscount)
-	setCostsFromResults(costData, resTotalStorage, "storage", 0.0, customDiscount)
-
-	resTotalLocalStorage, err := queryTotalLocalStorage.Await()
-	if err != nil {
-		return nil, err
-	}
-
-	if len(resTotalLocalStorage) > 0 {
-		setCostsFromResults(costData, resTotalLocalStorage, "localstorage", 0.0, customDiscount)
-	}
-
-	cpuBreakdownMap := map[string]*ClusterCostsBreakdown{}
-	ramBreakdownMap := map[string]*ClusterCostsBreakdown{}
-	pvUsedCostMap := map[string]float64{}
-	if withBreakdown {
-		resCPUModePct, _ := queryCPUModePct.Await()
-		resRAMSystemPct, _ := queryRAMSystemPct.Await()
-		resRAMUserPct, _ := queryRAMUserPct.Await()
-
-		if grp.HasErrors() {
-			return nil, grp.Error()
-		}
-
-		for _, result := range resCPUModePct {
-			clusterID := result.Cluster
-			if clusterID == "" {
-				clusterID = defaultClusterID
-			}
-			if _, ok := cpuBreakdownMap[clusterID]; !ok {
-				cpuBreakdownMap[clusterID] = &ClusterCostsBreakdown{}
-			}
-			cpuBD := cpuBreakdownMap[clusterID]
-
-			mode := result.Mode
-			if mode == "" {
-				log.Warnf("ComputeClusterCosts: unable to read CPU mode: %s", err)
-				mode = "other"
-			}
-
-			switch mode {
-			case "idle":
-				cpuBD.Idle += result.Data[0].Value
-			case "system":
-				cpuBD.System += result.Data[0].Value
-			case "user":
-				cpuBD.User += result.Data[0].Value
-			default:
-				cpuBD.Other += result.Data[0].Value
-			}
-		}
-
-		for _, result := range resRAMSystemPct {
-			clusterID := result.Cluster
-			if clusterID == "" {
-				clusterID = defaultClusterID
-			}
-			if _, ok := ramBreakdownMap[clusterID]; !ok {
-				ramBreakdownMap[clusterID] = &ClusterCostsBreakdown{}
-			}
-			ramBD := ramBreakdownMap[clusterID]
-			ramBD.System += result.Data[0].Value
-		}
-		for _, result := range resRAMUserPct {
-			clusterID := result.Cluster
-			if clusterID == "" {
-				clusterID = defaultClusterID
-			}
-			if _, ok := ramBreakdownMap[clusterID]; !ok {
-				ramBreakdownMap[clusterID] = &ClusterCostsBreakdown{}
-			}
-			ramBD := ramBreakdownMap[clusterID]
-			ramBD.User += result.Data[0].Value
-		}
-		for _, ramBD := range ramBreakdownMap {
-			remaining := 1.0
-			remaining -= ramBD.Other
-			remaining -= ramBD.System
-			remaining -= ramBD.User
-			ramBD.Idle = remaining
-		}
-
-		resUsedLocalStorage, err := queryUsedLocalStorage.Await()
-		if err != nil {
-			return nil, err
-		}
-
-		for _, result := range resUsedLocalStorage {
-			clusterID := result.Cluster
-			if clusterID == "" {
-				clusterID = defaultClusterID
-			}
-			pvUsedCostMap[clusterID] += result.Data[0].Value
-		}
-	}
-
-	if grp.HasErrors() {
-		for _, err := range grp.Errors() {
-			log.Errorf("ComputeClusterCosts: %s", err)
-		}
-		return nil, grp.Error()
-	}
-
-	// Convert intermediate structure to Costs instances
-	costsByCluster := map[string]*ClusterCosts{}
-	for id, cd := range costData {
-		dataMins, ok := dataMinsByCluster[id]
-		if !ok {
-			dataMins = mins
-			log.Warnf("Cluster cost data count not found for cluster %s", id)
-		}
-		costs, err := NewClusterCostsFromCumulative(cd["cpu"], cd["gpu"], cd["ram"], cd["storage"]+cd["localstorage"], window, offset, dataMins/timeutil.MinsPerHour)
-		if err != nil {
-			log.Warnf("Failed to parse cluster costs on %s (%s) from cumulative data: %+v", window, offset, cd)
-			return nil, err
-		}
-
-		if cpuBD, ok := cpuBreakdownMap[id]; ok {
-			costs.CPUBreakdown = cpuBD
-		}
-		if ramBD, ok := ramBreakdownMap[id]; ok {
-			costs.RAMBreakdown = ramBD
-		}
-		costs.StorageBreakdown = &ClusterCostsBreakdown{}
-		if pvUC, ok := pvUsedCostMap[id]; ok {
-			costs.StorageBreakdown.Idle = (costs.StorageCumulative - pvUC) / costs.StorageCumulative
-			costs.StorageBreakdown.User = pvUC / costs.StorageCumulative
-		}
-		costs.DataMinutes = dataMins
-		costsByCluster[id] = costs
-	}
-
-	return costsByCluster, nil
-}
-
-type Totals struct {
-	TotalCost   [][]string `json:"totalcost"`
-	CPUCost     [][]string `json:"cpucost"`
-	MemCost     [][]string `json:"memcost"`
-	StorageCost [][]string `json:"storageCost"`
-}
-
-func resultToTotals(qrs []*source.ClusterResult) ([][]string, error) {
-	if len(qrs) == 0 {
-		return [][]string{}, fmt.Errorf("not enough data available in the selected time range")
-	}
-
-	result := qrs[0]
-	totals := [][]string{}
-	for _, value := range result.Data {
-		d0 := fmt.Sprintf("%f", value.Timestamp)
-		d1 := fmt.Sprintf("%f", value.Value)
-		toAppend := []string{
-			d0,
-			d1,
-		}
-		totals = append(totals, toAppend)
-	}
-	return totals, nil
-}
-
-// ClusterCostsOverTime gives the full cluster costs over time
-func ClusterCostsOverTime(dataSource source.OpenCostDataSource, provider models.Provider, start, end time.Time, window, offset time.Duration) (*Totals, error) {
-	providerName := ""
-
-	if clusterInfo, err := provider.ClusterInfo(); err != nil {
-		providerName = clusterInfo["provider"]
-	}
-
-	grp := source.NewQueryGroup()
-
-	qCores := source.WithGroup(grp, dataSource.QueryClusterCores(start, end, window))
-	qRAM := source.WithGroup(grp, dataSource.QueryClusterRAM(start, end, window))
-	qStorage := source.WithGroup(grp, dataSource.QueryClusterStorageByProvider(providerName, start, end, window))
-	qTotal := source.WithGroup(grp, dataSource.QueryClusterTotalByProvider(providerName, start, end, window))
-
-	resultClusterCores, _ := qCores.Await()
-	resultClusterRAM, _ := qRAM.Await()
-	resultStorage, _ := qStorage.Await()
-	resultTotal, _ := qTotal.Await()
-
-	if grp.HasErrors() {
-		return nil, grp.Error()
-	}
-
-	coreTotal, err := resultToTotals(resultClusterCores)
-	if err != nil {
-		log.Infof("[Warning] ClusterCostsOverTime: no cpu data: %s", err)
-		return nil, err
-	}
-
-	ramTotal, err := resultToTotals(resultClusterRAM)
-	if err != nil {
-		log.Infof("[Warning] ClusterCostsOverTime: no ram data: %s", err)
-		return nil, err
-	}
-
-	storageTotal, err := resultToTotals(resultStorage)
-	if err != nil {
-		log.Infof("[Warning] ClusterCostsOverTime: no storage data: %s", err)
-	}
-
-	clusterTotal, err := resultToTotals(resultTotal)
-	if err != nil {
-		// If clusterTotal query failed, it's likely because there are no PVs, which
-		// causes the qTotal query to return no data. Instead, query only node costs.
-		// If that fails, return an error because something is actually wrong.
-		qNodes := source.WithGroup(grp, dataSource.QueryClusterNodesByProvider(providerName, start, end, window))
-
-		resultNodes, err := qNodes.Await()
-		if err != nil {
-			return nil, err
-		}
-
-		clusterTotal, err = resultToTotals(resultNodes)
-		if err != nil {
-			log.Infof("[Warning] ClusterCostsOverTime: no node data: %s", err)
-			return nil, err
-		}
-	}
-
-	return &Totals{
-		TotalCost:   clusterTotal,
-		CPUCost:     coreTotal,
-		MemCost:     ramTotal,
-		StorageCost: storageTotal,
-	}, nil
-}
-
 func pvCosts(
 	diskMap map[DiskIdentifier]*Disk,
 	resolution time.Duration,

pkg/costmodel/cluster_helpers.go

@@ -1,6 +1,7 @@
 package costmodel
 
 import (
+	"fmt"
 	"strconv"
 	"time"
 
@@ -576,7 +577,7 @@ func loadBalancerKeyGen(result *source.LBActiveMinutesResult) (LoadBalancerIdent
 	return LoadBalancerIdentifier{
 		Cluster:   cluster,
 		Namespace: namespace,
-		Name:      name,
+		Name:      fmt.Sprintf("%s/%s", namespace, name), // TODO:ETL this is kept for backwards-compatibility, but not good,
 		IngressIP: ingressIp,
 	}, true
 }

pkg/costmodel/clusterinfo.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/opencost/opencost/core/pkg/clusters"
 	"github.com/opencost/opencost/core/pkg/log"
-	"github.com/opencost/opencost/core/pkg/source"
 	"github.com/opencost/opencost/core/pkg/util/json"
 	cloudProvider "github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/config"
@@ -35,19 +34,10 @@ func writeClusterProfile(clusterInfo map[string]string) {
 	clusterInfo[clusters.ClusterInfoProfileKey] = clusterProfile
 }
 
-// writeDataSourceMetaData includes the configured thanos flags on the cluster info
-func writeDataSourceMetaData(dataSource source.OpenCostDataSource, clusterInfo map[string]string) {
-	md := dataSource.MetaData()
-	for k, v := range md {
-		clusterInfo[k] = v
-	}
-}
-
 // localClusterInfoProvider gets the local cluster info from the cloud provider and kubernetes
 type localClusterInfoProvider struct {
-	k8s        kubernetes.Interface
-	dataSource source.OpenCostDataSource
-	provider   cloudProvider.Provider
+	k8s      kubernetes.Interface
+	provider cloudProvider.Provider
 }
 
 // GetClusterInfo returns a string map containing the local cluster info
@@ -73,18 +63,16 @@ func (dlcip *localClusterInfoProvider) GetClusterInfo() map[string]string {
 
 	writeClusterProfile(data)
 	writeReportingFlags(data)
-	writeDataSourceMetaData(dlcip.dataSource, data)
 
 	return data
 }
 
 // NewLocalClusterInfoProvider creates a new clusters.LocalClusterInfoProvider implementation for providing local
 // cluster information
-func NewLocalClusterInfoProvider(k8s kubernetes.Interface, dataSource source.OpenCostDataSource, cloud cloudProvider.Provider) clusters.ClusterInfoProvider {
+func NewLocalClusterInfoProvider(k8s kubernetes.Interface, cloud cloudProvider.Provider) clusters.ClusterInfoProvider {
 	return &localClusterInfoProvider{
-		k8s:        k8s,
-		dataSource: dataSource,
-		provider:   cloud,
+		k8s:      k8s,
+		provider: cloud,
 	}
 }
 

pkg/costmodel/costmodel.go

@@ -136,14 +136,15 @@ func (cm *CostModel) ComputeCostData(start, end time.Time) (map[string]*CostData
 	clusterID := env.GetClusterID()
 	cp := cm.Provider
 	ds := cm.DataSource
+	mq := ds.Metrics()
 
 	grp := source.NewQueryGroup()
 
-	resChRAMUsage := source.WithGroup(grp, ds.QueryRAMUsageAvg(start, end))
-	resChCPUUsage := source.WithGroup(grp, ds.QueryCPUUsageAvg(start, end))
-	resChNetZoneRequests := source.WithGroup(grp, ds.QueryNetZoneGiB(start, end))
-	resChNetRegionRequests := source.WithGroup(grp, ds.QueryNetRegionGiB(start, end))
-	resChNetInternetRequests := source.WithGroup(grp, ds.QueryNetInternetGiB(start, end))
+	resChRAMUsage := source.WithGroup(grp, mq.QueryRAMUsageAvg(start, end))
+	resChCPUUsage := source.WithGroup(grp, mq.QueryCPUUsageAvg(start, end))
+	resChNetZoneRequests := source.WithGroup(grp, mq.QueryNetZoneGiB(start, end))
+	resChNetRegionRequests := source.WithGroup(grp, mq.QueryNetRegionGiB(start, end))
+	resChNetInternetRequests := source.WithGroup(grp, mq.QueryNetInternetGiB(start, end))
 
 	// Pull pod information from k8s API
 	podlist := cm.Cache.GetAllPods()
@@ -607,8 +608,9 @@ func findUnmountedPVCostData(clusterMap clusters.ClusterMap, unmountedPVs map[st
 
 func findDeletedPodInfo(dataSource source.OpenCostDataSource, missingContainers map[string]*CostData, start, end time.Time) error {
 	if len(missingContainers) > 0 {
+		mq := dataSource.Metrics()
 
-		podLabelsResCh := dataSource.QueryPodLabels(start, end)
+		podLabelsResCh := mq.QueryPodLabels(start, end)
 		podLabelsResult, err := podLabelsResCh.Await()
 		if err != nil {
 			log.Errorf("failed to parse historical pod labels: %s", err.Error())
@@ -642,10 +644,11 @@ func findDeletedNodeInfo(dataSource source.OpenCostDataSource, missingNodes map[
 		defer measureTime(time.Now(), profileThreshold, "Finding Deleted Node Info")
 
 		grp := source.NewQueryGroup()
+		mq := dataSource.Metrics()
 
-		cpuCostResCh := source.WithGroup(grp, dataSource.QueryNodeCPUPricePerHr(start, end))
-		ramCostResCh := source.WithGroup(grp, dataSource.QueryNodeRAMPricePerGiBHr(start, end))
-		gpuCostResCh := source.WithGroup(grp, dataSource.QueryNodeGPUPricePerHr(start, end))
+		cpuCostResCh := source.WithGroup(grp, mq.QueryNodeCPUPricePerHr(start, end))
+		ramCostResCh := source.WithGroup(grp, mq.QueryNodeRAMPricePerGiBHr(start, end))
+		gpuCostResCh := source.WithGroup(grp, mq.QueryNodeGPUPricePerHr(start, end))
 
 		cpuCostRes, _ := cpuCostResCh.Await()
 		ramCostRes, _ := ramCostResCh.Await()
@@ -877,11 +880,7 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 			}
 		}
 
-		if _, ok := pmd.PricingTypeCounts[cnode.PricingType]; ok {
-			pmd.PricingTypeCounts[cnode.PricingType]++
-		} else {
-			pmd.PricingTypeCounts[cnode.PricingType] = 1
-		}
+		pmd.PricingTypeCounts[cnode.PricingType]++
 
 		// newCnode builds upon cnode but populates/overrides certain fields.
 		// cnode was populated leveraging cloud provider public pricing APIs.

pkg/costmodel/handlers.go

@@ -30,7 +30,7 @@ func (a *Accesses) ComputeAssetsHandler(w http.ResponseWriter, r *http.Request,
 
 	filterString := qp.Get("filter", "")
 
-	assetSet, err := a.computeAssetsFromCostmodel(window, filterString)
+	assetSet, err := a.ComputeAssetsFromCostmodel(window, filterString)
 	if err != nil {
 		http.Error(w, fmt.Sprintf("Error getting assets: %s", err), http.StatusInternalServerError)
 		return
@@ -55,7 +55,7 @@ func (a *Accesses) ComputeAssetsCarbonHandler(w http.ResponseWriter, r *http.Req
 
 	filterString := qp.Get("filter", "")
 
-	assetSet, err := a.computeAssetsFromCostmodel(window, filterString)
+	assetSet, err := a.ComputeAssetsFromCostmodel(window, filterString)
 	if err != nil {
 		http.Error(w, fmt.Sprintf("Error getting assets: %s", err), http.StatusInternalServerError)
 		return
@@ -70,7 +70,7 @@ func (a *Accesses) ComputeAssetsCarbonHandler(w http.ResponseWriter, r *http.Req
 	w.Write(WrapData(carbonEstimates, nil))
 }
 
-func (a *Accesses) computeAssetsFromCostmodel(window opencost.Window, filterString string) (*opencost.AssetSet, error) {
+func (a *Accesses) ComputeAssetsFromCostmodel(window opencost.Window, filterString string) (*opencost.AssetSet, error) {
 
 	assetSet, err := a.Model.ComputeAssets(*window.Start(), *window.End())
 	if err != nil {

pkg/costmodel/key.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	"github.com/opencost/opencost/core/pkg/opencost"
-	"github.com/opencost/opencost/core/pkg/source"
 	"github.com/opencost/opencost/pkg/env"
 )
 
@@ -48,36 +47,6 @@ func getUnmountedPodKey(cluster string) podKey {
 	return newPodKey(cluster, opencost.UnmountedSuffix, opencost.UnmountedSuffix)
 }
 
-// resultPodKey converts a Prometheus query result to a podKey by looking
-// up values associated with the given label names. For example, passing
-// "cluster_id" for clusterLabel will use the value of the label "cluster_id"
-// as the podKey's Cluster field. If a given field does not exist on the
-// result, an error is returned. (The only exception to that is clusterLabel,
-// which we expect may not exist, but has a default value.)
-func resultPodKey(res *source.QueryResult) (podKey, error) {
-	key := podKey{}
-
-	cluster, err := res.GetCluster()
-	if err != nil {
-		cluster = env.GetClusterID()
-	}
-	key.Cluster = cluster
-
-	namespace, err := res.GetNamespace()
-	if err != nil {
-		return key, err
-	}
-	key.Namespace = namespace
-
-	pod, err := res.GetPod()
-	if err != nil {
-		return key, err
-	}
-	key.Pod = pod
-
-	return key, nil
-}
-
 type namespaceKey struct {
 	Cluster   string
 	Namespace string
@@ -94,30 +63,6 @@ func newNamespaceKey(cluster, namespace string) namespaceKey {
 	}
 }
 
-// resultNamespaceKey converts a Prometheus query result to a namespaceKey by
-// looking up values associated with the given label names. For example,
-// passing "cluster_id" for clusterLabel will use the value of the label
-// "cluster_id" as the namespaceKey's Cluster field. If a given field does not
-// exist on the result, an error is returned. (The only exception to that is
-// clusterLabel, which we expect may not exist, but has a default value.)
-func resultNamespaceKey(res *source.QueryResult) (namespaceKey, error) {
-	key := namespaceKey{}
-
-	cluster, err := res.GetCluster()
-	if err != nil {
-		cluster = env.GetClusterID()
-	}
-	key.Cluster = cluster
-
-	namespace, err := res.GetNamespace()
-	if err != nil {
-		return key, err
-	}
-	key.Namespace = namespace
-
-	return key, nil
-}
-
 func newResultNamespaceKey(cluster string, namespace string) (namespaceKey, error) {
 	if cluster == "" {
 		cluster = env.GetClusterID()
@@ -184,36 +129,6 @@ func newServiceKey(cluster, namespace, service string) serviceKey {
 	}
 }
 
-// resultServiceKey converts a Prometheus query result to a serviceKey by
-// looking up values associated with the given label names. For example,
-// passing "cluster_id" for clusterLabel will use the value of the label
-// "cluster_id" as the serviceKey's Cluster field. If a given field does not
-// exist on the result, an error is returned. (The only exception to that is
-// clusterLabel, which we expect may not exist, but has a default value.)
-func resultServiceKey(res *source.QueryResult, serviceLabel string) (serviceKey, error) {
-	key := serviceKey{}
-
-	cluster, err := res.GetCluster()
-	if err != nil {
-		cluster = env.GetClusterID()
-	}
-	key.Cluster = cluster
-
-	namespace, err := res.GetNamespace()
-	if err != nil {
-		return key, err
-	}
-	key.Namespace = namespace
-
-	service, err := res.GetString(serviceLabel)
-	if err != nil {
-		return key, err
-	}
-	key.Service = service
-
-	return key, nil
-}
-
 func newResultServiceKey(cluster, namespace, service string) (serviceKey, error) {
 	if cluster == "" {
 		cluster = env.GetClusterID()
@@ -246,30 +161,6 @@ func newNodeKey(cluster, node string) nodeKey {
 	}
 }
 
-// resultNodeKey converts a Prometheus query result to a nodeKey by
-// looking up values associated with the given label names. For example,
-// passing "cluster_id" for clusterLabel will use the value of the label
-// "cluster_id" as the nodeKey's Cluster field. If a given field does not
-// exist on the result, an error is returned. (The only exception to that is
-// clusterLabel, which we expect may not exist, but has a default value.)
-func resultNodeKey(res *source.QueryResult) (nodeKey, error) {
-	key := nodeKey{}
-
-	cluster, err := res.GetCluster()
-	if err != nil {
-		cluster = env.GetClusterID()
-	}
-	key.Cluster = cluster
-
-	node, err := res.GetNode()
-	if err != nil {
-		return key, err
-	}
-	key.Node = node
-
-	return key, nil
-}
-
 func newResultNodeKey(cluster string, node string) (nodeKey, error) {
 	if cluster == "" {
 		cluster = env.GetClusterID()
@@ -300,36 +191,6 @@ func newPVCKey(cluster, namespace, persistentVolumeClaim string) pvcKey {
 	}
 }
 
-// resultPVCKey converts a Prometheus query result to a pvcKey by
-// looking up values associated with the given label names. For example,
-// passing "cluster_id" for clusterLabel will use the value of the label
-// "cluster_id" as the pvcKey's Cluster field. If a given field does not
-// exist on the result, an error is returned. (The only exception to that is
-// clusterLabel, which we expect may not exist, but has a default value.)
-func resultPVCKey(res *source.QueryResult, pvcLabel string) (pvcKey, error) {
-	key := pvcKey{}
-
-	cluster, err := res.GetCluster()
-	if err != nil {
-		cluster = env.GetClusterID()
-	}
-	key.Cluster = cluster
-
-	namespace, err := res.GetNamespace()
-	if err != nil {
-		return key, err
-	}
-	key.Namespace = namespace
-
-	pvc, err := res.GetString(pvcLabel)
-	if err != nil {
-		return key, err
-	}
-	key.PersistentVolumeClaim = pvc
-
-	return key, nil
-}
-
 // resultPVCKey converts a Prometheus query result to a pvcKey by
 // looking up values associated with the given label names. For example,
 // passing "cluster_id" for clusterLabel will use the value of the label
@@ -368,30 +229,6 @@ func newPVKey(cluster, persistentVolume string) pvKey {
 	}
 }
 
-// resultPVKey converts a Prometheus query result to a pvKey by
-// looking up values associated with the given label names. For example,
-// passing "cluster_id" for clusterLabel will use the value of the label
-// "cluster_id" as the pvKey's Cluster field. If a given field does not
-// exist on the result, an error is returned. (The only exception to that is
-// clusterLabel, which we expect may not exist, but has a default value.)
-func resultPVKey(res *source.QueryResult, persistentVolumeLabel string) (pvKey, error) {
-	key := pvKey{}
-
-	cluster, err := res.GetCluster()
-	if err != nil {
-		cluster = env.GetClusterID()
-	}
-	key.Cluster = cluster
-
-	persistentVolume, err := res.GetString(persistentVolumeLabel)
-	if err != nil {
-		return key, err
-	}
-	key.PersistentVolume = persistentVolume
-
-	return key, nil
-}
-
 func newResultPVKey(cluster, pv string) (pvKey, error) {
 	if cluster == "" {
 		cluster = env.GetClusterID()

pkg/costmodel/router.go

@@ -26,7 +26,6 @@ import (
 	"github.com/opencost/opencost/pkg/customcost"
 	"github.com/opencost/opencost/pkg/kubeconfig"
 	"github.com/opencost/opencost/pkg/metrics"
-	"github.com/opencost/opencost/pkg/services"
 	"github.com/opencost/opencost/pkg/util/watcher"
 
 	"github.com/julienschmidt/httprouter"
@@ -87,8 +86,6 @@ type Accesses struct {
 	// settings will be published in a pub/sub model
 	settingsSubscribers map[string][]chan string
 	settingsMutex       sync.Mutex
-	// registered http service instances
-	httpServices services.HTTPServices
 }
 
 // GetCacheExpiration looks up and returns custom cache expiration for the given duration.
@@ -113,25 +110,6 @@ func (a *Accesses) GetCacheRefresh(dur time.Duration) time.Duration {
 	return mins
 }
 
-func (a *Accesses) ClusterCostsFromCacheHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
-	w.Header().Set("Content-Type", "application/json")
-
-	duration := 24 * time.Hour
-	offset := time.Minute
-	durationHrs := "24h"
-	fmtOffset := "1m"
-	dataSource := a.DataSource
-
-	key := fmt.Sprintf("%s:%s", durationHrs, fmtOffset)
-	if data, valid := a.ClusterCostsCache.Get(key); valid {
-		clusterCosts := data.(map[string]*ClusterCosts)
-		w.Write(WrapDataWithMessage(clusterCosts, nil, "clusterCosts cache hit"))
-	} else {
-		data, err := a.ComputeClusterCosts(dataSource, a.CloudProvider, duration, offset, true)
-		w.Write(WrapDataWithMessage(data, err, fmt.Sprintf("clusterCosts cache miss: %s", key)))
-	}
-}
-
 type Response struct {
 	Code    int         `json:"code"`
 	Status  string      `json:"status"`
@@ -377,104 +355,6 @@ func (a *Accesses) CostDataModel(w http.ResponseWriter, r *http.Request, ps http
 	}
 }
 
-func (a *Accesses) ClusterCosts(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
-	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("Access-Control-Allow-Origin", "*")
-
-	window := r.URL.Query().Get("window")
-	offset := r.URL.Query().Get("offset")
-
-	if window == "" {
-		w.Write(WrapData(nil, fmt.Errorf("missing window argument")))
-		return
-	}
-	windowDur, err := timeutil.ParseDuration(window)
-	if err != nil {
-		w.Write(WrapData(nil, fmt.Errorf("error parsing window (%s): %s", window, err)))
-		return
-	}
-
-	// offset is not a required parameter
-	var offsetDur time.Duration
-	if offset != "" {
-		offsetDur, err = timeutil.ParseDuration(offset)
-		if err != nil {
-			w.Write(WrapData(nil, fmt.Errorf("error parsing offset (%s): %s", offset, err)))
-			return
-		}
-	}
-	/*
-		useThanos, _ := strconv.ParseBool(r.URL.Query().Get("multi"))
-
-		if useThanos && !thanos.IsEnabled() {
-			w.Write(WrapData(nil, fmt.Errorf("Multi=true while Thanos is not enabled.")))
-			return
-		}
-
-
-		var client prometheus.Client
-		if useThanos {
-			client = a.ThanosClient
-			offsetDur = thanos.OffsetDuration()
-
-		} else {
-			client = a.PrometheusClient
-		}
-	*/
-
-	data, err := a.ComputeClusterCosts(a.DataSource, a.CloudProvider, windowDur, offsetDur, true)
-	w.Write(WrapData(data, err))
-}
-
-func (a *Accesses) ClusterCostsOverTime(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
-	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("Access-Control-Allow-Origin", "*")
-
-	startString := r.URL.Query().Get("start")
-	endString := r.URL.Query().Get("end")
-	window := r.URL.Query().Get("window")
-	offset := r.URL.Query().Get("offset")
-
-	if window == "" {
-		w.Write(WrapData(nil, fmt.Errorf("missing window argument")))
-		return
-	}
-	windowDur, err := timeutil.ParseDuration(window)
-	if err != nil {
-		w.Write(WrapData(nil, fmt.Errorf("error parsing window (%s): %s", window, err)))
-		return
-	}
-
-	// offset is not a required parameter
-	var offsetDur time.Duration
-	if offset != "" {
-		offsetDur, err = timeutil.ParseDuration(offset)
-		if err != nil {
-			w.Write(WrapData(nil, fmt.Errorf("error parsing offset (%s): %s", offset, err)))
-			return
-		}
-	}
-
-	const layout = "2006-01-02T15:04:05.000Z"
-
-	start, err := time.Parse(layout, startString)
-	if err != nil {
-		log.Errorf("Error parsing time %s. Error: %s", startString, err.Error())
-		w.Write(WrapData(nil, fmt.Errorf("error parsing 'start': %s: %w", startString, err)))
-		return
-	}
-
-	end, err := time.Parse(layout, endString)
-	if err != nil {
-		log.Errorf("Error parsing time %s. Error: %s", endString, err.Error())
-		w.Write(WrapData(nil, fmt.Errorf("error parsing 'end': %s: %w", endString, err)))
-		return
-	}
-
-	data, err := ClusterCostsOverTime(a.DataSource, a.CloudProvider, start, end, windowDur, offsetDur)
-	w.Write(WrapData(data, err))
-}
-
 func (a *Accesses) GetAllNodePricing(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")
 	w.Header().Set("Access-Control-Allow-Origin", "*")
@@ -788,6 +668,39 @@ func Initialize(router *httprouter.Router, additionalConfigWatchers ...*watcher.
 		}
 	}
 
+	// Kubernetes API setup
+	kubeClientset, err := kubeconfig.LoadKubeClient("")
+	if err != nil {
+		log.Fatalf("Failed to build Kubernetes client: %s", err.Error())
+	}
+
+	// Create Kubernetes Cluster Cache + Watchers
+	k8sCache := clustercache.NewKubernetesClusterCache(kubeClientset)
+	k8sCache.Run()
+
+	// Create ConfigFileManager for synchronization of shared configuration
+	confManager := config.NewConfigFileManager(&config.ConfigFileManagerOpts{
+		BucketStoreConfig: env.GetKubecostConfigBucket(),
+		LocalConfigPath:   "/",
+	})
+
+	configPrefix := env.GetConfigPathWithDefault("/var/configs/")
+
+	cloudProviderKey := env.GetCloudProviderAPIKey()
+	cloudProvider, err := provider.NewProvider(k8sCache, cloudProviderKey, confManager)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	// ClusterInfo Provider to provide the cluster map with local and remote cluster data
+	var clusterInfoProvider clusters.ClusterInfoProvider
+	if env.IsClusterInfoFileEnabled() {
+		clusterInfoFile := confManager.ConfigFileAt(path.Join(configPrefix, "cluster-info.json"))
+		clusterInfoProvider = NewConfiguredClusterInfoProvider(clusterInfoFile)
+	} else {
+		clusterInfoProvider = NewLocalClusterInfoProvider(kubeClientset, cloudProvider)
+	}
+
 	const maxRetries = 10
 	const retryInterval = 10 * time.Second
 
@@ -797,7 +710,7 @@ func Initialize(router *httprouter.Router, additionalConfigWatchers ...*watcher.
 	dataSource, _ := retry.Retry(
 		ctx,
 		func() (source.OpenCostDataSource, error) {
-			ds, e := prom.NewDefaultPrometheusDataSource()
+			ds, e := prom.NewDefaultPrometheusDataSource(clusterInfoProvider)
 			if e != nil {
 				if source.IsRetryable(e) {
 					return nil, e
@@ -817,30 +730,6 @@ func Initialize(router *httprouter.Router, additionalConfigWatchers ...*watcher.
 		panic(fatalErr)
 	}
 
-	// Kubernetes API setup
-	kubeClientset, err := kubeconfig.LoadKubeClient("")
-	if err != nil {
-		log.Fatalf("Failed to build Kubernetes client: %s", err.Error())
-	}
-
-	// Create ConfigFileManager for synchronization of shared configuration
-	confManager := config.NewConfigFileManager(&config.ConfigFileManagerOpts{
-		BucketStoreConfig: env.GetKubecostConfigBucket(),
-		LocalConfigPath:   "/",
-	})
-
-	configPrefix := env.GetConfigPathWithDefault("/var/configs/")
-
-	// Create Kubernetes Cluster Cache + Watchers
-	k8sCache := clustercache.NewKubernetesClusterCache(kubeClientset)
-	k8sCache.Run()
-
-	cloudProviderKey := env.GetCloudProviderAPIKey()
-	cloudProvider, err := provider.NewProvider(k8sCache, cloudProviderKey, confManager)
-	if err != nil {
-		panic(err.Error())
-	}
-
 	// Append the pricing config watcher
 	kubecostNamespace := env.GetKubecostNamespace()
 
@@ -849,16 +738,7 @@ func Initialize(router *httprouter.Router, additionalConfigWatchers ...*watcher.
 	configWatchers.AddWatcher(metrics.GetMetricsConfigWatcher())
 	configWatchers.Watch()
 
-	// ClusterInfo Provider to provide the cluster map with local and remote cluster data
-	var clusterInfoProvider clusters.ClusterInfoProvider
-	if env.IsClusterInfoFileEnabled() {
-		clusterInfoFile := confManager.ConfigFileAt(path.Join(configPrefix, "cluster-info.json"))
-		clusterInfoProvider = NewConfiguredClusterInfoProvider(clusterInfoFile)
-	} else {
-		clusterInfoProvider = NewLocalClusterInfoProvider(kubeClientset, dataSource, cloudProvider)
-	}
-
-	clusterMap := dataSource.NewClusterMap(clusterInfoProvider)
+	clusterMap := dataSource.ClusterMap()
 
 	// cache responses from model and aggregation for a default of 10 minutes;
 	// clear expired responses every 20 minutes
@@ -882,7 +762,6 @@ func Initialize(router *httprouter.Router, additionalConfigWatchers ...*watcher.
 	metricsEmitter := NewCostModelMetricsEmitter(k8sCache, cloudProvider, clusterInfoProvider, costModel)
 
 	a := &Accesses{
-		httpServices:        services.NewCostModelServices(),
 		DataSource:          dataSource,
 		KubeClientSet:       kubeClientset,
 		ClusterCache:        k8sCache,
@@ -906,30 +785,17 @@ func Initialize(router *httprouter.Router, additionalConfigWatchers ...*watcher.
 		log.Infof("Failed to download pricing data: %s", err)
 	}
 
-	// NOTE: (bolt) this only warms the cache for cluster costs.
-	if env.IsCacheWarmingEnabled() {
-		log.Infof("Init: ClusterCosts cache warming enabled")
-		a.warmAggregateCostModelCache()
-	} else {
-		log.Infof("Init: ClusterCosts cache warming disabled")
-	}
-
 	if !env.IsKubecostMetricsPodEnabled() {
 		a.MetricsEmitter.Start()
 	}
 
-	a.httpServices.RegisterAll(router)
 	a.DataSource.RegisterEndPoints(router)
 
 	router.GET("/costDataModel", a.CostDataModel)
 	router.GET("/allocation/compute", a.ComputeAllocationHandler)
 	router.GET("/allocation/compute/summary", a.ComputeAllocationHandlerSummary)
-
 	router.GET("/allNodePricing", a.GetAllNodePricing)
 	router.POST("/refreshPricing", a.RefreshPricingData)
-	router.GET("/clusterCostsOverTime", a.ClusterCostsOverTime)
-	router.GET("/clusterCosts", a.ClusterCosts)
-	router.GET("/clusterCostsFromCache", a.ClusterCostsFromCacheHandler)
 	router.GET("/managementPlatform", a.ManagementPlatform)
 	router.GET("/clusterInfo", a.ClusterInfo)
 	router.GET("/clusterInfoMap", a.GetClusterInfoMap)

pkg/services/clusters/boltdbstorage.go

@@ -1,94 +0,0 @@
-package clusters
-
-import (
-	bolt "go.etcd.io/bbolt"
-)
-
-// BoltDBClusterStorage is a boltdb implementation of a database used to store cluster definitions
-type BoltDBClusterStorage struct {
-	bucket []byte
-	db     *bolt.DB
-}
-
-// NewBoltDBClusterStorage creates a new boltdb backed ClusterStorage implementation
-func NewBoltDBClusterStorage(bucket string, db *bolt.DB) (ClusterStorage, error) {
-	bucketKey := []byte(bucket)
-
-	err := db.Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists(bucketKey)
-		if err != nil {
-			return err
-		}
-
-		return nil
-	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	return &BoltDBClusterStorage{
-		bucket: bucketKey,
-		db:     db,
-	}, nil
-}
-
-// AddIfNotExists Adds the entry if the key does not exist
-func (cs *BoltDBClusterStorage) AddIfNotExists(key string, cluster []byte) error {
-	return cs.db.Update(func(tx *bolt.Tx) error {
-		k := []byte(key)
-		bucket := tx.Bucket(cs.bucket)
-
-		if bucket.Get(k) != nil {
-			return nil
-		}
-		return bucket.Put(k, cluster)
-	})
-}
-
-// AddOrUpdate Adds the encoded cluster to storage if it doesn't exist. Otherwise, update the existing
-// value with the provided.
-func (cs *BoltDBClusterStorage) AddOrUpdate(key string, cluster []byte) error {
-	return cs.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket(cs.bucket)
-
-		return bucket.Put([]byte(key), cluster)
-	})
-}
-
-// Remove Removes a key from the cluster storage
-func (cs *BoltDBClusterStorage) Remove(key string) error {
-	return cs.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket(cs.bucket)
-
-		return bucket.Delete([]byte(key))
-	})
-}
-
-// Each Iterates through all key/values for the storage and calls the handler func. If a handler returns
-// an error, the iteration stops.
-func (cs *BoltDBClusterStorage) Each(handler func(string, []byte) error) error {
-	return cs.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket(cs.bucket)
-
-		return bucket.ForEach(func(k, v []byte) error {
-			// Allow the bytes to live outside transaction by copy
-			key := make([]byte, len(k))
-			value := make([]byte, len(v))
-
-			copy(key, k)
-			copy(value, v)
-
-			if err := handler(string(key), value); err != nil {
-				return err
-			}
-
-			return nil
-		})
-	})
-}
-
-// Close Closes the backing storage
-func (cs *BoltDBClusterStorage) Close() error {
-	return cs.db.Close()
-}

pkg/services/clusters/clustermanager.go

@@ -1,253 +0,0 @@
-package clusters
-
-import (
-	"encoding/base64"
-	"fmt"
-	"os"
-	"strings"
-
-	"github.com/google/uuid"
-
-	"github.com/opencost/opencost/core/pkg/log"
-	"github.com/opencost/opencost/core/pkg/util/fileutil"
-	"github.com/opencost/opencost/core/pkg/util/json"
-
-	"sigs.k8s.io/yaml"
-)
-
-// The details key used to provide auth information
-const DetailsAuthKey = "auth"
-
-// Authentication Information
-type ClusterConfigEntryAuth struct {
-	// The type of authentication provider to use
-	Type string `yaml:"type"`
-
-	// Data expressed as a secret
-	SecretName string `yaml:"secretName,omitempty"`
-
-	// Any data specifically needed by the auth provider
-	Data string `yaml:"data,omitempty"`
-
-	// User and Password as a possible input
-	User string `yaml:"user,omitempty"`
-	Pass string `yaml:"pass,omitempty"`
-}
-
-// Cluster definition from a configuration yaml
-type ClusterConfigEntry struct {
-	Name    string                  `yaml:"name"`
-	Address string                  `yaml:"address"`
-	Auth    *ClusterConfigEntryAuth `yaml:"auth,omitempty"`
-	Details map[string]interface{}  `yaml:"details,omitempty"`
-}
-
-// ClusterDefinition
-type ClusterDefinition struct {
-	ID      string                 `json:"id,omitempty"`
-	Name    string                 `json:"name"`
-	Address string                 `json:"address"`
-	Details map[string]interface{} `json:"details,omitempty"`
-}
-
-// ClusterStorage interface defines an implementation prototype for a storage responsible
-// for ClusterDefinition instances
-type ClusterStorage interface {
-	// Add only if the key does not exist
-	AddIfNotExists(key string, cluster []byte) error
-
-	// Adds the encoded cluster to storage if it doesn't exist. Otherwise, update the existing
-	// value with the provided.
-	AddOrUpdate(key string, cluster []byte) error
-
-	// Removes a key from the cluster storage
-	Remove(key string) error
-
-	// Iterates through all key/values for the storage and calls the handler func. If a handler returns
-	// an error, the iteration stops.
-	Each(handler func(string, []byte) error) error
-
-	// Closes the backing storage
-	Close() error
-}
-
-// ClusterManager provides an implementation
-type ClusterManager struct {
-	storage ClusterStorage
-	// cache   map[string]*ClusterDefinition
-}
-
-// Creates a new ClusterManager instance using the provided storage
-func NewClusterManager(storage ClusterStorage) *ClusterManager {
-	return &ClusterManager{
-		storage: storage,
-	}
-}
-
-// Creates a new ClusterManager instance using the provided storage and populates a
-// yaml configured list of clusters
-func NewConfiguredClusterManager(storage ClusterStorage, config string) *ClusterManager {
-	clusterManager := NewClusterManager(storage)
-
-	exists, err := fileutil.FileExists(config)
-	if !exists {
-		if err != nil {
-			log.Errorf("Failed to load config file: %s. Error: %s", config, err.Error())
-		}
-		return clusterManager
-	}
-
-	data, err := os.ReadFile(config)
-	if err != nil {
-		return clusterManager
-	}
-
-	var entries []*ClusterConfigEntry
-	err = yaml.Unmarshal(data, &entries)
-	if err != nil {
-		return clusterManager
-	}
-
-	for _, entry := range entries {
-		details := entry.Details
-		if details == nil {
-			details = make(map[string]interface{})
-		}
-
-		if entry.Auth != nil {
-			authData, err := getAuth(entry.Auth)
-			if err != nil {
-				log.Errorf("%s", err)
-			} else {
-				details[DetailsAuthKey] = authData
-			}
-		}
-
-		clusterManager.Add(ClusterDefinition{
-			ID:      entry.Name,
-			Name:    entry.Name,
-			Address: entry.Address,
-			Details: details,
-		})
-	}
-
-	return clusterManager
-}
-
-// Add Adds a cluster definition, but will not update an existing entry.
-func (cm *ClusterManager) Add(cluster ClusterDefinition) (*ClusterDefinition, error) {
-	// First time add
-	if cluster.ID == "" {
-		cluster.ID = uuid.New().String()
-	}
-
-	data, err := json.Marshal(cluster)
-	if err != nil {
-		return nil, err
-	}
-
-	err = cm.storage.AddIfNotExists(cluster.ID, data)
-	if err != nil {
-		return nil, err
-	}
-
-	return &cluster, nil
-}
-
-// AddOrUpdate will add the cluster definition if it doesn't exist, or update the existing definition
-// if it does exist.
-func (cm *ClusterManager) AddOrUpdate(cluster ClusterDefinition) (*ClusterDefinition, error) {
-	// First time add
-	if cluster.ID == "" {
-		cluster.ID = uuid.New().String()
-	}
-
-	data, err := json.Marshal(cluster)
-	if err != nil {
-		return nil, err
-	}
-
-	err = cm.storage.AddOrUpdate(cluster.ID, data)
-	if err != nil {
-		return nil, err
-	}
-
-	return &cluster, nil
-}
-
-// Remove will remove a cluster definition by id.
-func (cm *ClusterManager) Remove(id string) error {
-	return cm.storage.Remove(id)
-}
-
-// GetAll will return all of the cluster definitions
-func (cm *ClusterManager) GetAll() []*ClusterDefinition {
-	clusters := []*ClusterDefinition{}
-
-	err := cm.storage.Each(func(key string, cluster []byte) error {
-		var cd ClusterDefinition
-		err := json.Unmarshal(cluster, &cd)
-		if err != nil {
-			log.Errorf("Failed to unmarshal json cluster definition for key: %s", key)
-			return nil
-		}
-
-		clusters = append(clusters, &cd)
-		return nil
-	})
-
-	if err != nil {
-		log.Infof("[Error] Failed to load list of clusters: %s", err.Error())
-	}
-
-	return clusters
-}
-
-// Close will close the backing database
-func (cm *ClusterManager) Close() error {
-	return cm.storage.Close()
-}
-
-func toBasicAuth(user, pass string) string {
-	return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", user, pass)))
-}
-
-func fileFromSecret(secretName string) string {
-	return fmt.Sprintf("/var/secrets/%s/auth", secretName)
-}
-
-func fromSecret(secretName string) (string, error) {
-	file := fileFromSecret(secretName)
-	exists, err := fileutil.FileExists(file)
-	if !exists || err != nil {
-		return "", fmt.Errorf("Failed to locate secret: %s", file)
-	}
-
-	data, err := os.ReadFile(file)
-	if err != nil {
-		return "", fmt.Errorf("Failed to load secret: %s", file)
-	}
-
-	return base64.StdEncoding.EncodeToString(data), nil
-}
-
-func getAuth(auth *ClusterConfigEntryAuth) (string, error) {
-	// We only support basic auth currently
-	if !strings.EqualFold(auth.Type, "basic") {
-		return "", fmt.Errorf("Authentication Type: '%s' is not supported", auth.Type)
-	}
-
-	if auth.SecretName != "" {
-		return fromSecret(auth.SecretName)
-	}
-
-	if auth.Data != "" {
-		return auth.Data, nil
-	}
-
-	if auth.User != "" && auth.Pass != "" {
-		return toBasicAuth(auth.User, auth.Pass), nil
-	}
-
-	return "", fmt.Errorf("No valid basic auth parameters provided.")
-}

pkg/services/clusters/clustersendpoints.go

@@ -1,113 +0,0 @@
-package clusters
-
-import (
-	"errors"
-	"io"
-	"net/http"
-
-	"github.com/julienschmidt/httprouter"
-
-	"github.com/opencost/opencost/core/pkg/log"
-	"github.com/opencost/opencost/core/pkg/util/json"
-)
-
-// DataEnvelope is a generic wrapper struct for http response data
-type DataEnvelope struct {
-	Code   int         `json:"code"`
-	Status string      `json:"status"`
-	Data   interface{} `json:"data"`
-}
-
-// ClusterManagerHTTPService is an implementation of HTTPService which provides
-// the frontend with the ability to manage stored cluster definitions.
-type ClusterManagerHTTPService struct {
-	manager *ClusterManager
-}
-
-// NewClusterManagerHTTPService creates a new cluster management http service
-func NewClusterManagerHTTPService(manager *ClusterManager) *ClusterManagerHTTPService {
-	return &ClusterManagerHTTPService{
-		manager: manager,
-	}
-}
-
-// Register assigns the endpoints and returns an error on failure.
-func (cme *ClusterManagerHTTPService) Register(router *httprouter.Router) error {
-	router.GET("/clusters", cme.GetAllClusters)
-	router.PUT("/clusters", cme.PutCluster)
-	router.DELETE("/clusters/:id", cme.DeleteCluster)
-
-	return nil
-}
-
-func (cme *ClusterManagerHTTPService) GetAllClusters(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
-	w.Header().Set("Content-Type", "application/json")
-
-	clusters := cme.manager.GetAll()
-	w.Write(wrapData(clusters, nil))
-}
-
-func (cme *ClusterManagerHTTPService) PutCluster(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
-	w.Header().Set("Content-Type", "application/json")
-
-	data, err := io.ReadAll(r.Body)
-	if err != nil {
-		w.Write(wrapData(nil, err))
-		return
-	}
-
-	var clusterDef ClusterDefinition
-	err = json.Unmarshal(data, &clusterDef)
-	if err != nil {
-		w.Write(wrapData(nil, err))
-		return
-	}
-
-	cd, err := cme.manager.AddOrUpdate(clusterDef)
-	if err != nil {
-		w.Write(wrapData(nil, err))
-		return
-	}
-
-	w.Write(wrapData(cd, nil))
-}
-
-func (cme *ClusterManagerHTTPService) DeleteCluster(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
-	w.Header().Set("Content-Type", "application/json")
-
-	clusterID := ps.ByName("id")
-	if clusterID == "" {
-		w.Write(wrapData(nil, errors.New("Failed to locate cluster with empty id.")))
-		return
-	}
-
-	err := cme.manager.Remove(clusterID)
-	if err != nil {
-		w.Write(wrapData(nil, err))
-		return
-	}
-
-	w.Write(wrapData("success", nil))
-}
-
-func wrapData(data interface{}, err error) []byte {
-	var resp []byte
-
-	if err != nil {
-		log.Infof("Error returned to client: %s", err.Error())
-		resp, _ = json.Marshal(&DataEnvelope{
-			Code:   http.StatusInternalServerError,
-			Status: "error",
-			Data:   err.Error(),
-		})
-	} else {
-		resp, _ = json.Marshal(&DataEnvelope{
-			Code:   http.StatusOK,
-			Status: "success",
-			Data:   data,
-		})
-
-	}
-
-	return resp
-}

pkg/services/clusters/mapdbstorage.go

@@ -1,53 +0,0 @@
-package clusters
-
-// MapDBClusterStorage is a map implementation of a database used to store cluster definitions
-type MapDBClusterStorage struct {
-	store map[string][]byte
-}
-
-// NewMapDBClusterStorage creates a new map backed ClusterStorage implementation
-func NewMapDBClusterStorage() ClusterStorage {
-	return &MapDBClusterStorage{
-		store: make(map[string][]byte),
-	}
-}
-
-// AddIfNotExists Adds the entry if the key does not exist
-func (cs *MapDBClusterStorage) AddIfNotExists(key string, cluster []byte) error {
-	if _, ok := cs.store[key]; !ok {
-		cs.store[key] = cluster
-	}
-	return nil
-}
-
-// AddOrUpdate Adds the encoded cluster to storage if it doesn't exist. Otherwise, update the existing
-// value with the provided.
-func (cs *MapDBClusterStorage) AddOrUpdate(key string, cluster []byte) error {
-	cs.store[key] = cluster
-	return nil
-}
-
-// Remove Removes a key from the cluster storage
-func (cs *MapDBClusterStorage) Remove(key string) error {
-	delete(cs.store, key)
-	return nil
-}
-
-// Each Iterates through all key/values for the storage and calls the handler func. If a handler returns
-// an error, the iteration stops.
-func (cs *MapDBClusterStorage) Each(handler func(string, []byte) error) error {
-	for k, v := range cs.store {
-		value := make([]byte, len(v))
-		copy(value, v)
-
-		if err := handler(k, value); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// Close Closes the backing storage
-func (cs *MapDBClusterStorage) Close() error {
-	return nil
-}

pkg/services/clusterservice.go

@@ -1,22 +0,0 @@
-package services
-
-import (
-	"path"
-
-	"github.com/opencost/opencost/pkg/env"
-	"github.com/opencost/opencost/pkg/services/clusters"
-)
-
-// NewClusterManagerService creates a new HTTPService implementation driving cluster definition management
-// for the frontend
-func NewClusterManagerService() HTTPService {
-	return clusters.NewClusterManagerHTTPService(newClusterManager())
-}
-
-// newClusterManager creates a new cluster manager instance for use in the service
-func newClusterManager() *clusters.ClusterManager {
-	clustersConfigFile := path.Join(env.GetCostAnalyzerVolumeMountPath(), "clusters/default-clusters.yaml")
-
-	// Return a memory-backed cluster manager populated by configmap
-	return clusters.NewConfiguredClusterManager(clusters.NewMapDBClusterStorage(), clustersConfigFile)
-}

pkg/services/services.go

@@ -1,65 +0,0 @@
-package services
-
-import (
-	"sync"
-
-	"github.com/julienschmidt/httprouter"
-	"github.com/opencost/opencost/core/pkg/log"
-)
-
-// HTTPService defines an implementation prototype for an object capable of registering
-// endpoints on an http router which provide an service relevant to the cost-model.
-type HTTPService interface {
-	// Register assigns the endpoints and returns an error on failure.
-	Register(*httprouter.Router) error
-}
-
-// HTTPServices defines an implementation prototype for an object capable of managing and registering
-// predefined HTTPService routes
-type HTTPServices interface {
-	// Add a HTTPService implementation for registration
-	Add(service HTTPService)
-
-	// RegisterAll registers all the services added with the provided router
-	RegisterAll(*httprouter.Router) error
-}
-
-type defaultHTTPServices struct {
-	sync.Mutex
-	services []HTTPService
-}
-
-// Add a HTTPService implementation for
-func (dhs *defaultHTTPServices) Add(service HTTPService) {
-	if service == nil {
-		log.Warnf("Attempting to Add nil HTTPService")
-		return
-	}
-
-	dhs.Lock()
-	defer dhs.Unlock()
-
-	dhs.services = append(dhs.services, service)
-}
-
-// RegisterAll registers all the services added with the provided router
-func (dhs *defaultHTTPServices) RegisterAll(router *httprouter.Router) error {
-	dhs.Lock()
-	defer dhs.Unlock()
-
-	for _, svc := range dhs.services {
-		svc.Register(router)
-	}
-
-	return nil
-}
-
-// NewCostModelServices creates an HTTPServices implementation containing any predefined
-// http services used with the cost-model
-func NewCostModelServices() HTTPServices {
-	return &defaultHTTPServices{
-		services: []HTTPService{
-			NewClusterManagerService(),
-		},
-	}
-}