Merge branch 'develop' into bolt/opencost-mods

configs/pricing_schema_gpu_labels.csv

@@ -0,0 +1,2 @@
+EndTimestamp,InstanceID,Region,AssetClass,InstanceIDField,InstanceType,MarketPriceHourly,Version
+2019-04-17 23:34:22 UTC,labelfoo,,gpulabel,foo,,0.75,

core/pkg/filter/legacy/allcut.go

@@ -1,10 +0,0 @@
-package legacy
-
-// AllCut is a filter that matches nothing. This is useful
-// for applications like authorization, where a user/group/role may be disallowed
-// from viewing data entirely.
-type AllCut[T any] struct{}
-
-func (ac AllCut[T]) String() string { return "(AllCut)" }
-
-func (ac AllCut[T]) Matches(T) bool { return false }

core/pkg/filter/legacy/allpass.go

@@ -1,9 +0,0 @@
-package legacy
-
-// AllPass is a filter that matches everything and is the same as no filter. It is implemented here as a guard
-// against universal operations occurring in the absence of filters.
-type AllPass[T any] struct{}
-
-func (n AllPass[T]) String() string { return "(AllPass)" }
-
-func (n AllPass[T]) Matches(T) bool { return true }

core/pkg/filter/legacy/and.go

@@ -1,36 +0,0 @@
-package legacy
-
-import (
-	"fmt"
-)
-
-// And is a set of filters that should be evaluated as a logical
-// AND.
-type And[T any] struct {
-	Filters []Filter[T]
-}
-
-func (a And[T]) String() string {
-	s := "(and"
-	for _, f := range a.Filters {
-		s += fmt.Sprintf(" %s", f)
-	}
-
-	s += ")"
-	return s
-}
-
-func (a And[T]) Matches(that T) bool {
-	filters := a.Filters
-	if len(filters) == 0 {
-		return true
-	}
-
-	for _, filter := range filters {
-		if !filter.Matches(that) {
-			return false
-		}
-	}
-
-	return true
-}

core/pkg/filter/legacy/cloudcost/cloudcost.go

@@ -1,155 +0,0 @@
-package cloudcost
-
-import (
-	"reflect"
-	"strings"
-
-	filter "github.com/opencost/opencost/core/pkg/filter/legacy"
-	"github.com/opencost/opencost/core/pkg/log"
-	"github.com/opencost/opencost/core/pkg/opencost"
-	"github.com/opencost/opencost/core/pkg/util/mapper"
-)
-
-type CloudCostFilter struct {
-	AccountIDs       []string `json:"accountIDs,omitempty"`
-	Categories       []string `json:"categories,omitempty"`
-	InvoiceEntityIDs []string `json:"invoiceEntityIDs,omitempty"`
-	Labels           []string `json:"labels,omitempty"`
-	Providers        []string `json:"providers,omitempty"`
-	ProviderIDs      []string `json:"providerIDs,omitempty"`
-	Services         []string `json:"services,omitempty"`
-}
-
-func (g *CloudCostFilter) Equals(that CloudCostFilter) bool {
-	return reflect.DeepEqual(g.AccountIDs, that.AccountIDs) &&
-		reflect.DeepEqual(g.Categories, that.Categories) &&
-		reflect.DeepEqual(g.InvoiceEntityIDs, that.InvoiceEntityIDs) &&
-		reflect.DeepEqual(g.Labels, that.Labels) &&
-		reflect.DeepEqual(g.Providers, that.Providers) &&
-		reflect.DeepEqual(g.ProviderIDs, that.ProviderIDs) &&
-		reflect.DeepEqual(g.Services, that.Services)
-}
-func parseWildcardEnd(rawFilterValue string) (string, bool) {
-	return strings.TrimSuffix(rawFilterValue, "*"), strings.HasSuffix(rawFilterValue, "*")
-}
-
-func CloudCostFilterFromParams(pmr mapper.PrimitiveMapReader) filter.Filter[*opencost.CloudCost] {
-	ccFilter := convertFilterQueryParams(pmr)
-	return ParseCloudCostFilter(ccFilter)
-}
-
-func convertFilterQueryParams(pmr mapper.PrimitiveMapReader) CloudCostFilter {
-	return CloudCostFilter{
-		AccountIDs:       pmr.GetList("filterAccountIDs", ","),
-		Categories:       pmr.GetList("filterCategories", ","),
-		InvoiceEntityIDs: pmr.GetList("filterInvoiceEntityIDs", ","),
-		Labels:           pmr.GetList("filterLabels", ","),
-		Providers:        pmr.GetList("filterProviders", ","),
-		ProviderIDs:      pmr.GetList("filterProviderIDs", ","),
-		Services:         pmr.GetList("filterServices", ","),
-	}
-}
-func ParseCloudCostFilter(filters CloudCostFilter) filter.Filter[*opencost.CloudCost] {
-	result := filter.And[*opencost.CloudCost]{
-		Filters: []filter.Filter[*opencost.CloudCost]{},
-	}
-
-	if len(filters.InvoiceEntityIDs) > 0 {
-		result.Filters = append(result.Filters, filterV1SingleValueFromList(filters.InvoiceEntityIDs, opencost.CloudCostInvoiceEntityIDProp))
-	}
-
-	if len(filters.AccountIDs) > 0 {
-		result.Filters = append(result.Filters, filterV1SingleValueFromList(filters.AccountIDs, opencost.CloudCostAccountIDProp))
-	}
-
-	if len(filters.Providers) > 0 {
-		result.Filters = append(result.Filters, filterV1SingleValueFromList(filters.Providers, opencost.CloudCostProviderProp))
-	}
-
-	if len(filters.ProviderIDs) > 0 {
-		result.Filters = append(result.Filters, filterV1SingleValueFromList(filters.ProviderIDs, opencost.CloudCostProviderIDProp))
-	}
-
-	if len(filters.Services) > 0 {
-		result.Filters = append(result.Filters, filterV1SingleValueFromList(filters.Services, opencost.CloudCostServiceProp))
-	}
-
-	if len(filters.Categories) > 0 {
-		result.Filters = append(result.Filters, filterV1SingleValueFromList(filters.Categories, opencost.CloudCostCategoryProp))
-	}
-
-	if len(filters.Labels) > 0 {
-		result.Filters = append(result.Filters, filterV1DoubleValueFromList(filters.Labels, opencost.CloudCostLabelProp))
-	}
-
-	if len(result.Filters) == 0 {
-		return nil
-	}
-
-	return result
-}
-
-func filterV1SingleValueFromList(rawFilterValues []string, field string) filter.Filter[*opencost.CloudCost] {
-	result := filter.Or[*opencost.CloudCost]{
-		Filters: []filter.Filter[*opencost.CloudCost]{},
-	}
-
-	for _, filterValue := range rawFilterValues {
-		filterValue = strings.TrimSpace(filterValue)
-		filterValue, wildcard := parseWildcardEnd(filterValue)
-
-		subFilter := filter.StringProperty[*opencost.CloudCost]{
-			Field: field,
-			Op:    filter.StringEquals,
-			Value: filterValue,
-		}
-
-		if wildcard {
-			subFilter.Op = filter.StringStartsWith
-		}
-
-		result.Filters = append(result.Filters, subFilter)
-	}
-
-	return result
-}
-
-// filterV1DoubleValueFromList creates an OR of key:value equality filters for
-// colon-split filter values.
-//
-// The v1 query language (e.g. "filterLabels=app:foo,l2:bar") uses OR within
-// a field (e.g. label[app] = foo OR label[l2] = bar)
-func filterV1DoubleValueFromList(rawFilterValuesUnsplit []string, filterField string) filter.Filter[*opencost.CloudCost] {
-	result := filter.Or[*opencost.CloudCost]{
-		Filters: []filter.Filter[*opencost.CloudCost]{},
-	}
-
-	for _, unsplit := range rawFilterValuesUnsplit {
-		if unsplit != "" {
-			split := strings.Split(unsplit, ":")
-			if len(split) != 2 {
-				log.Warnf("illegal key/value filter (ignoring): %s", unsplit)
-				continue
-			}
-			labelName := strings.TrimSpace(split[0])
-			val := strings.TrimSpace(split[1])
-			val, wildcard := parseWildcardEnd(val)
-
-			subFilter := filter.StringMapProperty[*opencost.CloudCost]{
-				Field: filterField,
-				// All v1 filters are equality comparisons
-				Op:    filter.StringMapEquals,
-				Key:   labelName,
-				Value: val,
-			}
-
-			if wildcard {
-				subFilter.Op = filter.StringMapStartsWith
-			}
-
-			result.Filters = append(result.Filters, subFilter)
-		}
-	}
-
-	return result
-}

core/pkg/filter/legacy/cloudcost/cloudcost_test.go

@@ -1,134 +0,0 @@
-package cloudcost
-
-import (
-	"testing"
-)
-
-type CloudCostFilterEqualsTestcase struct {
-	name     string
-	this     CloudCostFilter
-	that     CloudCostFilter
-	expected bool
-}
-
-func TestCloudCostFilter_Equals(t *testing.T) {
-	testCases := []CloudCostFilterEqualsTestcase{
-		{
-			name: "both filters nil",
-			this: CloudCostFilter{
-				AccountIDs:       nil,
-				Categories:       nil,
-				InvoiceEntityIDs: nil,
-				Labels:           nil,
-				Providers:        nil,
-				ProviderIDs:      nil,
-				Services:         nil,
-			},
-			that: CloudCostFilter{
-				AccountIDs:       nil,
-				Categories:       nil,
-				InvoiceEntityIDs: nil,
-				Labels:           nil,
-				Providers:        nil,
-				ProviderIDs:      nil,
-				Services:         nil,
-			},
-			expected: true,
-		},
-		{
-			name: "both filters not nil and matching",
-			this: CloudCostFilter{
-				AccountIDs:       []string{"account1", "account2", "account3"},
-				Categories:       []string{"category1", "category2"},
-				InvoiceEntityIDs: []string{"invoice1", "invoice2"},
-				Labels:           []string{"label1", "label2"},
-				Providers:        []string{"provider1", "provider2"},
-				ProviderIDs:      []string{"provider1", "provider2"},
-				Services:         []string{"s1"},
-			},
-			that: CloudCostFilter{
-				AccountIDs:       []string{"account1", "account2", "account3"},
-				Categories:       []string{"category1", "category2"},
-				InvoiceEntityIDs: []string{"invoice1", "invoice2"},
-				Labels:           []string{"label1", "label2"},
-				Providers:        []string{"provider1", "provider2"},
-				ProviderIDs:      []string{"provider1", "provider2"},
-				Services:         []string{"s1"},
-			},
-			expected: true,
-		},
-		{
-			name: "both filters diff count",
-			this: CloudCostFilter{
-				AccountIDs:       []string{"account1", "account2", "account3"},
-				Categories:       []string{"category1", "category2"},
-				InvoiceEntityIDs: []string{"invoice1", "invoice2"},
-				Labels:           []string{"label1", "label2"},
-				Providers:        []string{"provider1", "provider2"},
-				ProviderIDs:      []string{"provider1", "provider2"},
-				Services:         []string{"s1"},
-			},
-			that: CloudCostFilter{
-				AccountIDs:       []string{"account1", "account2"},
-				Categories:       []string{"category1", "category2"},
-				InvoiceEntityIDs: []string{"invoice1", "invoice2"},
-				Labels:           []string{"label1", "label2"},
-				Providers:        []string{"provider1", "provider2"},
-				ProviderIDs:      []string{"provider1", "provider2"},
-				Services:         []string{"s1"},
-			},
-			expected: false,
-		},
-		{
-			name: "slight mismatch",
-			this: CloudCostFilter{
-				AccountIDs:       []string{"account1", "account2", "account3"},
-				Categories:       []string{"category1", "category2"},
-				InvoiceEntityIDs: []string{"invoice1", "invoice2"},
-				Labels:           []string{"label1", "label2"},
-				Providers:        []string{"provider1", "provider2"},
-				ProviderIDs:      []string{"provider1", "provider2"},
-				Services:         []string{"s1"},
-			},
-			that: CloudCostFilter{
-				AccountIDs:       []string{"account10", "account2", "account3"},
-				Categories:       []string{"category1", "category2"},
-				InvoiceEntityIDs: []string{"invoice1", "invoice2"},
-				Labels:           []string{"label1", "label2"},
-				Providers:        []string{"provider1", "provider2"},
-				ProviderIDs:      []string{"provider1", "provider2"},
-				Services:         []string{"s1"},
-			},
-			expected: false,
-		},
-		{
-			name: "one nil, one not",
-			this: CloudCostFilter{
-				AccountIDs:       []string{"account1", "account2", "account3"},
-				Categories:       []string{"category1", "category2"},
-				InvoiceEntityIDs: []string{"invoice1", "invoice2"},
-				Labels:           []string{"label1", "label2"},
-				Providers:        []string{"provider1", "provider2"},
-				ProviderIDs:      []string{"provider1", "provider2"},
-				Services:         []string{"s1"},
-			},
-			that: CloudCostFilter{
-				AccountIDs:       nil,
-				Categories:       nil,
-				InvoiceEntityIDs: nil,
-				Labels:           nil,
-				Providers:        nil,
-				ProviderIDs:      nil,
-				Services:         nil,
-			},
-			expected: false,
-		},
-	}
-
-	for _, tc := range testCases {
-		got := tc.this.Equals(tc.that)
-		if got != tc.expected {
-			t.Fatalf("expected %t, got: %t for test case: %s", tc.expected, got, tc.name)
-		}
-	}
-}

core/pkg/filter/legacy/filter.go

@@ -1,20 +0,0 @@
-package legacy
-
-// Filter represents anything that can be used to filter given generic type T.
-//
-// Implement this interface with caution. While it is generic, it
-// is intended to be introspectable so query handlers can perform various
-// optimizations. These optimizations include:
-// - Routing a query to the most optimal cache
-// - Querying backing data stores efficiently (e.g. translation to SQL)
-//
-// Custom implementations of this interface outside of this package should not
-// expect to receive these benefits. Passing a custom implementation to a
-// handler may in errors.
-type Filter[T any] interface {
-	String() string
-
-	// Matches is the canonical in-Go function for determining if T
-	// matches a filter.
-	Matches(T) bool
-}

core/pkg/filter/legacy/filter_test.go

@@ -1,1073 +0,0 @@
-package legacy_test
-
-import (
-	"testing"
-
-	filter "github.com/opencost/opencost/core/pkg/filter/legacy"
-	"github.com/opencost/opencost/core/pkg/opencost"
-)
-
-func Test_String_Matches(t *testing.T) {
-	cases := []struct {
-		name   string
-		a      *opencost.Allocation
-		filter filter.Filter[*opencost.Allocation]
-
-		expected bool
-	}{
-		{
-			name: "ClusterID Equals -> true",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Cluster: "cluster-one",
-				},
-			},
-			filter: filter.StringProperty[*opencost.Allocation]{
-				Field: opencost.AllocationClusterProp,
-				Op:    filter.StringEquals,
-				Value: "cluster-one",
-			},
-
-			expected: true,
-		},
-		{
-			name: "ClusterID StartsWith -> true",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Cluster: "cluster-one",
-				},
-			},
-			filter: filter.StringProperty[*opencost.Allocation]{
-				Field: opencost.AllocationClusterProp,
-				Op:    filter.StringStartsWith,
-				Value: "cluster",
-			},
-
-			expected: true,
-		},
-		{
-			name: "ClusterID StartsWith -> false",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Cluster: "k8s-one",
-				},
-			},
-			filter: filter.StringProperty[*opencost.Allocation]{
-				Field: opencost.AllocationClusterProp,
-				Op:    filter.StringStartsWith,
-				Value: "cluster",
-			},
-
-			expected: false,
-		},
-		{
-			name: "ClusterID empty StartsWith '' -> true",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Cluster: "",
-				},
-			},
-			filter: filter.StringProperty[*opencost.Allocation]{
-				Field: opencost.AllocationClusterProp,
-				Op:    filter.StringStartsWith,
-				Value: "",
-			},
-
-			expected: true,
-		},
-		{
-			name: "ClusterID nonempty StartsWith '' -> true",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Cluster: "abc",
-				},
-			},
-			filter: filter.StringProperty[*opencost.Allocation]{
-				Field: opencost.AllocationClusterProp,
-				Op:    filter.StringStartsWith,
-				Value: "",
-			},
-
-			expected: true,
-		},
-		{
-			name: "Node Equals -> true",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Node: "node123",
-				},
-			},
-			filter: filter.StringProperty[*opencost.Allocation]{
-				Field: opencost.AllocationNodeProp,
-				Op:    filter.StringEquals,
-				Value: "node123",
-			},
-
-			expected: true,
-		},
-		{
-			name: "Namespace Equals Unallocated -> true",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Namespace: "",
-				},
-			},
-			filter: filter.StringProperty[*opencost.Allocation]{
-				Field: opencost.AllocationNamespaceProp,
-				Op:    filter.StringEquals,
-				Value: opencost.UnallocatedSuffix,
-			},
-
-			expected: true,
-		},
-		{
-			name: "ControllerKind Equals -> true",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					ControllerKind: "deployment", // We generally store controller kinds as all lowercase
-				},
-			},
-			filter: filter.StringProperty[*opencost.Allocation]{
-				Field: opencost.AllocationControllerKindProp,
-				Op:    filter.StringEquals,
-				Value: "deployment",
-			},
-
-			expected: true,
-		},
-		{
-			name: "ControllerName Equals -> true",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Controller: "kc-cost-analyzer",
-				},
-			},
-			filter: filter.StringProperty[*opencost.Allocation]{
-				Field: opencost.AllocationControllerProp,
-				Op:    filter.StringEquals,
-				Value: "kc-cost-analyzer",
-			},
-
-			expected: true,
-		},
-		{
-			name: "Pod (with UID) Equals -> true",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Pod: "pod-123 UID-ABC",
-				},
-			},
-			filter: filter.StringProperty[*opencost.Allocation]{
-				Field: opencost.AllocationPodProp,
-				Op:    filter.StringEquals,
-				Value: "pod-123 UID-ABC",
-			},
-
-			expected: true,
-		},
-		{
-			name: "Container Equals -> true",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Container: "cost-model",
-				},
-			},
-			filter: filter.StringProperty[*opencost.Allocation]{
-				Field: opencost.AllocationContainerProp,
-				Op:    filter.StringEquals,
-				Value: "cost-model",
-			},
-
-			expected: true,
-		},
-		{
-			name: `namespace unallocated -> true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Namespace: "",
-				},
-			},
-			filter: filter.StringProperty[*opencost.Allocation]{
-				Field: opencost.AllocationNamespaceProp,
-				Op:    filter.StringEquals,
-				Value: opencost.UnallocatedSuffix,
-			},
-
-			expected: true,
-		},
-	}
-
-	for _, c := range cases {
-		result := c.filter.Matches(c.a)
-
-		if result != c.expected {
-			t.Errorf("%s: expected %t, got %t", c.name, c.expected, result)
-		}
-	}
-}
-
-func Test_StringSlice_Matches(t *testing.T) {
-	cases := []struct {
-		name   string
-		a      *opencost.Allocation
-		filter filter.Filter[*opencost.Allocation]
-
-		expected bool
-	}{
-		{
-			name: `services contains -> true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Services: []string{"serv1", "serv2"},
-				},
-			},
-			filter: filter.StringSliceProperty[*opencost.Allocation]{
-				Field: opencost.AllocationServiceProp,
-				Op:    filter.StringSliceContains,
-				Value: "serv2",
-			},
-
-			expected: true,
-		},
-		{
-			name: `services contains -> false`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Services: []string{"serv1", "serv2"},
-				},
-			},
-			filter: filter.StringSliceProperty[*opencost.Allocation]{
-				Field: opencost.AllocationServiceProp,
-				Op:    filter.StringSliceContains,
-				Value: "serv3",
-			},
-
-			expected: false,
-		},
-		{
-			name: `services contains unallocated -> false`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Services: []string{"serv1", "serv2"},
-				},
-			},
-			filter: filter.StringSliceProperty[*opencost.Allocation]{
-				Field: opencost.AllocationServiceProp,
-				Op:    filter.StringSliceContains,
-				Value: opencost.UnallocatedSuffix,
-			},
-
-			expected: false,
-		},
-		{
-			name: `services contains unallocated -> true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Services: []string{},
-				},
-			},
-			filter: filter.StringSliceProperty[*opencost.Allocation]{
-				Field: opencost.AllocationServiceProp,
-				Op:    filter.StringSliceContains,
-				Value: opencost.UnallocatedSuffix,
-			},
-
-			expected: true,
-		},
-	}
-
-	for _, c := range cases {
-		result := c.filter.Matches(c.a)
-
-		if result != c.expected {
-			t.Errorf("%s: expected %t, got %t", c.name, c.expected, result)
-		}
-	}
-}
-
-func Test_StringMap_Matches(t *testing.T) {
-	cases := []struct {
-		name   string
-		a      *opencost.Allocation
-		filter filter.Filter[*opencost.Allocation]
-
-		expected bool
-	}{
-		{
-			name: `label[app]="foo" -> true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Labels: map[string]string{
-						"app": "foo",
-					},
-				},
-			},
-			filter: filter.StringMapProperty[*opencost.Allocation]{
-				Field: opencost.AllocationLabelProp,
-				Op:    filter.StringMapEquals,
-				Key:   "app",
-				Value: "foo",
-			},
-
-			expected: true,
-		},
-		{
-			name: `label[app]="foo" -> different value -> false`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Labels: map[string]string{
-						"app": "bar",
-					},
-				},
-			},
-			filter: filter.StringMapProperty[*opencost.Allocation]{
-				Field: opencost.AllocationLabelProp,
-				Op:    filter.StringMapEquals,
-				Key:   "app",
-				Value: "foo",
-			},
-
-			expected: false,
-		},
-		{
-			name: `label[app]="foo" -> label missing -> false`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Labels: map[string]string{
-						"someotherlabel": "someothervalue",
-					},
-				},
-			},
-			filter: filter.StringMapProperty[*opencost.Allocation]{
-				Field: opencost.AllocationLabelProp,
-				Op:    filter.StringMapEquals,
-				Key:   "app",
-				Value: "foo",
-			},
-
-			expected: false,
-		},
-		{
-			name: `label[app]=Unallocated -> label missing -> true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Labels: map[string]string{
-						"someotherlabel": "someothervalue",
-					},
-				},
-			},
-			filter: filter.StringMapProperty[*opencost.Allocation]{
-				Field: opencost.AllocationLabelProp,
-				Op:    filter.StringMapEquals,
-				Key:   "app",
-				Value: opencost.UnallocatedSuffix,
-			},
-
-			expected: true,
-		},
-		{
-			name: `label[app]=Unallocated -> label present -> false`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Labels: map[string]string{
-						"app": "test",
-					},
-				},
-			},
-			filter: filter.StringMapProperty[*opencost.Allocation]{
-				Field: opencost.AllocationLabelProp,
-				Op:    filter.StringMapEquals,
-				Key:   "app",
-				Value: opencost.UnallocatedSuffix,
-			},
-
-			expected: false,
-		},
-		{
-			name: `annotation[prom_modified_name]="testing123" -> true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Annotations: map[string]string{
-						"prom_modified_name": "testing123",
-					},
-				},
-			},
-			filter: filter.StringMapProperty[*opencost.Allocation]{
-				Field: opencost.AllocationAnnotationProp,
-				Op:    filter.StringMapEquals,
-				Key:   "prom_modified_name",
-				Value: "testing123",
-			},
-
-			expected: true,
-		},
-		{
-			name: `annotation[app]="foo" -> different value -> false`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Annotations: map[string]string{
-						"app": "bar",
-					},
-				},
-			},
-			filter: filter.StringMapProperty[*opencost.Allocation]{
-				Field: opencost.AllocationAnnotationProp,
-				Op:    filter.StringMapEquals,
-				Key:   "app",
-				Value: "foo",
-			},
-
-			expected: false,
-		},
-		{
-			name: `annotation[app]="foo" -> annotation missing -> false`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Annotations: map[string]string{
-						"someotherannotation": "someothervalue",
-					},
-				},
-			},
-			filter: filter.StringMapProperty[*opencost.Allocation]{
-				Field: opencost.AllocationAnnotationProp,
-				Op:    filter.StringMapEquals,
-				Key:   "app",
-				Value: "foo",
-			},
-
-			expected: false,
-		},
-	}
-
-	for _, c := range cases {
-		result := c.filter.Matches(c.a)
-
-		if result != c.expected {
-			t.Errorf("%s: expected %t, got %t", c.name, c.expected, result)
-		}
-	}
-}
-
-func Test_Not_Matches(t *testing.T) {
-	cases := []struct {
-		name   string
-		a      *opencost.Allocation
-		filter filter.Filter[*opencost.Allocation]
-
-		expected bool
-	}{
-		{
-			name: "Namespace NotEquals -> false",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Namespace: "kube-system",
-				},
-			},
-			filter: filter.Not[*opencost.Allocation]{
-				Filter: filter.StringProperty[*opencost.Allocation]{
-					Field: opencost.AllocationNamespaceProp,
-					Op:    filter.StringEquals,
-					Value: "kube-system",
-				},
-			},
-
-			expected: false,
-		},
-		{
-			name: "Namespace NotEquals Unallocated -> true",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Namespace: "kube-system",
-				},
-			},
-			filter: filter.Not[*opencost.Allocation]{
-				Filter: filter.StringProperty[*opencost.Allocation]{
-					Field: opencost.AllocationNamespaceProp,
-					Op:    filter.StringEquals,
-					Value: opencost.UnallocatedSuffix,
-				},
-			},
-			expected: true,
-		},
-		{
-			name: "Namespace NotEquals Unallocated -> false",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Namespace: "",
-				},
-			},
-			filter: filter.Not[*opencost.Allocation]{
-				Filter: filter.StringProperty[*opencost.Allocation]{
-					Field: opencost.AllocationNamespaceProp,
-					Op:    filter.StringEquals,
-					Value: opencost.UnallocatedSuffix,
-				},
-			},
-
-			expected: false,
-		},
-
-		{
-			name: `label[app]!=Unallocated -> label missing -> false`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Labels: map[string]string{
-						"someotherlabel": "someothervalue",
-					},
-				},
-			},
-			filter: filter.Not[*opencost.Allocation]{
-				Filter: filter.StringMapProperty[*opencost.Allocation]{
-					Field: opencost.AllocationLabelProp,
-					Op:    filter.StringMapEquals,
-					Key:   "app",
-					Value: opencost.UnallocatedSuffix,
-				},
-			},
-			expected: false,
-		},
-		{
-			name: `label[app]!=Unallocated -> label present -> true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Labels: map[string]string{
-						"app": "test",
-					},
-				},
-			},
-			filter: filter.Not[*opencost.Allocation]{
-				Filter: filter.StringMapProperty[*opencost.Allocation]{
-					Field: opencost.AllocationLabelProp,
-					Op:    filter.StringMapEquals,
-					Key:   "app",
-					Value: opencost.UnallocatedSuffix,
-				},
-			},
-			expected: true,
-		},
-		{
-			name: `label[app]!="foo" -> label missing -> true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Labels: map[string]string{
-						"someotherlabel": "someothervalue",
-					},
-				},
-			},
-			filter: filter.Not[*opencost.Allocation]{
-				Filter: filter.StringMapProperty[*opencost.Allocation]{
-					Field: opencost.AllocationLabelProp,
-					Op:    filter.StringMapEquals,
-					Key:   "app",
-					Value: "foo",
-				},
-			},
-
-			expected: true,
-		},
-		{
-			name: `annotation[prom_modified_name]="testing123" -> true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Annotations: map[string]string{
-						"prom_modified_name": "testing123",
-					},
-				},
-			},
-			filter: filter.StringMapProperty[*opencost.Allocation]{
-				Field: opencost.AllocationAnnotationProp,
-				Op:    filter.StringMapEquals,
-				Key:   "prom_modified_name",
-				Value: "testing123",
-			},
-
-			expected: true,
-		},
-		{
-			name: `annotation[app]="foo" -> different value -> false`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Annotations: map[string]string{
-						"app": "bar",
-					},
-				},
-			},
-			filter: filter.StringMapProperty[*opencost.Allocation]{
-				Field: opencost.AllocationAnnotationProp,
-				Op:    filter.StringMapEquals,
-				Key:   "app",
-				Value: "foo",
-			},
-
-			expected: false,
-		},
-		{
-			name: `annotation[app]="foo" -> annotation missing -> false`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Annotations: map[string]string{
-						"someotherannotation": "someothervalue",
-					},
-				},
-			},
-			filter: filter.StringMapProperty[*opencost.Allocation]{
-				Field: opencost.AllocationAnnotationProp,
-				Op:    filter.StringMapEquals,
-				Key:   "app",
-				Value: "foo",
-			},
-
-			expected: false,
-		},
-		{
-			name: `annotation[app]!="foo" -> annotation missing -> true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Annotations: map[string]string{
-						"someotherannotation": "someothervalue",
-					},
-				},
-			},
-			filter: filter.Not[*opencost.Allocation]{
-				Filter: filter.StringMapProperty[*opencost.Allocation]{
-					Field: opencost.AllocationAnnotationProp,
-					Op:    filter.StringMapEquals,
-					Key:   "app",
-					Value: "foo",
-				},
-			},
-
-			expected: true,
-		},
-		{
-			name: `namespace unallocated -> true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Namespace: "",
-				},
-			},
-			filter: filter.StringProperty[*opencost.Allocation]{
-				Field: opencost.AllocationNamespaceProp,
-				Op:    filter.StringEquals,
-				Value: opencost.UnallocatedSuffix,
-			},
-
-			expected: true,
-		},
-		{
-			name: `services contains -> true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Services: []string{"serv1", "serv2"},
-				},
-			},
-			filter: filter.StringSliceProperty[*opencost.Allocation]{
-				Field: opencost.AllocationServiceProp,
-				Op:    filter.StringSliceContains,
-				Value: "serv2",
-			},
-
-			expected: true,
-		},
-		{
-			name: `services contains -> false`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Services: []string{"serv1", "serv2"},
-				},
-			},
-			filter: filter.StringSliceProperty[*opencost.Allocation]{
-				Field: opencost.AllocationServiceProp,
-				Op:    filter.StringSliceContains,
-				Value: "serv3",
-			},
-
-			expected: false,
-		},
-		{
-			name: `services notcontains -> true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Services: []string{"serv1", "serv2"},
-				},
-			},
-			filter: filter.Not[*opencost.Allocation]{
-				Filter: filter.StringSliceProperty[*opencost.Allocation]{
-					Field: opencost.AllocationServiceProp,
-					Op:    filter.StringSliceContains,
-					Value: "serv3",
-				},
-			},
-			expected: true,
-		},
-		{
-			name: `services notcontains -> false`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Services: []string{"serv1", "serv2"},
-				},
-			},
-			filter: filter.Not[*opencost.Allocation]{
-				Filter: filter.StringSliceProperty[*opencost.Allocation]{
-					Field: opencost.AllocationServiceProp,
-					Op:    filter.StringSliceContains,
-					Value: "serv2",
-				},
-			},
-
-			expected: false,
-		},
-		{
-			name: `services notcontains unallocated -> true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Services: []string{"serv1", "serv2"},
-				},
-			},
-			filter: filter.Not[*opencost.Allocation]{
-				Filter: filter.StringSliceProperty[*opencost.Allocation]{
-					Field: opencost.AllocationServiceProp,
-					Op:    filter.StringSliceContains,
-					Value: opencost.UnallocatedSuffix,
-				},
-			},
-
-			expected: true,
-		},
-		{
-			name: `services notcontains unallocated -> false`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Services: []string{},
-				},
-			},
-			filter: filter.Not[*opencost.Allocation]{
-				Filter: filter.StringSliceProperty[*opencost.Allocation]{
-					Field: opencost.AllocationServiceProp,
-					Op:    filter.StringSliceContains,
-					Value: opencost.UnallocatedSuffix,
-				},
-			},
-
-			expected: false,
-		},
-		{
-			name: `services containsprefix -> true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Services: []string{"serv1", "serv2"},
-				},
-			},
-			filter: filter.StringSliceProperty[*opencost.Allocation]{
-				Field: opencost.AllocationServiceProp,
-				Op:    filter.StringSliceContainsPrefix,
-				Value: "serv",
-			},
-
-			expected: true,
-		},
-		{
-			name: `services containsprefix -> false`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Services: []string{"foo", "bar"},
-				},
-			},
-			filter: filter.StringSliceProperty[*opencost.Allocation]{
-				Field: opencost.AllocationServiceProp,
-				Op:    filter.StringSliceContainsPrefix,
-				Value: "serv",
-			},
-
-			expected: false,
-		},
-		{
-			name: `services contains unallocated -> false`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Services: []string{"serv1", "serv2"},
-				},
-			},
-			filter: filter.StringSliceProperty[*opencost.Allocation]{
-				Field: opencost.AllocationServiceProp,
-				Op:    filter.StringSliceContains,
-				Value: opencost.UnallocatedSuffix,
-			},
-
-			expected: false,
-		},
-		{
-			name: `services contains unallocated -> true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Services: []string{},
-				},
-			},
-			filter: filter.StringSliceProperty[*opencost.Allocation]{
-				Field: opencost.AllocationServiceProp,
-				Op:    filter.StringSliceContains,
-				Value: opencost.UnallocatedSuffix,
-			},
-
-			expected: true,
-		},
-	}
-
-	for _, c := range cases {
-		result := c.filter.Matches(c.a)
-
-		if result != c.expected {
-			t.Errorf("%s: expected %t, got %t", c.name, c.expected, result)
-		}
-	}
-}
-
-func Test_None_Matches(t *testing.T) {
-	cases := []struct {
-		name string
-		a    *opencost.Allocation
-	}{
-		{
-			name: "nil",
-			a:    nil,
-		},
-		{
-			name: "nil properties",
-			a: &opencost.Allocation{
-				Properties: nil,
-			},
-		},
-		{
-			name: "empty properties",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{},
-			},
-		},
-		{
-			name: "ClusterID",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Cluster: "cluster-one",
-				},
-			},
-		},
-		{
-			name: "Node",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Node: "node123",
-				},
-			},
-		},
-		{
-			name: "Namespace",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Namespace: "kube-system",
-				},
-			},
-		},
-		{
-			name: "ControllerKind",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					ControllerKind: "deployment", // We generally store controller kinds as all lowercase
-				},
-			},
-		},
-		{
-			name: "ControllerName",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Controller: "kc-cost-analyzer",
-				},
-			},
-		},
-		{
-			name: "Pod",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Pod: "pod-123 UID-ABC",
-				},
-			},
-		},
-		{
-			name: "Container",
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Container: "cost-model",
-				},
-			},
-		},
-		{
-			name: `label`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Labels: map[string]string{
-						"app": "foo",
-					},
-				},
-			},
-		},
-		{
-			name: `annotation`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Annotations: map[string]string{
-						"prom_modified_name": "testing123",
-					},
-				},
-			},
-		},
-		{
-			name: `services`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Services: []string{"serv1", "serv2"},
-				},
-			},
-		},
-	}
-
-	for _, c := range cases {
-		result := filter.AllCut[*opencost.Allocation]{}.Matches(c.a)
-
-		if result {
-			t.Errorf("%s: should have been rejected", c.name)
-		}
-	}
-}
-
-func Test_And_Matches(t *testing.T) {
-	cases := []struct {
-		name   string
-		a      *opencost.Allocation
-		filter filter.Filter[*opencost.Allocation]
-
-		expected bool
-	}{
-		{
-			name: `label[app]="foo" and namespace="kubecost" -> both true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Namespace: "kubecost",
-					Labels: map[string]string{
-						"app": "foo",
-					},
-				},
-			},
-			filter: filter.And[*opencost.Allocation]{[]filter.Filter[*opencost.Allocation]{
-				filter.StringMapProperty[*opencost.Allocation]{
-					Field: opencost.AllocationLabelProp,
-					Op:    filter.StringMapEquals,
-					Key:   "app",
-					Value: "foo",
-				},
-				filter.StringProperty[*opencost.Allocation]{
-					Field: opencost.AllocationNamespaceProp,
-					Op:    filter.StringEquals,
-					Value: "kubecost",
-				},
-			}},
-			expected: true,
-		},
-		{
-			name: `label[app]="foo" and namespace="kubecost" -> first true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Namespace: "kube-system",
-					Labels: map[string]string{
-						"app": "foo",
-					},
-				},
-			},
-			filter: filter.And[*opencost.Allocation]{[]filter.Filter[*opencost.Allocation]{
-				filter.StringMapProperty[*opencost.Allocation]{
-					Field: opencost.AllocationLabelProp,
-					Op:    filter.StringMapEquals,
-					Key:   "app",
-					Value: "foo",
-				},
-				filter.StringProperty[*opencost.Allocation]{
-					Field: opencost.AllocationNamespaceProp,
-					Op:    filter.StringEquals,
-					Value: "kubecost",
-				},
-			}},
-			expected: false,
-		},
-		{
-			name: `label[app]="foo" and namespace="kubecost" -> second true`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Namespace: "kubecost",
-					Labels: map[string]string{
-						"app": "bar",
-					},
-				},
-			},
-			filter: filter.And[*opencost.Allocation]{[]filter.Filter[*opencost.Allocation]{
-				filter.StringMapProperty[*opencost.Allocation]{
-					Field: opencost.AllocationLabelProp,
-					Op:    filter.StringMapEquals,
-					Key:   "app",
-					Value: "foo",
-				},
-				filter.StringProperty[*opencost.Allocation]{
-					Field: opencost.AllocationNamespaceProp,
-					Op:    filter.StringEquals,
-					Value: "kubecost",
-				},
-			}},
-			expected: false,
-		},
-		{
-			name: `label[app]="foo" and namespace="kubecost" -> both false`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Namespace: "kube-system",
-					Labels: map[string]string{
-						"app": "bar",
-					},
-				},
-			},
-			filter: filter.And[*opencost.Allocation]{[]filter.Filter[*opencost.Allocation]{
-				filter.StringMapProperty[*opencost.Allocation]{
-					Field: opencost.AllocationLabelProp,
-					Op:    filter.StringMapEquals,
-					Key:   "app",
-					Value: "foo",
-				},
-				filter.StringProperty[*opencost.Allocation]{
-					Field: opencost.AllocationNamespaceProp,
-					Op:    filter.StringEquals,
-					Value: "kubecost",
-				},
-			}},
-			expected: false,
-		},
-		{
-			name: `(and none) matches nothing`,
-			a: &opencost.Allocation{
-				Properties: &opencost.AllocationProperties{
-					Namespace: "kube-system",
-					Labels: map[string]string{
-						"app": "bar",
-					},
-				},
-			},
-			filter: filter.And[*opencost.Allocation]{[]filter.Filter[*opencost.Allocation]{
-				filter.AllCut[*opencost.Allocation]{},
-			}},
-			expected: false,
-		},
-	}
-
-	for _, c := range cases {
-		result := c.filter.Matches(c.a)
-
-		if result != c.expected {
-			t.Errorf("%s: expected %t, got %t", c.name, c.expected, result)
-		}
-	}
-}

core/pkg/filter/legacy/not.go

@@ -1,17 +0,0 @@
-package legacy
-
-import "fmt"
-
-// Not negates any filter contained within it
-type Not[T any] struct {
-	Filter Filter[T]
-}
-
-func (n Not[T]) String() string {
-	return fmt.Sprintf("(not %s)", n.Filter.String())
-}
-
-// Matches inverts the result of the child filter
-func (n Not[T]) Matches(that T) bool {
-	return !n.Filter.Matches(that)
-}

core/pkg/filter/legacy/or.go

@@ -1,36 +0,0 @@
-package legacy
-
-import (
-	"fmt"
-)
-
-// Or is a set of filters that should be evaluated as a logical
-// OR.
-type Or[T any] struct {
-	Filters []Filter[T]
-}
-
-func (o Or[T]) String() string {
-	s := "(or"
-	for _, f := range o.Filters {
-		s += fmt.Sprintf(" %s", f)
-	}
-
-	s += ")"
-	return s
-}
-
-func (o Or[T]) Matches(that T) bool {
-	filters := o.Filters
-	if len(filters) == 0 {
-		return true
-	}
-
-	for _, filter := range filters {
-		if filter.Matches(that) {
-			return true
-		}
-	}
-
-	return false
-}

core/pkg/filter/legacy/stringmapproperty.go

@@ -1,83 +0,0 @@
-package legacy
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/opencost/opencost/core/pkg/log"
-)
-
-const unallocatedSuffix = "__unallocated__"
-
-type StringMapPropertied interface {
-	StringMapProperty(string) (map[string]string, error)
-}
-
-// StringMapOperation is an enum that represents operations that can be performed
-// when filtering (equality, inequality, etc.)
-type StringMapOperation string
-
-const (
-	// StringMapHasKey passes if the map has the provided key
-	StringMapHasKey StringMapOperation = "stringmapcontains"
-
-	StringMapStartsWith = "stringmapstartswith"
-
-	// StringMapEquals when the given key and value match
-	StringMapEquals = "stringmapequals"
-)
-
-// StringMapProperty is the lowest-level type of filter. It represents
-// a filter operation (equality, inequality, etc.) on a property that contains a string map
-type StringMapProperty[T StringMapPropertied] struct {
-	Field string
-	Op    StringMapOperation
-	Key   string
-	Value string
-}
-
-func (smp StringMapProperty[T]) String() string {
-	return fmt.Sprintf(`(%s %s[%s] "%s")`, smp.Op, smp.Field, smp.Key, smp.Value)
-}
-
-func (smp StringMapProperty[T]) Matches(that T) bool {
-
-	thatMap, err := that.StringMapProperty(smp.Field)
-	if err != nil {
-		log.Errorf("Filter: StringMapProperty: could not retrieve field %s: %s", smp.Field, err.Error())
-		return false
-	}
-
-	valueToCompare, keyIsPresent := thatMap[smp.Key]
-
-	switch smp.Op {
-	case StringMapHasKey:
-		return keyIsPresent
-	case StringMapEquals:
-		// namespace:"__unallocated__" should match a.Properties.Namespace = ""
-		// label[app]:"__unallocated__" should match _, ok := Labels[app]; !ok
-		if !keyIsPresent || valueToCompare == "" {
-			return smp.Value == unallocatedSuffix
-		}
-
-		if valueToCompare == smp.Value {
-			return true
-		}
-
-	case StringMapStartsWith:
-		if !keyIsPresent {
-			return false
-		}
-
-		// We don't need special __unallocated__ logic here because a query
-		// asking for "__unallocated__" won't have a wildcard and unallocated
-		// properties are the empty string.
-
-		return strings.HasPrefix(valueToCompare, smp.Value)
-	default:
-		log.Errorf("Filter: StringMapProperty: Unhandled filter op. This is a filter implementation error and requires immediate patching. Op: %s", smp.Op)
-		return false
-	}
-
-	return false
-}

core/pkg/filter/legacy/stringproperty.go

@@ -1,83 +0,0 @@
-package legacy
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/opencost/opencost/core/pkg/log"
-)
-
-// StringPropertied is used to validate the name of a property field and return its value
-type StringPropertied interface {
-	// StringProperty acts as a validator and getter for a structs string properties
-	StringProperty(string) (string, error)
-}
-
-// StringOperation is an enum that represents operations that can be performed
-// when filtering (equality, inequality, etc.)
-type StringOperation string
-
-// If you add a FilterOp, MAKE SURE TO UPDATE ALL FILTER IMPLEMENTATIONS! Go
-// does not enforce exhaustive pattern matching on "enum" types.
-const (
-	// StringEquals is the equality operator
-	// "kube-system" FilterEquals "kube-system" = true
-	// "kube-syste" FilterEquals "kube-system" = false
-	StringEquals StringOperation = "stringequals"
-
-	// StringStartsWith matches strings with the given prefix.
-	// "kube-system" StartsWith "kube" = true
-	//
-	// When comparing with a field represented by an array/slice, this is like
-	// applying FilterContains to every element of the slice.
-	StringStartsWith = "stringstartswith"
-)
-
-// StringProperty is the lowest-level type of filter. It represents
-// a filter operation (equality, inequality, etc.) on a field with a string value (namespace,
-// node, pod, etc.).
-type StringProperty[T StringPropertied] struct {
-	Field string
-	Op    StringOperation
-
-	// Value is for _all_ filters. A filter of 'namespace:"kubecost"' has
-	// Value="kubecost"
-	Value string
-}
-
-func (sp StringProperty[T]) String() string {
-	return fmt.Sprintf(`(%s %s "%s")`, sp.Op, sp.Field, sp.Value)
-}
-
-func (sp StringProperty[T]) Matches(that T) bool {
-
-	thatString, err := that.StringProperty(sp.Field)
-	if err != nil {
-		log.Errorf("Filter: StringProperty: could not retrieve field %s: %s", sp.Field, err.Error())
-		return false
-	}
-
-	switch sp.Op {
-	case StringEquals:
-		// namespace:"__unallocated__" should match a.Properties.Namespace = ""
-		if thatString == "" {
-			return sp.Value == unallocatedSuffix
-		}
-
-		if thatString == sp.Value {
-			return true
-		}
-	case StringStartsWith:
-
-		// We don't need special __unallocated__ logic here because a query
-		// asking for "__unallocated__" won't have a wildcard and unallocated
-		// properties are the empty string.
-
-		return strings.HasPrefix(thatString, sp.Value)
-	default:
-		log.Errorf("Filter: StringProperty: Unhandled filter op. This is a filter implementation error and requires immediate patching. Op: %s", sp.Op)
-		return false
-	}
-
-	return false
-}

core/pkg/filter/legacy/stringsliceproperty.go

@@ -1,80 +0,0 @@
-package legacy
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/opencost/opencost/core/pkg/log"
-)
-
-type StringSlicePropertied interface {
-	StringSliceProperty(string) ([]string, error)
-}
-
-// StringSliceOperation is an enum that represents operations that can be performed
-// when filtering (equality, inequality, etc.)
-type StringSliceOperation string
-
-const (
-	// StringSliceContains is an array/slice membership operator
-	// ["a", "b", "c"] FilterContains "a" = true
-	StringSliceContains StringSliceOperation = "stringslicecontains"
-
-	// StringSliceContainsPrefix is like FilterContains, but using StartsWith instead
-	// of Equals.
-	// ["kube-system", "abc123"] ContainsPrefix ["kube"] = true
-	StringSliceContainsPrefix = "stringslicecontainsprefix"
-)
-
-// StringSliceProperty is the lowest-level type of filter. It represents
-// a filter operation (equality, inequality, etc.) on a property that contains a string slice
-type StringSliceProperty[T StringSlicePropertied] struct {
-	Field string
-	Op    StringSliceOperation
-
-	Value string
-}
-
-func (ssp StringSliceProperty[T]) String() string {
-	return fmt.Sprintf(`(%s %s "%s")`, ssp.Op, ssp.Field, ssp.Value)
-}
-
-func (ssp StringSliceProperty[T]) Matches(that T) bool {
-
-	thatSlice, err := that.StringSliceProperty(ssp.Field)
-	if err != nil {
-		log.Errorf("Filter: StringSliceProperty: could not retrieve field %s: %s", ssp.Field, err.Error())
-		return false
-	}
-
-	switch ssp.Op {
-
-	case StringSliceContains:
-		if len(thatSlice) == 0 {
-			return ssp.Value == unallocatedSuffix
-		}
-
-		for _, s := range thatSlice {
-			if s == ssp.Value {
-				return true
-			}
-		}
-	case StringSliceContainsPrefix:
-		// We don't need special __unallocated__ logic here because a query
-		// asking for "__unallocated__" won't have a wildcard and unallocated
-		// properties are the empty string.
-
-		for _, s := range thatSlice {
-			if strings.HasPrefix(s, ssp.Value) {
-				return true
-			}
-		}
-
-		return false
-	default:
-		log.Errorf("Filter: StringSliceProperty: Unhandled filter op. This is a filter implementation error and requires immediate patching. Op: %s", ssp.Op)
-		return false
-	}
-
-	return false
-}

core/pkg/filter/legacy/window.go

@@ -1,40 +0,0 @@
-package legacy
-
-//
-//import (
-//	"fmt"
-//	"github.com/opencost/opencost/pkg/kubecost"
-//	"github.com/opencost/opencost/pkg/log"
-//)
-//
-//type Windowed interface {
-//	GetWindow() opencost.Window
-//}
-//
-//// WindowOperation are operations that can be performed on types that have windows
-//type WindowOperation string
-//
-//const (
-//	WindowContains WindowOperation = "windowcontains"
-//)
-//
-//// WindowCondition is a filter can be used on any type that has a window and implements GetWindow()
-//type WindowCondition[T Windowed] struct {
-//	Window opencost.Window
-//	Op     WindowOperation
-//}
-//
-//func (wc WindowCondition[T]) String() string {
-//	return fmt.Sprintf(`(%s "%s")`, wc.Op, wc.Window.String())
-//}
-//
-//func (wc WindowCondition[T]) Matches(that T) bool {
-//	thatWindow := that.GetWindow()
-//	switch wc.Op {
-//	case WindowContains:
-//		return wc.Window.ContainsWindow(thatWindow)
-//	default:
-//		log.Errorf("Filter: Window: Unhandled filter operation. This is a filter implementation error and requires immediate patching. Op: %s", wc.Op)
-//		return false
-//	}
-//}

core/pkg/filter/legacy/window_test.go

@@ -1,112 +0,0 @@
-package legacy_test
-
-// import (
-// 	"github.com/opencost/opencost/pkg/kubecost"
-// 	"testing"
-// 	"time"
-// )
-
-// type windowedImpl struct {
-// 	opencost.Window
-// }
-
-// func (w *windowedImpl) GetWindow() opencost.Window {
-// 	return w.Window
-// }
-
-// func newWindowedImpl(start, end *time.Time) *windowedImpl {
-// 	return &windowedImpl{opencost.NewWindow(start, end)}
-// }
-
-// func Test_WindowContains_Matches(t *testing.T) {
-// 	noon := time.Date(2022, 9, 29, 12, 0, 0, 0, time.UTC)
-// 	one := noon.Add(time.Hour)
-// 	two := one.Add(time.Hour)
-// 	three := two.Add(time.Hour)
-// 	cases := map[string]struct {
-// 		windowed *windowedImpl
-// 		filter   Filter[*windowedImpl]
-// 		expected bool
-// 	}{
-// 		"fully contains": {
-// 			windowed: newWindowedImpl(&one, &two),
-// 			filter: WindowCondition[*windowedImpl]{
-// 				Window: opencost.NewWindow(&noon, &three),
-// 				Op:     WindowContains,
-// 			},
-
-// 			expected: true,
-// 		},
-// 		"window matches": {
-// 			windowed: newWindowedImpl(&one, &two),
-// 			filter: WindowCondition[*windowedImpl]{
-// 				Window: opencost.NewWindow(&one, &two),
-// 				Op:     WindowContains,
-// 			},
-
-// 			expected: true,
-// 		},
-// 		"contains start": {
-// 			windowed: newWindowedImpl(&one, &three),
-// 			filter: WindowCondition[*windowedImpl]{
-// 				Window: opencost.NewWindow(&noon, &two),
-// 				Op:     WindowContains,
-// 			},
-
-// 			expected: false,
-// 		},
-// 		"contains end": {
-// 			windowed: newWindowedImpl(&noon, &two),
-// 			filter: WindowCondition[*windowedImpl]{
-// 				Window: opencost.NewWindow(&one, &three),
-// 				Op:     WindowContains,
-// 			},
-
-// 			expected: false,
-// 		},
-// 		"window start = filter end": {
-// 			windowed: newWindowedImpl(&one, &two),
-// 			filter: WindowCondition[*windowedImpl]{
-// 				Window: opencost.NewWindow(&noon, &one),
-// 				Op:     WindowContains,
-// 			},
-
-// 			expected: false,
-// 		},
-// 		"window end = filter start": {
-// 			windowed: newWindowedImpl(&noon, &one),
-// 			filter: WindowCondition[*windowedImpl]{
-// 				Window: opencost.NewWindow(&one, &two),
-// 				Op:     WindowContains,
-// 			},
-
-// 			expected: false,
-// 		},
-// 		"window before": {
-// 			windowed: newWindowedImpl(&noon, &one),
-// 			filter: WindowCondition[*windowedImpl]{
-// 				Window: opencost.NewWindow(&two, &three),
-// 				Op:     WindowContains,
-// 			},
-
-// 			expected: false,
-// 		},
-// 		"window after": {
-// 			windowed: newWindowedImpl(&two, &three),
-// 			filter: WindowCondition[*windowedImpl]{
-// 				Window: opencost.NewWindow(&noon, &one),
-// 				Op:     WindowContains,
-// 			},
-
-// 			expected: false,
-// 		},
-// 	}
-
-// 	for name, c := range cases {
-// 		result := c.filter.Matches(c.windowed)
-
-// 		if result != c.expected {
-// 			t.Errorf("%s: expected %t, got %t", name, c.expected, result)
-// 		}
-// 	}
-// }

core/pkg/opencost/allocationfilter_test.go

@@ -3,7 +3,7 @@ package opencost
 import (
 	"testing"
 
-	filter21 "github.com/opencost/opencost/core/pkg/filter"
+	"github.com/opencost/opencost/core/pkg/filter"
 	afilter "github.com/opencost/opencost/core/pkg/filter/allocation"
 	"github.com/opencost/opencost/core/pkg/filter/ast"
 	"github.com/opencost/opencost/core/pkg/filter/ops"
@@ -21,7 +21,7 @@ func Test_AllocationFilterCondition_Matches(t *testing.T) {
 	cases := []struct {
 		name   string
 		a      *Allocation
-		filter filter21.Filter
+		filter filter.Filter
 
 		expected bool
 	}{
@@ -639,7 +639,7 @@ func Test_AllocationFilterAnd_Matches(t *testing.T) {
 	cases := []struct {
 		name   string
 		a      *Allocation
-		filter filter21.Filter
+		filter filter.Filter
 
 		expected bool
 	}{
@@ -740,7 +740,7 @@ func Test_AllocationFilterOr_Matches(t *testing.T) {
 	cases := []struct {
 		name   string
 		a      *Allocation
-		filter filter21.Filter
+		filter filter.Filter
 
 		expected bool
 	}{

core/pkg/opencost/asset.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 	"time"
 
-	filter21 "github.com/opencost/opencost/core/pkg/filter"
+	"github.com/opencost/opencost/core/pkg/filter"
 	"github.com/opencost/opencost/core/pkg/filter/ast"
 	"github.com/opencost/opencost/core/pkg/filter/matcher"
 	"github.com/opencost/opencost/core/pkg/log"
@@ -3772,7 +3772,7 @@ func (asr *AssetSetRange) newAccumulation() (*AssetSet, error) {
 
 type AssetAggregationOptions struct {
 	SharedHourlyCosts map[string]float64
-	Filter            filter21.Filter
+	Filter            filter.Filter
 	LabelConfig       *LabelConfig
 }
 

core/pkg/opencost/asset_json.go

@@ -6,6 +6,7 @@ import (
 	"reflect"
 	"time"
 
+	"github.com/opencost/opencost/core/pkg/log"
 	"github.com/opencost/opencost/core/pkg/util/json"
 )
 
@@ -74,9 +75,9 @@ func (a *Any) InterfaceToAny(itf interface{}) error {
 	a.Labels = labels
 	a.Start = start
 	a.End = end
-	a.Window = Window{
-		start: &start,
-		end:   &end,
+
+	if _, found := fmap["window"]; found {
+		a.Window = toWindow(fmap["window"].(map[string]interface{}))
 	}
 
 	if adjustment, err := getTypedVal(fmap["adjustment"]); err == nil {
@@ -154,9 +155,8 @@ func (ca *Cloud) InterfaceToCloud(itf interface{}) error {
 	ca.Labels = labels
 	ca.Start = start
 	ca.End = end
-	ca.Window = Window{
-		start: &start,
-		end:   &end,
+	if _, found := fmap["window"]; found {
+		ca.Window = toWindow(fmap["window"].(map[string]interface{}))
 	}
 
 	if adjustment, err := getTypedVal(fmap["adjustment"]); err == nil {
@@ -221,21 +221,10 @@ func (cm *ClusterManagement) InterfaceToClusterManagement(itf interface{}) error
 		labels[k] = v.(string)
 	}
 
-	// parse start and end strings to time.Time
-	start, err := time.Parse(time.RFC3339, fmap["start"].(string))
-	if err != nil {
-		return err
-	}
-	end, err := time.Parse(time.RFC3339, fmap["end"].(string))
-	if err != nil {
-		return err
-	}
-
 	cm.Properties = &properties
 	cm.Labels = labels
-	cm.Window = Window{
-		start: &start,
-		end:   &end,
+	if _, found := fmap["window"]; found {
+		cm.Window = toWindow(fmap["window"].(map[string]interface{}))
 	}
 
 	if Cost, err := getTypedVal(fmap["totalCost"]); err == nil {
@@ -331,9 +320,8 @@ func (d *Disk) InterfaceToDisk(itf interface{}) error {
 	d.Labels = labels
 	d.Start = start
 	d.End = end
-	d.Window = Window{
-		start: &start,
-		end:   &end,
+	if _, found := fmap["window"]; found {
+		d.Window = toWindow(fmap["window"].(map[string]interface{}))
 	}
 	d.Breakdown = &breakdown
 
@@ -448,9 +436,8 @@ func (n *Network) InterfaceToNetwork(itf interface{}) error {
 	n.Labels = labels
 	n.Start = start
 	n.End = end
-	n.Window = Window{
-		start: &start,
-		end:   &end,
+	if _, found := fmap["window"]; found {
+		n.Window = toWindow(fmap["window"].(map[string]interface{}))
 	}
 
 	if adjustment, err := getTypedVal(fmap["adjustment"]); err == nil {
@@ -559,9 +546,8 @@ func (n *Node) InterfaceToNode(itf interface{}) error {
 	n.Labels = labels
 	n.Start = start
 	n.End = end
-	n.Window = Window{
-		start: &start,
-		end:   &end,
+	if _, found := fmap["window"]; found {
+		n.Window = toWindow(fmap["window"].(map[string]interface{}))
 	}
 	n.CPUBreakdown = &cpuBreakdown
 	n.RAMBreakdown = &ramBreakdown
@@ -670,9 +656,8 @@ func (lb *LoadBalancer) InterfaceToLoadBalancer(itf interface{}) error {
 	lb.Labels = labels
 	lb.Start = start
 	lb.End = end
-	lb.Window = Window{
-		start: &start,
-		end:   &end,
+	if _, found := fmap["window"]; found {
+		lb.Window = toWindow(fmap["window"].(map[string]interface{}))
 	}
 
 	if adjustment, err := getTypedVal(fmap["adjustment"]); err == nil {
@@ -741,21 +726,10 @@ func (sa *SharedAsset) InterfaceToSharedAsset(itf interface{}) error {
 		labels[k] = v.(string)
 	}
 
-	// parse start and end strings to time.Time
-	start, err := time.Parse(time.RFC3339, fmap["start"].(string))
-	if err != nil {
-		return err
-	}
-	end, err := time.Parse(time.RFC3339, fmap["end"].(string))
-	if err != nil {
-		return err
-	}
-
 	sa.Properties = &properties
 	sa.Labels = labels
-	sa.Window = Window{
-		start: &start,
-		end:   &end,
+	if _, found := fmap["window"]; found {
+		sa.Window = toWindow(fmap["window"].(map[string]interface{}))
 	}
 
 	if Cost, err := getTypedVal(fmap["totalCost"]); err == nil {
@@ -973,6 +947,35 @@ func toAssetProp(fproperties map[string]interface{}) AssetProperties {
 
 }
 
+func toWindow(fproperties map[string]interface{}) Window {
+
+	var start, end time.Time
+	var err error
+
+	if startStr, v := fproperties["start"].(string); v {
+		start, err = time.Parse(time.RFC3339, startStr)
+
+		if err != nil {
+			log.Errorf("error parsing window start from string %s, setting as 0 time", startStr)
+			start = time.Time{}
+		}
+
+	}
+
+	if endStr, v := fproperties["end"].(string); v {
+		end, err = time.Parse(time.RFC3339, endStr)
+
+		if err != nil {
+			log.Errorf("error parsing window end from string %s, setting as 0 time", endStr)
+			end = time.Time{}
+		}
+
+	}
+
+	return NewClosedWindow(start, end)
+
+}
+
 // Creates an Breakdown directly from map[string]interface{}
 func toBreakdown(fproperties map[string]interface{}) Breakdown {
 	var breakdown Breakdown

core/pkg/opencost/bingen.go

@@ -22,8 +22,6 @@ package opencost
 
 // Default Version Set (uses -version flag passed) includes shared resources
 // @bingen:generate:Window
-// @bingen:generate:Coverage
-// @bingen:generate:CoverageSet
 
 // Asset Version Set: Includes Asset pipeline specific resources
 // @bingen:set[name=Assets,version=21]
@@ -72,4 +70,4 @@ package opencost
 // @bingen:generate:CloudCostLabels
 // @bingen:end
 
-//go:generate bingen -package=opencost -version=17 -buffer=github.com/opencost/opencost/core/pkg/util
+//go:generate bingen -package=opencost -version=18 -buffer=github.com/opencost/opencost/core/pkg/util

core/pkg/opencost/cloudcost.go

@@ -5,9 +5,6 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/opencost/opencost/core/pkg/filter"
-	"github.com/opencost/opencost/core/pkg/filter/ast"
-	legacyfilter "github.com/opencost/opencost/core/pkg/filter/legacy"
 	"github.com/opencost/opencost/core/pkg/log"
 	"github.com/opencost/opencost/core/pkg/util/timeutil"
 )
@@ -287,57 +284,6 @@ func (ccs *CloudCostSet) Equal(that *CloudCostSet) bool {
 	return true
 }
 
-func (ccs *CloudCostSet) Filter(filters legacyfilter.Filter[*CloudCost]) *CloudCostSet {
-	if ccs == nil {
-		return nil
-	}
-
-	if filters == nil {
-		return ccs.Clone()
-	}
-
-	result := ccs.cloneSet()
-
-	for _, cc := range ccs.CloudCosts {
-		if filters.Matches(cc) {
-			result.Insert(cc.Clone())
-		}
-	}
-
-	return result
-}
-
-func (ccs *CloudCostSet) Filter21(filters filter.Filter) (*CloudCostSet, error) {
-	if ccs == nil {
-		return nil, nil
-	}
-
-	if filters == nil {
-		return ccs.Clone(), nil
-	}
-
-	compiler := NewCloudCostMatchCompiler()
-	var err error
-	matcher, err := compiler.Compile(filters)
-	if err != nil {
-		return ccs.Clone(), fmt.Errorf("compiling filter '%s': %w", ast.ToPreOrderShortString(filters), err)
-	}
-
-	if matcher == nil {
-		return ccs.Clone(), fmt.Errorf("unexpected nil filter")
-	}
-
-	result := ccs.cloneSet()
-
-	for _, cc := range ccs.CloudCosts {
-		if matcher.Matches(cc) {
-			result.Insert(cc.Clone())
-		}
-	}
-
-	return result, nil
-}
-
 // Insert adds a CloudCost to a CloudCostSet using its AggregationProperties and LabelConfig
 // to determine the key where it will be inserted
 func (ccs *CloudCostSet) Insert(cc *CloudCost) error {

core/pkg/opencost/cloudusage.go

@@ -1,13 +0,0 @@
-package opencost
-
-// CloudUsage is temporarily aliased as the Cloud Asset type until further infrastructure and pages can be built to support its usage
-type CloudUsage = Cloud
-
-// CloudUsageSet is temporarily aliased as the AssetSet until further infrastructure and pages can be built to support its usage
-type CloudUsageSet = AssetSet
-
-// CloudUsageSetRange is temporarily aliased as the AssetSetRange until further infrastructure and pages can be built to support its usage
-type CloudUsageSetRange = AssetSetRange
-
-// CloudUsageAggregationOptions is temporarily aliased as the AssetAggregationOptions until further infrastructure and pages can be built to support its usage
-type CloudUsageAggregationOptions = AssetAggregationOptions

core/pkg/opencost/coverage.go

@@ -1,131 +0,0 @@
-package opencost
-
-import (
-	"time"
-
-	filter "github.com/opencost/opencost/core/pkg/filter/legacy"
-	"github.com/opencost/opencost/core/pkg/log"
-)
-
-// Coverage This is a placeholder struct which can be replaced by a more specific implementation later
-type Coverage struct {
-	Window   Window    `json:"window"`
-	Type     string    `json:"type"`
-	Count    int       `json:"count"`
-	Updated  time.Time `json:"updated"`
-	Errors   []string  `json:"errors"`
-	Warnings []string  `json:"warnings"`
-}
-
-func (c *Coverage) GetWindow() Window {
-	return c.Window
-}
-
-func (c *Coverage) Key() string {
-	return c.Type
-}
-
-func (c *Coverage) IsEmpty() bool {
-	if c == nil {
-		log.Warnf("calling IsEmpty() on a nil Coverage")
-		return true
-	}
-	return c.Type == "" && c.Count == 0 && len(c.Errors) == 0 && len(c.Warnings) == 0 && c.Updated == time.Time{}
-}
-
-func (c *Coverage) Clone() *Coverage {
-	if c == nil {
-		log.Warnf("calling Clone() on a nil Coverage")
-		return nil
-	}
-	var errors []string
-	if len(c.Errors) > 0 {
-		errors = make([]string, len(c.Errors))
-		copy(errors, c.Errors)
-	}
-	var warnings []string
-	if len(c.Warnings) > 0 {
-		warnings = make([]string, len(c.Warnings))
-		copy(warnings, c.Warnings)
-	}
-	return &Coverage{
-		Window:   c.Window.Clone(),
-		Type:     c.Type,
-		Count:    c.Count,
-		Updated:  c.Updated,
-		Errors:   errors,
-		Warnings: warnings,
-	}
-}
-
-// Coverage This is a placeholder struct which can be replaced by a more specific implementation later
-type CoverageSet struct {
-	Window Window               `json:"window"`
-	Items  map[string]*Coverage `json:"items"`
-}
-
-func NewCoverageSet(start, end time.Time) *CoverageSet {
-	return &CoverageSet{
-		Window: NewWindow(&start, &end),
-		Items:  map[string]*Coverage{},
-	}
-}
-
-func (cs *CoverageSet) GetWindow() Window {
-	return cs.Window
-}
-
-func (cs *CoverageSet) IsEmpty() bool {
-	if cs == nil {
-		log.Warnf("calling IsEmpty() on a nil CoverageSet")
-		return true
-	}
-	for _, item := range cs.Items {
-		if !item.IsEmpty() {
-			return false
-		}
-	}
-	return true
-}
-
-func (cs *CoverageSet) Clone() *CoverageSet {
-	var items map[string]*Coverage
-	if cs.Items != nil {
-		items = make(map[string]*Coverage, len(cs.Items))
-		for k, item := range cs.Items {
-			items[k] = item.Clone()
-		}
-
-	}
-	return &CoverageSet{
-		Window: cs.Window.Clone(),
-		Items:  items,
-	}
-}
-
-func (cs *CoverageSet) Insert(coverage *Coverage) {
-	if cs.Items == nil {
-		cs.Items = map[string]*Coverage{}
-	}
-	cs.Items[coverage.Key()] = coverage
-}
-
-func (cs *CoverageSet) Filter(filters filter.Filter[*Coverage]) *CoverageSet {
-	if cs == nil {
-		return nil
-	}
-
-	if filters == nil {
-		return cs.Clone()
-	}
-
-	result := NewCoverageSet(*cs.Window.start, *cs.Window.end)
-
-	for _, c := range cs.Items {
-		if filters.Matches(c) {
-			result.Insert(c.Clone())
-		}
-	}
-
-	return result
-}

core/pkg/opencost/opencost_codecs.go

@@ -35,7 +35,7 @@ const (
 
 const (
 	// DefaultCodecVersion is used for any resources listed in the Default version set
-	DefaultCodecVersion uint8 = 17
+	DefaultCodecVersion uint8 = 18
 
 	// AssetsCodecVersion is used for any resources listed in the Assets version set
 	AssetsCodecVersion uint8 = 21
@@ -70,8 +70,6 @@ var typeMap map[string]reflect.Type = map[string]reflect.Type{
 	"CloudCostSetRange":     reflect.TypeOf((*CloudCostSetRange)(nil)).Elem(),
 	"ClusterManagement":     reflect.TypeOf((*ClusterManagement)(nil)).Elem(),
 	"CostMetric":            reflect.TypeOf((*CostMetric)(nil)).Elem(),
-	"Coverage":              reflect.TypeOf((*Coverage)(nil)).Elem(),
-	"CoverageSet":           reflect.TypeOf((*CoverageSet)(nil)).Elem(),
 	"Disk":                  reflect.TypeOf((*Disk)(nil)).Elem(),
 	"GPUAllocation":         reflect.TypeOf((*GPUAllocation)(nil)).Elem(),
 	"LbAllocation":          reflect.TypeOf((*LbAllocation)(nil)).Elem(),
@@ -4670,437 +4668,6 @@ func (target *CostMetric) UnmarshalBinaryWithContext(ctx *DecodingContext) (err
 	return nil
 }
 
-//--------------------------------------------------------------------------
-//  Coverage
-//--------------------------------------------------------------------------
-
-// MarshalBinary serializes the internal properties of this Coverage instance
-// into a byte array
-func (target *Coverage) MarshalBinary() (data []byte, err error) {
-	ctx := &EncodingContext{
-		Buffer: util.NewBuffer(),
-		Table:  nil,
-	}
-
-	e := target.MarshalBinaryWithContext(ctx)
-	if e != nil {
-		return nil, e
-	}
-
-	encBytes := ctx.Buffer.Bytes()
-	return encBytes, nil
-}
-
-// MarshalBinaryWithContext serializes the internal properties of this Coverage instance
-// into a byte array leveraging a predefined context.
-func (target *Coverage) MarshalBinaryWithContext(ctx *EncodingContext) (err error) {
-	// panics are recovered and propagated as errors
-	defer func() {
-		if r := recover(); r != nil {
-			if e, ok := r.(error); ok {
-				err = e
-			} else if s, ok := r.(string); ok {
-				err = fmt.Errorf("Unexpected panic: %s", s)
-			} else {
-				err = fmt.Errorf("Unexpected panic: %+v", r)
-			}
-		}
-	}()
-
-	buff := ctx.Buffer
-	buff.WriteUInt8(DefaultCodecVersion) // version
-
-	// --- [begin][write][struct](Window) ---
-	buff.WriteInt(0) // [compatibility, unused]
-	errA := target.Window.MarshalBinaryWithContext(ctx)
-	if errA != nil {
-		return errA
-	}
-	// --- [end][write][struct](Window) ---
-
-	if ctx.IsStringTable() {
-		a := ctx.Table.AddOrGet(target.Type)
-		buff.WriteInt(a) // write table index
-	} else {
-		buff.WriteString(target.Type) // write string
-	}
-	buff.WriteInt(target.Count) // write int
-	// --- [begin][write][reference](time.Time) ---
-	b, errB := target.Updated.MarshalBinary()
-	if errB != nil {
-		return errB
-	}
-	buff.WriteInt(len(b))
-	buff.WriteBytes(b)
-	// --- [end][write][reference](time.Time) ---
-
-	if target.Errors == nil {
-		buff.WriteUInt8(uint8(0)) // write nil byte
-	} else {
-		buff.WriteUInt8(uint8(1)) // write non-nil byte
-
-		// --- [begin][write][slice]([]string) ---
-		buff.WriteInt(len(target.Errors)) // array length
-		for i := 0; i < len(target.Errors); i++ {
-			if ctx.IsStringTable() {
-				c := ctx.Table.AddOrGet(target.Errors[i])
-				buff.WriteInt(c) // write table index
-			} else {
-				buff.WriteString(target.Errors[i]) // write string
-			}
-		}
-		// --- [end][write][slice]([]string) ---
-
-	}
-	if target.Warnings == nil {
-		buff.WriteUInt8(uint8(0)) // write nil byte
-	} else {
-		buff.WriteUInt8(uint8(1)) // write non-nil byte
-
-		// --- [begin][write][slice]([]string) ---
-		buff.WriteInt(len(target.Warnings)) // array length
-		for j := 0; j < len(target.Warnings); j++ {
-			if ctx.IsStringTable() {
-				d := ctx.Table.AddOrGet(target.Warnings[j])
-				buff.WriteInt(d) // write table index
-			} else {
-				buff.WriteString(target.Warnings[j]) // write string
-			}
-		}
-		// --- [end][write][slice]([]string) ---
-
-	}
-	return nil
-}
-
-// UnmarshalBinary uses the data passed byte array to set all the internal properties of
-// the Coverage type
-func (target *Coverage) UnmarshalBinary(data []byte) error {
-	var table []string
-	buff := util.NewBufferFromBytes(data)
-
-	// string table header validation
-	if isBinaryTag(data, BinaryTagStringTable) {
-		buff.ReadBytes(len(BinaryTagStringTable)) // strip tag length
-		tl := buff.ReadInt()                      // table length
-		if tl > 0 {
-			table = make([]string, tl, tl)
-			for i := 0; i < tl; i++ {
-				table[i] = buff.ReadString()
-			}
-		}
-	}
-
-	ctx := &DecodingContext{
-		Buffer: buff,
-		Table:  table,
-	}
-
-	err := target.UnmarshalBinaryWithContext(ctx)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// UnmarshalBinaryWithContext uses the context containing a string table and binary buffer to set all the internal properties of
-// the Coverage type
-func (target *Coverage) UnmarshalBinaryWithContext(ctx *DecodingContext) (err error) {
-	// panics are recovered and propagated as errors
-	defer func() {
-		if r := recover(); r != nil {
-			if e, ok := r.(error); ok {
-				err = e
-			} else if s, ok := r.(string); ok {
-				err = fmt.Errorf("Unexpected panic: %s", s)
-			} else {
-				err = fmt.Errorf("Unexpected panic: %+v", r)
-			}
-		}
-	}()
-
-	buff := ctx.Buffer
-	version := buff.ReadUInt8()
-
-	if version > DefaultCodecVersion {
-		return fmt.Errorf("Invalid Version Unmarshaling Coverage. Expected %d or less, got %d", DefaultCodecVersion, version)
-	}
-
-	// --- [begin][read][struct](Window) ---
-	a := &Window{}
-	buff.ReadInt() // [compatibility, unused]
-	errA := a.UnmarshalBinaryWithContext(ctx)
-	if errA != nil {
-		return errA
-	}
-	target.Window = *a
-	// --- [end][read][struct](Window) ---
-
-	var c string
-	if ctx.IsStringTable() {
-		d := buff.ReadInt() // read string index
-		c = ctx.Table[d]
-	} else {
-		c = buff.ReadString() // read string
-	}
-	b := c
-	target.Type = b
-
-	e := buff.ReadInt() // read int
-	target.Count = e
-
-	// --- [begin][read][reference](time.Time) ---
-	f := &time.Time{}
-	g := buff.ReadInt()    // byte array length
-	h := buff.ReadBytes(g) // byte array
-	errB := f.UnmarshalBinary(h)
-	if errB != nil {
-		return errB
-	}
-	target.Updated = *f
-	// --- [end][read][reference](time.Time) ---
-
-	if buff.ReadUInt8() == uint8(0) {
-		target.Errors = nil
-	} else {
-		// --- [begin][read][slice]([]string) ---
-		l := buff.ReadInt() // array len
-		k := make([]string, l)
-		for i := 0; i < l; i++ {
-			var m string
-			var o string
-			if ctx.IsStringTable() {
-				p := buff.ReadInt() // read string index
-				o = ctx.Table[p]
-			} else {
-				o = buff.ReadString() // read string
-			}
-			n := o
-			m = n
-
-			k[i] = m
-		}
-		target.Errors = k
-		// --- [end][read][slice]([]string) ---
-
-	}
-	if buff.ReadUInt8() == uint8(0) {
-		target.Warnings = nil
-	} else {
-		// --- [begin][read][slice]([]string) ---
-		r := buff.ReadInt() // array len
-		q := make([]string, r)
-		for j := 0; j < r; j++ {
-			var s string
-			var u string
-			if ctx.IsStringTable() {
-				w := buff.ReadInt() // read string index
-				u = ctx.Table[w]
-			} else {
-				u = buff.ReadString() // read string
-			}
-			t := u
-			s = t
-
-			q[j] = s
-		}
-		target.Warnings = q
-		// --- [end][read][slice]([]string) ---
-
-	}
-	return nil
-}
-
-//--------------------------------------------------------------------------
-//  CoverageSet
-//--------------------------------------------------------------------------
-
-// MarshalBinary serializes the internal properties of this CoverageSet instance
-// into a byte array
-func (target *CoverageSet) MarshalBinary() (data []byte, err error) {
-	ctx := &EncodingContext{
-		Buffer: util.NewBuffer(),
-		Table:  nil,
-	}
-
-	e := target.MarshalBinaryWithContext(ctx)
-	if e != nil {
-		return nil, e
-	}
-
-	encBytes := ctx.Buffer.Bytes()
-	return encBytes, nil
-}
-
-// MarshalBinaryWithContext serializes the internal properties of this CoverageSet instance
-// into a byte array leveraging a predefined context.
-func (target *CoverageSet) MarshalBinaryWithContext(ctx *EncodingContext) (err error) {
-	// panics are recovered and propagated as errors
-	defer func() {
-		if r := recover(); r != nil {
-			if e, ok := r.(error); ok {
-				err = e
-			} else if s, ok := r.(string); ok {
-				err = fmt.Errorf("Unexpected panic: %s", s)
-			} else {
-				err = fmt.Errorf("Unexpected panic: %+v", r)
-			}
-		}
-	}()
-
-	buff := ctx.Buffer
-	buff.WriteUInt8(DefaultCodecVersion) // version
-
-	// --- [begin][write][struct](Window) ---
-	buff.WriteInt(0) // [compatibility, unused]
-	errA := target.Window.MarshalBinaryWithContext(ctx)
-	if errA != nil {
-		return errA
-	}
-	// --- [end][write][struct](Window) ---
-
-	if target.Items == nil {
-		buff.WriteUInt8(uint8(0)) // write nil byte
-	} else {
-		buff.WriteUInt8(uint8(1)) // write non-nil byte
-
-		// --- [begin][write][map](map[string]*Coverage) ---
-		buff.WriteInt(len(target.Items)) // map length
-		for v, z := range target.Items {
-			if ctx.IsStringTable() {
-				a := ctx.Table.AddOrGet(v)
-				buff.WriteInt(a) // write table index
-			} else {
-				buff.WriteString(v) // write string
-			}
-			if z == nil {
-				buff.WriteUInt8(uint8(0)) // write nil byte
-			} else {
-				buff.WriteUInt8(uint8(1)) // write non-nil byte
-
-				// --- [begin][write][struct](Coverage) ---
-				buff.WriteInt(0) // [compatibility, unused]
-				errB := z.MarshalBinaryWithContext(ctx)
-				if errB != nil {
-					return errB
-				}
-				// --- [end][write][struct](Coverage) ---
-
-			}
-		}
-		// --- [end][write][map](map[string]*Coverage) ---
-
-	}
-	return nil
-}
-
-// UnmarshalBinary uses the data passed byte array to set all the internal properties of
-// the CoverageSet type
-func (target *CoverageSet) UnmarshalBinary(data []byte) error {
-	var table []string
-	buff := util.NewBufferFromBytes(data)
-
-	// string table header validation
-	if isBinaryTag(data, BinaryTagStringTable) {
-		buff.ReadBytes(len(BinaryTagStringTable)) // strip tag length
-		tl := buff.ReadInt()                      // table length
-		if tl > 0 {
-			table = make([]string, tl, tl)
-			for i := 0; i < tl; i++ {
-				table[i] = buff.ReadString()
-			}
-		}
-	}
-
-	ctx := &DecodingContext{
-		Buffer: buff,
-		Table:  table,
-	}
-
-	err := target.UnmarshalBinaryWithContext(ctx)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// UnmarshalBinaryWithContext uses the context containing a string table and binary buffer to set all the internal properties of
-// the CoverageSet type
-func (target *CoverageSet) UnmarshalBinaryWithContext(ctx *DecodingContext) (err error) {
-	// panics are recovered and propagated as errors
-	defer func() {
-		if r := recover(); r != nil {
-			if e, ok := r.(error); ok {
-				err = e
-			} else if s, ok := r.(string); ok {
-				err = fmt.Errorf("Unexpected panic: %s", s)
-			} else {
-				err = fmt.Errorf("Unexpected panic: %+v", r)
-			}
-		}
-	}()
-
-	buff := ctx.Buffer
-	version := buff.ReadUInt8()
-
-	if version > DefaultCodecVersion {
-		return fmt.Errorf("Invalid Version Unmarshaling CoverageSet. Expected %d or less, got %d", DefaultCodecVersion, version)
-	}
-
-	// --- [begin][read][struct](Window) ---
-	a := &Window{}
-	buff.ReadInt() // [compatibility, unused]
-	errA := a.UnmarshalBinaryWithContext(ctx)
-	if errA != nil {
-		return errA
-	}
-	target.Window = *a
-	// --- [end][read][struct](Window) ---
-
-	if buff.ReadUInt8() == uint8(0) {
-		target.Items = nil
-	} else {
-		// --- [begin][read][map](map[string]*Coverage) ---
-		c := buff.ReadInt() // map len
-		b := make(map[string]*Coverage, c)
-		for i := 0; i < c; i++ {
-			var v string
-			var e string
-			if ctx.IsStringTable() {
-				f := buff.ReadInt() // read string index
-				e = ctx.Table[f]
-			} else {
-				e = buff.ReadString() // read string
-			}
-			d := e
-			v = d
-
-			var z *Coverage
-			if buff.ReadUInt8() == uint8(0) {
-				z = nil
-			} else {
-				// --- [begin][read][struct](Coverage) ---
-				g := &Coverage{}
-				buff.ReadInt() // [compatibility, unused]
-				errB := g.UnmarshalBinaryWithContext(ctx)
-				if errB != nil {
-					return errB
-				}
-				z = g
-				// --- [end][read][struct](Coverage) ---
-
-			}
-			b[v] = z
-		}
-		target.Items = b
-		// --- [end][read][map](map[string]*Coverage) ---
-
-	}
-	return nil
-}
-
 //--------------------------------------------------------------------------
 //  Disk
 //--------------------------------------------------------------------------

core/pkg/opencost/query.go

@@ -4,17 +4,9 @@ import (
 	"strings"
 	"time"
 
-	filter21 "github.com/opencost/opencost/core/pkg/filter"
+	"github.com/opencost/opencost/core/pkg/filter"
 )
 
-// Querier is an aggregate interface which has the ability to query each Kubecost store type
-type Querier interface {
-	AllocationQuerier
-	SummaryAllocationQuerier
-	AssetQuerier
-	CloudUsageQuerier
-}
-
 // AllocationQuerier interface defining api for requesting Allocation data
 type AllocationQuerier interface {
 	QueryAllocation(start, end time.Time, opts *AllocationQueryOptions) (*AllocationSetRange, error)
@@ -30,18 +22,13 @@ type AssetQuerier interface {
 	QueryAsset(start, end time.Time, opts *AssetQueryOptions) (*AssetSetRange, error)
 }
 
-// CloudUsageQuerier interface defining api for requesting CloudUsage data
-type CloudUsageQuerier interface {
-	QueryCloudUsage(start, end time.Time, opts *CloudUsageQueryOptions) (*CloudUsageSetRange, error)
-}
-
 // AllocationQueryOptions defines optional parameters for querying an Allocation Store
 type AllocationQueryOptions struct {
 	Accumulate              AccumulateOption
 	AggregateBy             []string
 	Compute                 bool
 	DisableAggregatedStores bool
-	Filter                  filter21.Filter
+	Filter                  filter.Filter
 	IdleByNode              bool
 	IncludeExternal         bool
 	IncludeIdle             bool
@@ -99,113 +86,9 @@ type AssetQueryOptions struct {
 	Compute                 bool
 	DisableAdjustments      bool
 	DisableAggregatedStores bool
-	Filter                  filter21.Filter
+	Filter                  filter.Filter
 	IncludeCloud            bool
 	SharedHourlyCosts       map[string]float64
 	Step                    time.Duration
 	LabelConfig             *LabelConfig
 }
-
-// CloudUsageQueryOptions define optional parameters for querying a Store
-type CloudUsageQueryOptions struct {
-	Accumulate   bool
-	AggregateBy  []string
-	Compute      bool
-	Filter       filter21.Filter
-	FilterValues CloudUsageFilter
-	LabelConfig  *LabelConfig
-}
-
-type CloudUsageFilter struct {
-	Categories  []string            `json:"categories"`
-	Providers   []string            `json:"providers"`
-	ProviderIDs []string            `json:"providerIDs"`
-	Accounts    []string            `json:"accounts"`
-	Projects    []string            `json:"projects"`
-	Services    []string            `json:"services"`
-	Labels      map[string][]string `json:"labels"`
-}
-
-// QueryAllocationAsync provide a functions for retrieving results from any AllocationQuerier Asynchronously
-func QueryAllocationAsync(allocationQuerier AllocationQuerier, start, end time.Time, opts *AllocationQueryOptions) (chan *AllocationSetRange, chan error) {
-	asrCh := make(chan *AllocationSetRange)
-	errCh := make(chan error)
-
-	go func(asrCh chan *AllocationSetRange, errCh chan error) {
-		defer close(asrCh)
-		defer close(errCh)
-
-		asr, err := allocationQuerier.QueryAllocation(start, end, opts)
-		if err != nil {
-			errCh <- err
-			return
-		}
-
-		asrCh <- asr
-	}(asrCh, errCh)
-
-	return asrCh, errCh
-}
-
-// QuerySummaryAllocationAsync provide a functions for retrieving results from any SummaryAllocationQuerier Asynchronously
-func QuerySummaryAllocationAsync(summaryAllocationQuerier SummaryAllocationQuerier, start, end time.Time, opts *AllocationQueryOptions) (chan *SummaryAllocationSetRange, chan error) {
-	asrCh := make(chan *SummaryAllocationSetRange)
-	errCh := make(chan error)
-
-	go func(asrCh chan *SummaryAllocationSetRange, errCh chan error) {
-		defer close(asrCh)
-		defer close(errCh)
-
-		asr, err := summaryAllocationQuerier.QuerySummaryAllocation(start, end, opts)
-		if err != nil {
-			errCh <- err
-			return
-		}
-
-		asrCh <- asr
-	}(asrCh, errCh)
-
-	return asrCh, errCh
-}
-
-// QueryAsseetAsync provide a functions for retrieving results from any AssetQuerier Asynchronously
-func QueryAssetAsync(assetQuerier AssetQuerier, start, end time.Time, opts *AssetQueryOptions) (chan *AssetSetRange, chan error) {
-	asrCh := make(chan *AssetSetRange)
-	errCh := make(chan error)
-
-	go func(asrCh chan *AssetSetRange, errCh chan error) {
-		defer close(asrCh)
-		defer close(errCh)
-
-		asr, err := assetQuerier.QueryAsset(start, end, opts)
-		if err != nil {
-			errCh <- err
-			return
-		}
-
-		asrCh <- asr
-	}(asrCh, errCh)
-
-	return asrCh, errCh
-}
-
-// QueryCloudUsageAsync provide a functions for retrieving results from any CloudUsageQuerier Asynchronously
-func QueryCloudUsageAsync(cloudUsageQuerier CloudUsageQuerier, start, end time.Time, opts *CloudUsageQueryOptions) (chan *CloudUsageSetRange, chan error) {
-	cusrCh := make(chan *CloudUsageSetRange)
-	errCh := make(chan error)
-
-	go func(cusrCh chan *CloudUsageSetRange, errCh chan error) {
-		defer close(cusrCh)
-		defer close(errCh)
-
-		cusr, err := cloudUsageQuerier.QueryCloudUsage(start, end, opts)
-		if err != nil {
-			errCh <- err
-			return
-		}
-
-		cusrCh <- cusr
-	}(cusrCh, errCh)
-
-	return cusrCh, errCh
-}

core/pkg/opencost/summaryallocation.go

@@ -71,15 +71,18 @@ func NewSummaryAllocation(alloc *Allocation, reconcile, reconcileNetwork bool) *
 		CPUCoreRequestAverage:  alloc.CPUCoreRequestAverage,
 		CPUCoreUsageAverage:    alloc.CPUCoreUsageAverage,
 		CPUCost:                alloc.CPUCost + alloc.CPUCostAdjustment,
+		CPUCostIdle:            alloc.CPUCostIdle,
 		GPURequestAverage:      gpuRequestAvg,
 		GPUUsageAverage:        gpuUsageAvg,
 		GPUCost:                alloc.GPUCost + alloc.GPUCostAdjustment,
+		GPUCostIdle:            alloc.GPUCostIdle,
 		NetworkCost:            alloc.NetworkCost + alloc.NetworkCostAdjustment,
 		LoadBalancerCost:       alloc.LoadBalancerCost + alloc.LoadBalancerCostAdjustment,
 		PVCost:                 alloc.PVCost() + alloc.PVCostAdjustment,
 		RAMBytesRequestAverage: alloc.RAMBytesRequestAverage,
 		RAMBytesUsageAverage:   alloc.RAMBytesUsageAverage,
 		RAMCost:                alloc.RAMCost + alloc.RAMCostAdjustment,
+		RAMCostIdle:            alloc.RAMCostIdle,
 		SharedCost:             alloc.SharedCost,
 		ExternalCost:           alloc.ExternalCost,
 		UnmountedPVCost:        alloc.UnmountedPVCost,

go.mod

@@ -55,8 +55,8 @@ require (
 	go.opentelemetry.io/otel v1.24.0
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
 	golang.org/x/oauth2 v0.23.0
-	golang.org/x/sync v0.8.0
-	golang.org/x/text v0.19.0
+	golang.org/x/sync v0.10.0
+	golang.org/x/text v0.21.0
 	google.golang.org/api v0.183.0
 	google.golang.org/protobuf v1.35.1
 	gopkg.in/yaml.v2 v2.4.0
@@ -175,11 +175,11 @@ require (
 	go.opentelemetry.io/otel/metric v1.24.0 // indirect
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
-	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
 	golang.org/x/mod v0.21.0 // indirect
-	golang.org/x/net v0.30.0 // indirect
-	golang.org/x/sys v0.26.0 // indirect
-	golang.org/x/term v0.25.0 // indirect
+	golang.org/x/net v0.33.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/term v0.27.0 // indirect
 	golang.org/x/time v0.7.0 // indirect
 	golang.org/x/tools v0.26.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
@@ -199,4 +199,4 @@ require (
 
 go 1.23.0
 
-toolchain go1.23.4
+toolchain go1.24.0

go.sum

@@ -605,8 +605,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
-golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -684,8 +684,8 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
-golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -711,8 +711,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
-golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -767,11 +767,11 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
-golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
-golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
+golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -780,8 +780,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
-golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

modules/prometheus-source/pkg/prom/datasource.go

@@ -3,7 +3,6 @@ package prom
 import (
 	"context"
 	"fmt"
-	"math"
 	"net/http"
 	"strconv"
 	"strings"
@@ -2204,19 +2203,3 @@ func wrapResults[T any](query string, decoder source.ResultDecoder[T], results [
 
 	return source.NewFuture(decoder, ch)
 }
-
-func snapResolutionMinute(res time.Duration) time.Duration {
-	resMins := int64(math.Trunc(res.Minutes()))
-	if resMins <= 0 {
-		resMins = 1
-	}
-	return time.Duration(resMins) * time.Minute
-}
-
-func formatResolutionMinutes(resMins int64) string {
-	if resMins%60 == 0 {
-		return fmt.Sprintf("%dh", resMins/60)
-	}
-
-	return fmt.Sprintf("%dm", resMins)
-}

modules/prometheus-source/pkg/prom/diagnostics.go

@@ -25,9 +25,6 @@ const (
 	// KubecostDiagnosticMetricID is the identifier for the metric used to determine if Kubecost metrics are being scraped.
 	KubecostDiagnosticMetricID = "kubecostMetric"
 
-	// NodeExporterDiagnosticMetricID is the identifier for the metric used to determine if NodeExporter metrics are being scraped.
-	NodeExporterDiagnosticMetricID = "neMetric"
-
 	// ScrapeIntervalDiagnosticMetricID is the identifier for the metric used to determine if prometheus has its own self-scraped
 	// metrics.
 	ScrapeIntervalDiagnosticMetricID = "scrapeInterval"
@@ -77,13 +74,6 @@ var diagnosticDefinitions map[string]*diagnosticDefinition = map[string]*diagnos
 		Label:       "Kubecost metrics available",
 		Description: "Determine if metrics from Kubecost are available during last 5 minutes.",
 	},
-	NodeExporterDiagnosticMetricID: {
-		ID:          NodeExporterDiagnosticMetricID,
-		QueryFmt:    `absent_over_time(node_cpu_seconds_total{%s}[5m] %s)`,
-		Label:       "Node-exporter metrics available",
-		Description: "Determine if metrics from node-exporter are available during last 5 minutes.",
-		DocLink:     fmt.Sprintf("%s#node-exporter-metrics-available", DocumentationBaseURL),
-	},
 	CAdvisorLabelDiagnosticMetricID: {
 		ID:          CAdvisorLabelDiagnosticMetricID,
 		QueryFmt:    `absent_over_time(container_cpu_usage_seconds_total{container!="",pod!="", %s}[5m] %s)`,

modules/prometheus-source/pkg/prom/result.go

@@ -17,52 +17,48 @@ var (
 	NaNWarning warning = newWarning("Found NaN value parsing vector data point for metric")
 )
 
-func DataFieldFormatErr(query string) error {
-	return fmt.Errorf("Data field improperly formatted in prometheus response fetching query '%s'", query)
+func DataFieldFormatErr(query string, promResponse interface{}) error {
+	return fmt.Errorf("Error parsing Prometheus response: 'data' field improperly formatted. Query: '%s'. Response: '%+v'", query, promResponse)
 }
 
-func DataPointFormatErr(query string) error {
-	return fmt.Errorf("Improperly formatted datapoint from Prometheus fetching query '%s'", query)
+func DataPointFormatErr(query string, promResponse interface{}) error {
+	return fmt.Errorf("Error parsing Prometheus response: improperly formatted datapoint. Query: '%s'. Response: '%+v'", query, promResponse)
 }
 
-func MetricFieldDoesNotExistErr(query string) error {
-	return fmt.Errorf("Metric field does not exist in data result vector fetching query '%s'", query)
+func MetricFieldDoesNotExistErr(query string, promResponse interface{}) error {
+	return fmt.Errorf("Error parsing Prometheus response: 'metric' field does not exist in data result vector. Query: '%s'. Response: '%+v'", query, promResponse)
 }
 
-func MetricFieldFormatErr(query string) error {
-	return fmt.Errorf("Metric field is improperly formatted fetching query '%s'", query)
+func MetricFieldFormatErr(query string, promResponse interface{}) error {
+	return fmt.Errorf("Error parsing Prometheus response: 'metric' field improperly formatted. Query: '%s'. Response: '%+v'", query, promResponse)
 }
 
 func NoDataErr(query string) error {
 	return source.NewNoDataError(query)
 }
 
-func PromUnexpectedResponseErr(query string) error {
-	return fmt.Errorf("Unexpected response from Prometheus fetching query '%s'", query)
+func PromUnexpectedResponseErr(query string, promResponse interface{}) error {
+	return fmt.Errorf("Error parsing Prometheus response: unexpected response. Query: '%s'. Response: '%+v'", query, promResponse)
 }
 
 func QueryResultNilErr(query string) error {
 	return source.NewCommError(query)
 }
 
-func ResultFieldDoesNotExistErr(query string) error {
-	return fmt.Errorf("Result field not does not exist in prometheus response fetching query '%s'", query)
+func ResultFieldDoesNotExistErr(query string, promResponse interface{}) error {
+	return fmt.Errorf("Error parsing Prometheus response: 'result' field does not exist. Query: '%s'. Response: '%+v'", query, promResponse)
 }
 
-func ResultFieldFormatErr(query string) error {
-	return fmt.Errorf("Result field improperly formatted in prometheus response fetching query '%s'", query)
+func ResultFieldFormatErr(query string, promResponse interface{}) error {
+	return fmt.Errorf("Error parsing Prometheus response: 'result' field improperly formatted. Query: '%s'. Response: '%+v'", query, promResponse)
 }
 
-func ResultFormatErr(query string) error {
-	return fmt.Errorf("Result is improperly formatted fetching query '%s'", query)
+func ResultFormatErr(query string, promResponse interface{}) error {
+	return fmt.Errorf("Error parsing Prometheus response: 'result' field improperly formatted. Query: '%s'. Response: '%+v'", query, promResponse)
 }
 
-func ValueFieldDoesNotExistErr(query string) error {
-	return fmt.Errorf("Value field does not exist in data result vector fetching query '%s'", query)
-}
-
-func ValueFieldFormatErr(query string) error {
-	return fmt.Errorf("Values field is improperly formatted fetching query '%s'", query)
+func ValueFieldDoesNotExistErr(query string, promResponse interface{}) error {
+	return fmt.Errorf("Error parsing Prometheus response: 'value' field does not exist in data result vector. Query: '%s'. Response: '%+v'", query, promResponse)
 }
 
 // NewQueryResultError returns a QueryResults object with an error set and does not parse a result.
@@ -96,17 +92,17 @@ func NewQueryResults(query string, queryResult interface{}, resultKeys *source.R
 	// Deep Check for proper formatting
 	d, ok := data.(map[string]interface{})
 	if !ok {
-		qrs.Error = DataFieldFormatErr(query)
+		qrs.Error = DataFieldFormatErr(query, data)
 		return qrs
 	}
 	resultData, ok := d["result"]
 	if !ok {
-		qrs.Error = ResultFieldDoesNotExistErr(query)
+		qrs.Error = ResultFieldDoesNotExistErr(query, d)
 		return qrs
 	}
 	resultsData, ok := resultData.([]interface{})
 	if !ok {
-		qrs.Error = ResultFieldFormatErr(query)
+		qrs.Error = ResultFieldFormatErr(query, resultData)
 		return qrs
 	}
 
@@ -117,18 +113,18 @@ func NewQueryResults(query string, queryResult interface{}, resultKeys *source.R
 	for _, val := range resultsData {
 		resultInterface, ok := val.(map[string]interface{})
 		if !ok {
-			qrs.Error = ResultFormatErr(query)
+			qrs.Error = ResultFormatErr(query, val)
 			return qrs
 		}
 
 		metricInterface, ok := resultInterface["metric"]
 		if !ok {
-			qrs.Error = MetricFieldDoesNotExistErr(query)
+			qrs.Error = MetricFieldDoesNotExistErr(query, resultInterface)
 			return qrs
 		}
 		metricMap, ok := metricInterface.(map[string]interface{})
 		if !ok {
-			qrs.Error = MetricFieldFormatErr(query)
+			qrs.Error = MetricFieldFormatErr(query, metricInterface)
 			return qrs
 		}
 
@@ -143,7 +139,7 @@ func NewQueryResults(query string, queryResult interface{}, resultKeys *source.R
 		if !isRange {
 			dataPoint, ok := resultInterface["value"]
 			if !ok {
-				qrs.Error = ValueFieldDoesNotExistErr(query)
+				qrs.Error = ValueFieldDoesNotExistErr(query, resultInterface)
 				return qrs
 			}
 
@@ -197,7 +193,7 @@ func parseDataPoint(query string, dataPoint interface{}) (*util.Vector, warning,
 
 	value, ok := dataPoint.([]interface{})
 	if !ok || len(value) != 2 {
-		return nil, w, DataPointFormatErr(query)
+		return nil, w, DataPointFormatErr(query, dataPoint)
 	}
 
 	strVal := value[1].(string)
@@ -233,7 +229,7 @@ func labelsForMetric(metricMap map[string]interface{}) string {
 func wrapPrometheusError(query string, qr interface{}) (string, error) {
 	e, ok := qr.(map[string]interface{})["error"]
 	if !ok {
-		return "", PromUnexpectedResponseErr(query)
+		return "", PromUnexpectedResponseErr(query, qr)
 	}
 	eStr, ok := e.(string)
 	return fmt.Sprintf("'%s' parsing query '%s'", eStr, query), nil

modules/prometheus-source/pkg/prom/result_test.go

@@ -0,0 +1,89 @@
+package prom
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+)
+
+func TestErrorFunctions(t *testing.T) {
+	testCases := []struct {
+		name     string
+		fn       func(string, any) error
+		query    string
+		response any
+	}{
+		{
+			name:     "DataFieldFormatErr",
+			fn:       DataFieldFormatErr,
+			query:    "avg(node_total_hourly_cost{}) by (node, cluster, provider_id)[24h:5m]",
+			response: map[string]string{"foo": "bar"},
+		},
+		{
+			name:     "DataPointFormatErr",
+			fn:       DataPointFormatErr,
+			query:    "avg(node_total_hourly_cost{}) by (node, cluster, provider_id)[24h:5m]",
+			response: []string{"invalid"},
+		},
+		{
+			name:     "MetricFieldDoesNotExistErr",
+			fn:       MetricFieldDoesNotExistErr,
+			query:    "avg(node_total_hourly_cost{}) by (node, cluster, provider_id)[24h:5m]",
+			response: map[string]any{"values": []any{}},
+		},
+		{
+			name:     "MetricFieldFormatErr",
+			fn:       MetricFieldFormatErr,
+			query:    "avg(node_total_hourly_cost{}) by (node, cluster, provider_id)[24h:5m]",
+			response: "invalid",
+		},
+		{
+			name:     "PromUnexpectedResponseErr",
+			fn:       PromUnexpectedResponseErr,
+			query:    "avg(node_total_hourly_cost{}) by (node, cluster, provider_id)[24h:5m]",
+			response: nil,
+		},
+		{
+			name:     "ResultFieldDoesNotExistErr",
+			fn:       ResultFieldDoesNotExistErr,
+			query:    "avg(node_total_hourly_cost{}) by (node, cluster, provider_id)[24h:5m]",
+			response: map[string]any{"resultType": "matrix"},
+		},
+		{
+			name:     "ResultFieldFormatErr",
+			fn:       ResultFieldFormatErr,
+			query:    "avg(node_total_hourly_cost{}) by (node, cluster, provider_id)[24h:5m]",
+			response: "invalid",
+		},
+		{
+			name:     "ResultFormatErr",
+			fn:       ResultFormatErr,
+			query:    "avg(node_total_hourly_cost{}) by (node, cluster, provider_id)[24h:5m]",
+			response: 123,
+		},
+		{
+			name:     "ValueFieldDoesNotExistErr",
+			fn:       ValueFieldDoesNotExistErr,
+			query:    "avg(node_total_hourly_cost{}) by (node, cluster, provider_id)[24h:5m]",
+			response: map[string]any{"metric": map[string]any{}},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.fn(tc.query, tc.response)
+			if err == nil {
+				t.Errorf("Expected error, got nil")
+				return
+			}
+
+			// Verify error contains key components without being overly strict about exact wording
+			if !strings.Contains(err.Error(), tc.query) {
+				t.Errorf("Error message missing query string '%s': %s", tc.query, err.Error())
+			}
+			if !strings.Contains(err.Error(), fmt.Sprintf("%+v", tc.response)) {
+				t.Errorf("Error message missing response value '%+v': %s", tc.response, err.Error())
+			}
+		})
+	}
+}

pkg/cloud/alibaba/provider.go

@@ -529,6 +529,10 @@ func (alibaba *Alibaba) NodePricing(key models.Key) (*models.Node, models.Pricin
 	return returnNode, meta, nil
 }
 
+func (alibaba *Alibaba) GpuPricing(nodeLabels map[string]string) (string, error) {
+	return "", nil
+}
+
 // PVPricing gives a pricing information of a specific PV given by PVkey
 func (alibaba *Alibaba) PVPricing(pvk models.PVKey) (*models.PV, error) {
 	alibaba.DownloadPricingDataLock.RLock()

pkg/cloud/aws/provider.go

@@ -341,12 +341,14 @@ var volTypes = map[string]string{
 	"EBS:VolumeP-IOPS.piops": "io1",
 	"EBS:VolumeUsage.st1":    "st1",
 	"EBS:VolumeUsage.piops":  "io1",
+	"EBS:VolumeUsage.io2":    "io2",
 	"gp2":                    "EBS:VolumeUsage.gp2",
 	"gp3":                    "EBS:VolumeUsage.gp3",
 	"standard":               "EBS:VolumeUsage",
 	"sc1":                    "EBS:VolumeUsage.sc1",
 	"io1":                    "EBS:VolumeUsage.piops",
 	"st1":                    "EBS:VolumeUsage.st1",
+	"io2":                    "EBS:VolumeUsage.io2",
 }
 
 var loadedAWSSecret bool = false
@@ -655,6 +657,10 @@ func (k *awsKey) getUsageType(labels map[string]string) string {
 	return ""
 }
 
+func (awsProvider *AWS) GpuPricing(nodeLabels map[string]string) (string, error) {
+	return "", nil
+}
+
 func (aws *AWS) PVPricing(pvk models.PVKey) (*models.PV, error) {
 	pricing, ok := aws.Pricing[pvk.Features()]
 	if !ok {

pkg/cloud/azure/provider.go

@@ -1597,6 +1597,10 @@ func (az *Azure) ApplyReservedInstancePricing(nodes map[string]*models.Node) {
 
 }
 
+func (az *Azure) GpuPricing(nodeLabels map[string]string) (string, error) {
+	return "", nil
+}
+
 func (az *Azure) PVPricing(pvk models.PVKey) (*models.PV, error) {
 	az.DownloadPricingDataLock.RLock()
 	defer az.DownloadPricingDataLock.RUnlock()

pkg/cloud/gcp/provider.go

@@ -1126,6 +1126,10 @@ func (gcp *GCP) DownloadPricingData() error {
 	return nil
 }
 
+func (gcp *GCP) GpuPricing(nodeLabels map[string]string) (string, error) {
+	return "", nil
+}
+
 func (gcp *GCP) PVPricing(pvk models.PVKey) (*models.PV, error) {
 	gcp.DownloadPricingDataLock.RLock()
 	defer gcp.DownloadPricingDataLock.RUnlock()

pkg/cloud/models/models.go

@@ -308,6 +308,7 @@ type Provider interface {
 	GetDisks() ([]byte, error)
 	GetOrphanedResources() ([]OrphanedResource, error)
 	NodePricing(Key) (*Node, PricingMetadata, error)
+	GpuPricing(map[string]string) (string, error)
 	PVPricing(PVKey) (*PV, error)
 	NetworkPricing() (*Network, error)           // TODO: add key interface arg for dynamic price fetching
 	LoadBalancerPricing() (*LoadBalancer, error) // TODO: add key interface arg for dynamic price fetching

pkg/cloud/oracle/provider.go

@@ -61,6 +61,10 @@ func (o *Oracle) NodePricing(key models.Key) (*models.Node, models.PricingMetada
 	return o.RateCardStore.ForKey(key, o.DefaultPricing)
 }
 
+func (o *Oracle) GpuPricing(nodeLabels map[string]string) (string, error) {
+	return "", nil
+}
+
 func (o *Oracle) PVPricing(pvk models.PVKey) (*models.PV, error) {
 	if err := o.ensurePricingData(); err != nil {
 		return nil, err

pkg/cloud/otc/provider.go

@@ -511,6 +511,11 @@ func (otc *OTC) GetOrphanedResources() ([]models.OrphanedResource, error) {
 	return []models.OrphanedResource{}, nil
 }
 
+// TODO: Implement method
+func (otc *OTC) GpuPricing(nodeLabels map[string]string) (string, error) {
+	return "", nil
+}
+
 // TODO: Implement method
 func (otc *OTC) AllNodePricing() (interface{}, error) {
 	return nil, nil

pkg/cloud/provider/csvprovider.go

@@ -39,6 +39,7 @@ type CSVProvider struct {
 	PricingPV               map[string]*price
 	PVMapField              string
 	GPUClassPricing         map[string]*price
+	GPULabelPricing         map[string]*price
 	GPUMapFields            []string // Fields in a node's labels that represent the GPU class.
 	UsesRegion              bool
 	DownloadPricingDataLock sync.RWMutex
@@ -67,6 +68,7 @@ func (c *CSVProvider) DownloadPricingData() error {
 	nodeclasscount := make(map[string]float64)
 	pvpricing := make(map[string]*price)
 	gpupricing := make(map[string]*price)
+	gpulabelpricing := make(map[string]*price)
 	c.GPUMapFields = make([]string, 0, 1)
 	header, err := csvutil.Header(price{}, "csv")
 	if err != nil {
@@ -97,6 +99,7 @@ func (c *CSVProvider) DownloadPricingData() error {
 			c.NodeClassCount = nodeclasscount
 			c.PricingPV = pvpricing
 			c.GPUClassPricing = gpupricing
+			c.GPULabelPricing = gpulabelpricing
 			return fmt.Errorf("Invalid s3 URI: %s", c.CSVLocation)
 		}
 	} else {
@@ -109,6 +112,7 @@ func (c *CSVProvider) DownloadPricingData() error {
 		c.NodeClassCount = nodeclasscount
 		c.PricingPV = pvpricing
 		c.GPUClassPricing = gpupricing
+		c.GPULabelPricing = gpulabelpricing
 		return nil
 	}
 	csvReader := csv.NewReader(csvr)
@@ -122,6 +126,7 @@ func (c *CSVProvider) DownloadPricingData() error {
 		c.NodeClassCount = nodeclasscount
 		c.PricingPV = pvpricing
 		c.GPUClassPricing = gpupricing
+		c.GPULabelPricing = gpulabelpricing
 		return err
 	}
 	for {
@@ -178,18 +183,22 @@ func (c *CSVProvider) DownloadPricingData() error {
 		} else if p.AssetClass == "gpu" {
 			gpupricing[key] = &p
 			c.GPUMapFields = append(c.GPUMapFields, strings.ToLower(p.InstanceIDField))
+		} else if p.AssetClass == "gpulabel" {
+			labelKeyValue := p.InstanceIDField + "=" + p.InstanceID
+			gpulabelpricing[labelKeyValue] = &p
 		} else {
 			log.Infof("Unrecognized asset class %s, defaulting to node", p.AssetClass)
 			pricing[key] = &p
 			c.NodeMapField = p.InstanceIDField
 		}
 	}
-	if len(pricing) > 0 {
+	if len(pricing) > 0 || len(gpupricing) > 0 || len(gpulabelpricing) > 0 {
 		c.Pricing = pricing
 		c.NodeClassPricing = nodeclasspricing
 		c.NodeClassCount = nodeclasscount
 		c.PricingPV = pvpricing
 		c.GPUClassPricing = gpupricing
+		c.GPULabelPricing = gpulabelpricing
 	} else {
 		log.DedupedWarningf(5, "No data received from csv at %s", c.CSVLocation)
 	}
@@ -288,6 +297,16 @@ func (c *CSVProvider) NodePricing(key models.Key) (*models.Node, models.PricingM
 	return node, models.PricingMetadata{}, nil
 }
 
+func (c *CSVProvider) GpuPricing(nodeLabels map[string]string) (string, error) {
+	for key, value := range nodeLabels {
+		labelKeyValue := key + "=" + value
+		if p, ok := c.GPULabelPricing[labelKeyValue]; ok {
+			return p.MarketPriceHourly, nil
+		}
+	}
+	return "", nil
+}
+
 func NodeValueFromMapField(m string, n *clustercache.Node, useRegion bool) string {
 	mf := strings.Split(m, ".")
 	toReturn := ""
@@ -313,10 +332,10 @@ func NodeValueFromMapField(m string, n *clustercache.Node, useRegion bool) strin
 		if mf[1] == "name" {
 			return toReturn + n.Name
 		} else if mf[1] == "labels" {
-			lkey := strings.Join(mf[2:len(mf)], ".")
+			lkey := strings.Join(mf[2:], ".")
 			return toReturn + n.Labels[lkey]
 		} else if mf[1] == "annotations" {
-			akey := strings.Join(mf[2:len(mf)], ".")
+			akey := strings.Join(mf[2:], ".")
 			return toReturn + n.Annotations[akey]
 		} else {
 			log.DedupedInfof(10, "Unsupported InstanceIDField %s in CSV For Node", m)
@@ -334,10 +353,10 @@ func PVValueFromMapField(m string, n *clustercache.PersistentVolume) string {
 		if mf[1] == "name" {
 			return n.Name
 		} else if mf[1] == "labels" {
-			lkey := strings.Join(mf[2:len(mf)], "")
+			lkey := strings.Join(mf[2:], "")
 			return n.Labels[lkey]
 		} else if mf[1] == "annotations" {
-			akey := strings.Join(mf[2:len(mf)], "")
+			akey := strings.Join(mf[2:], "")
 			return n.Annotations[akey]
 		} else {
 			log.Errorf("Unsupported InstanceIDField %s in CSV For PV", m)

pkg/cloud/provider/customprovider.go

@@ -256,6 +256,10 @@ func (*CustomProvider) QuerySQL(query string) ([]byte, error) {
 	return nil, nil
 }
 
+func (cp *CustomProvider) GpuPricing(nodeLabels map[string]string) (string, error) {
+	return "", nil
+}
+
 func (cp *CustomProvider) PVPricing(pvk models.PVKey) (*models.PV, error) {
 	cpricing, err := cp.Config.GetCustomPricingData()
 	if err != nil {

pkg/cloud/provider/provider.go

@@ -1,6 +1,7 @@
 package provider
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net"
@@ -10,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/opencost/opencost/core/pkg/util/retry"
 	"github.com/opencost/opencost/pkg/cloud/alibaba"
 	"github.com/opencost/opencost/pkg/cloud/aws"
 	"github.com/opencost/opencost/pkg/cloud/azure"
@@ -148,9 +150,24 @@ func ShareTenancyCosts(p models.Provider) bool {
 
 // NewProvider looks at the nodespec or provider metadata server to decide which provider to instantiate.
 func NewProvider(cache clustercache.ClusterCache, apiKey string, config *config.ConfigFileManager) (models.Provider, error) {
-	nodes := cache.GetAllNodes()
+	getAllNodesFunc := func() ([]*clustercache.Node, error) {
+		nodes := cache.GetAllNodes()
+		if len(nodes) == 0 {
+			return nil, fmt.Errorf("no nodes found in cluster cache")
+		}
+		return nodes, nil
+	}
+
+	var nodes []*clustercache.Node
+	if !env.IsETLReadOnlyMode() {
+		// the error can be ignored because getAllNodesFunc only errors if nodes is empty, a case which we explicitly
+		// handle by checking the length of nodes below
+		nodes, _ = retry.Retry(context.Background(), getAllNodesFunc, 10, time.Second)
+	} else {
+		nodes, _ = getAllNodesFunc()
+	}
 	if len(nodes) == 0 {
-		log.Infof("Could not locate any nodes for cluster.") // valid in ETL readonly mode
+		log.Infof("Could not locate any nodes for cluster.")
 		return &CustomProvider{
 			Clientset: cache,
 			Config:    NewProviderConfig(config, "default.json"),
@@ -291,6 +308,7 @@ func getClusterProperties(node *clustercache.Node) clusterProperties {
 	if env.IsUseCustomProvider() {
 		// Use CSV provider if set
 		if env.IsUseCSVProvider() {
+			log.Debug("using custom CSV provider")
 			cp.provider = opencost.CSVProvider
 		}
 		return cp
@@ -298,34 +316,43 @@ func getClusterProperties(node *clustercache.Node) clusterProperties {
 
 	// The second conditional is mainly if you're running opencost outside of GCE, say in a local environment.
 	if metadata.OnGCE() || strings.HasPrefix(providerID, "gce") {
+		log.Debug("using GCP provider")
 		cp.provider = opencost.GCPProvider
 		cp.configFileName = "gcp.json"
 		cp.projectID = gcp.ParseGCPProjectID(providerID)
 	} else if strings.HasPrefix(providerID, "aws") {
+		log.Debug("using AWS provider")
 		cp.provider = opencost.AWSProvider
 		cp.configFileName = "aws.json"
 	} else if strings.Contains(node.Status.NodeInfo.KubeletVersion, "eks") { // Additional check for EKS, via kubelet check
+		log.Debug("using AWS provider from EKS")
 		cp.provider = opencost.AWSProvider
 		cp.configFileName = "aws.json"
 	} else if strings.HasPrefix(providerID, "azure") {
+		log.Debug("using Azure provider")
 		cp.provider = opencost.AzureProvider
 		cp.configFileName = "azure.json"
 		cp.accountID = azure.ParseAzureSubscriptionID(providerID)
 	} else if strings.HasPrefix(providerID, "scaleway") { // the scaleway provider ID looks like scaleway://instance/<instance_id>
+		log.Debug("using Scaleway provider")
 		cp.provider = opencost.ScalewayProvider
 		cp.configFileName = "scaleway.json"
 	} else if strings.Contains(node.Status.NodeInfo.KubeletVersion, "aliyun") { // provider ID is not prefix with any distinct keyword like other providers
+		log.Debug("using Alibaba provider")
 		cp.provider = opencost.AlibabaProvider
 		cp.configFileName = "alibaba.json"
 	} else if strings.HasPrefix(providerID, "ocid") {
+		log.Debug("using Oracle provider")
 		cp.provider = opencost.OracleProvider
 		cp.configFileName = "oracle.json"
 	} else if _, ok := node.Labels["cce.cloud.com/cce-nodepool"]; ok { // The node label "cce.cloud.com/cce-nodepool" exists
+		log.Debug("using OTC provider")
 		cp.provider = opencost.OTCProvider
 		cp.configFileName = "otc.json"
 	}
 	// Override provider to CSV if CSVProvider is used and custom provider is not set
 	if env.IsUseCSVProvider() {
+		log.Debug("using CSV provider")
 		cp.provider = opencost.CSVProvider
 	}
 

pkg/cloud/scaleway/provider.go

@@ -223,6 +223,10 @@ func (c *Scaleway) GetPVKey(pv *clustercache.PersistentVolume, parameters map[st
 	}
 }
 
+func (c *Scaleway) GpuPricing(nodeLabels map[string]string) (string, error) {
+	return "", nil
+}
+
 func (c *Scaleway) PVPricing(pvk models.PVKey) (*models.PV, error) {
 	c.DownloadPricingDataLock.RLock()
 	defer c.DownloadPricingDataLock.RUnlock()

pkg/clustercache/clustercache2.go

@@ -1,8 +1,9 @@
 package clustercache
 
 import (
-	"context"
+	"sync"
 
+	"github.com/opencost/opencost/pkg/env"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
@@ -26,32 +27,60 @@ type KubernetesClusterCacheV2 struct {
 	replicationControllerStore *GenericStore[*v1.ReplicationController, *ReplicationController]
 	replicaSetStore            *GenericStore[*appsv1.ReplicaSet, *ReplicaSet]
 	pdbStore                   *GenericStore[*policyv1.PodDisruptionBudget, *PodDisruptionBudget]
+	stopCh                     chan struct{}
 }
 
 func NewKubernetesClusterCacheV2(clientset kubernetes.Interface) *KubernetesClusterCacheV2 {
-	ctx := context.TODO()
 	return &KubernetesClusterCacheV2{
-		namespaceStore:             CreateStoreAndWatch(ctx, clientset.CoreV1().RESTClient(), "namespaces", transformNamespace),
-		nodeStore:                  CreateStoreAndWatch(ctx, clientset.CoreV1().RESTClient(), "nodes", transformNode),
-		persistentVolumeClaimStore: CreateStoreAndWatch(ctx, clientset.CoreV1().RESTClient(), "persistentvolumeclaims", transformPersistentVolumeClaim),
-		persistentVolumeStore:      CreateStoreAndWatch(ctx, clientset.CoreV1().RESTClient(), "persistentvolumes", transformPersistentVolume),
-		podStore:                   CreateStoreAndWatch(ctx, clientset.CoreV1().RESTClient(), "pods", transformPod),
-		replicationControllerStore: CreateStoreAndWatch(ctx, clientset.CoreV1().RESTClient(), "replicationcontrollers", transformReplicationController),
-		serviceStore:               CreateStoreAndWatch(ctx, clientset.CoreV1().RESTClient(), "services", transformService),
-		daemonSetStore:             CreateStoreAndWatch(ctx, clientset.AppsV1().RESTClient(), "daemonsets", transformDaemonSet),
-		deploymentStore:            CreateStoreAndWatch(ctx, clientset.AppsV1().RESTClient(), "deployments", transformDeployment),
-		replicaSetStore:            CreateStoreAndWatch(ctx, clientset.AppsV1().RESTClient(), "replicasets", transformReplicaSet),
-		statefulSetStore:           CreateStoreAndWatch(ctx, clientset.AppsV1().RESTClient(), "statefulsets", transformStatefulSet),
-		storageClassStore:          CreateStoreAndWatch(ctx, clientset.StorageV1().RESTClient(), "storageclasses", transformStorageClass),
-		jobStore:                   CreateStoreAndWatch(ctx, clientset.BatchV1().RESTClient(), "jobs", transformJob),
-		pdbStore:                   CreateStoreAndWatch(ctx, clientset.PolicyV1().RESTClient(), "poddisruptionbudgets", transformPodDisruptionBudget),
+		namespaceStore:             CreateStore(clientset.CoreV1().RESTClient(), "namespaces", transformNamespace),
+		nodeStore:                  CreateStore(clientset.CoreV1().RESTClient(), "nodes", transformNode),
+		persistentVolumeClaimStore: CreateStore(clientset.CoreV1().RESTClient(), "persistentvolumeclaims", transformPersistentVolumeClaim),
+		persistentVolumeStore:      CreateStore(clientset.CoreV1().RESTClient(), "persistentvolumes", transformPersistentVolume),
+		podStore:                   CreateStore(clientset.CoreV1().RESTClient(), "pods", transformPod),
+		replicationControllerStore: CreateStore(clientset.CoreV1().RESTClient(), "replicationcontrollers", transformReplicationController),
+		serviceStore:               CreateStore(clientset.CoreV1().RESTClient(), "services", transformService),
+		daemonSetStore:             CreateStore(clientset.AppsV1().RESTClient(), "daemonsets", transformDaemonSet),
+		deploymentStore:            CreateStore(clientset.AppsV1().RESTClient(), "deployments", transformDeployment),
+		replicaSetStore:            CreateStore(clientset.AppsV1().RESTClient(), "replicasets", transformReplicaSet),
+		statefulSetStore:           CreateStore(clientset.AppsV1().RESTClient(), "statefulsets", transformStatefulSet),
+		storageClassStore:          CreateStore(clientset.StorageV1().RESTClient(), "storageclasses", transformStorageClass),
+		jobStore:                   CreateStore(clientset.BatchV1().RESTClient(), "jobs", transformJob),
+		pdbStore:                   CreateStore(clientset.PolicyV1().RESTClient(), "poddisruptionbudgets", transformPodDisruptionBudget),
+		stopCh:                     make(chan struct{}),
 	}
 }
 
 func (kcc *KubernetesClusterCacheV2) Run() {
+	var wg sync.WaitGroup
+
+	if !env.IsETLReadOnlyMode() {
+		wg.Add(14)
+
+		kcc.namespaceStore.Watch(kcc.stopCh, wg.Done)
+		kcc.nodeStore.Watch(kcc.stopCh, wg.Done)
+		kcc.persistentVolumeClaimStore.Watch(kcc.stopCh, wg.Done)
+		kcc.persistentVolumeStore.Watch(kcc.stopCh, wg.Done)
+		kcc.podStore.Watch(kcc.stopCh, wg.Done)
+		kcc.replicationControllerStore.Watch(kcc.stopCh, wg.Done)
+		kcc.serviceStore.Watch(kcc.stopCh, wg.Done)
+		kcc.daemonSetStore.Watch(kcc.stopCh, wg.Done)
+		kcc.deploymentStore.Watch(kcc.stopCh, wg.Done)
+		kcc.replicaSetStore.Watch(kcc.stopCh, wg.Done)
+		kcc.statefulSetStore.Watch(kcc.stopCh, wg.Done)
+		kcc.storageClassStore.Watch(kcc.stopCh, wg.Done)
+		kcc.jobStore.Watch(kcc.stopCh, wg.Done)
+		kcc.pdbStore.Watch(kcc.stopCh, wg.Done)
+	}
+
+	wg.Wait()
 }
 
 func (kcc *KubernetesClusterCacheV2) Stop() {
+	if kcc.stopCh != nil {
+		close(kcc.stopCh)
+
+		kcc.stopCh = nil
+	}
 }
 
 func (kcc *KubernetesClusterCacheV2) GetAllNamespaces() []*Namespace {

pkg/clustercache/store.go

@@ -1,7 +1,6 @@
 package clustercache
 
 import (
-	"context"
 	"sync"
 
 	v1 "k8s.io/api/core/v1"
@@ -17,6 +16,10 @@ type GenericStore[Input UIDGetter, Output any] struct {
 	mutex         sync.RWMutex
 	items         map[types.UID]Output
 	transformFunc func(input Input) Output
+
+	// storing this cyclic reflector allows us to defer watching
+	reflector *cache.Reflector
+	onInit    func()
 }
 
 // NewGenericStore creates a new instance of GenericStore.
@@ -31,8 +34,7 @@ type UIDGetter interface {
 	GetUID() types.UID
 }
 
-func CreateStoreAndWatch[Input UIDGetter, Output any](
-	ctx context.Context,
+func CreateStore[Input UIDGetter, Output any](
 	restClient rest.Interface,
 	resource string,
 	transformFunc func(input Input) Output,
@@ -40,11 +42,19 @@ func CreateStoreAndWatch[Input UIDGetter, Output any](
 	lw := cache.NewListWatchFromClient(restClient, resource, v1.NamespaceAll, fields.Everything())
 	store := NewGenericStore(transformFunc)
 	var zeroValue Input
-	reflector := cache.NewReflector(lw, zeroValue, store, 0)
-	go reflector.Run(ctx.Done())
+	store.reflector = cache.NewReflector(lw, zeroValue, store, 0)
+
 	return store
 }
 
+func (s *GenericStore[Input, Output]) Watch(stopCh <-chan struct{}, onInit func()) {
+	s.onInit = onInit
+
+	// reflector.Run() will eventually call Replace() on the store with the initial contents
+	// of the resource list. we'll call onInit after that happens the _first_ time
+	go s.reflector.Run(stopCh)
+}
+
 // Add inserts an object into the store.
 func (s *GenericStore[Input, Output]) Add(obj any) error {
 	return s.Update(obj)
@@ -96,16 +106,28 @@ func (s *GenericStore[Input, Output]) Replace(list []any, _ string) error {
 		}
 	}
 
+	// call onInit after the initial list has been processed
+	if s.onInit != nil {
+		s.onInit()
+		s.onInit = nil
+	}
+
 	return nil
 }
 
 // Stubs to satisfy the cache.Store interface
-func (s *GenericStore[Input, Output]) List() []interface{} { return nil }
-func (s *GenericStore[Input, Output]) ListKeys() []string  { return nil }
+func (s *GenericStore[Input, Output]) List() []interface{} {
+	return nil
+}
+func (s *GenericStore[Input, Output]) ListKeys() []string {
+	return nil
+}
 func (s *GenericStore[Input, Output]) Get(_ interface{}) (item interface{}, exists bool, err error) {
 	return nil, false, nil
 }
 func (s *GenericStore[Input, Output]) GetByKey(_ string) (item interface{}, exists bool, err error) {
 	return nil, false, nil
 }
-func (s *GenericStore[Input, Output]) Resync() error { return nil }
+func (s *GenericStore[Input, Output]) Resync() error {
+	return nil
+}

pkg/clustercache/watchcontroller.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/opencost/opencost/core/pkg/log"
-
+	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/fields"
 	rt "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/runtime"
@@ -55,7 +55,7 @@ type CachingWatchController struct {
 func NewCachingWatcher(restClient rest.Interface, resource string, resourceType rt.Object, namespace string, fieldSelector fields.Selector) WatchController {
 	resourceCache := cache.NewListWatchFromClient(restClient, resource, namespace, fieldSelector)
 	queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
-	indexer, informer := cache.NewIndexerInformer(resourceCache, resourceType, 0, cache.ResourceEventHandlerFuncs{
+	indexer, informer := cache.NewTransformingIndexerInformer(resourceCache, resourceType, 0, cache.ResourceEventHandlerFuncs{
 		AddFunc: func(obj interface{}) {
 			key, err := cache.MetaNamespaceKeyFunc(obj)
 			if err == nil {
@@ -76,7 +76,7 @@ func NewCachingWatcher(restClient rest.Interface, resource string, resourceType
 				queue.Add(key)
 			}
 		},
-	}, cache.Indexers{})
+	}, cache.Indexers{}, trimUnwantedFields)
 
 	return &CachingWatchController{
 		indexer:      indexer,
@@ -206,3 +206,12 @@ func (c *CachingWatchController) runWorker() {
 	for c.processNextItem() {
 	}
 }
+
+// trimUnwantedFields removes unwanted fields from the object
+// - managedFields as this metadata can be quite large
+func trimUnwantedFields(obj interface{}) (interface{}, error) {
+	if accessor, err := meta.Accessor(obj); err == nil {
+		accessor.SetManagedFields(nil)
+	}
+	return obj, nil
+}

pkg/costmodel/allocation_helpers.go

@@ -11,7 +11,6 @@ import (
 	"github.com/opencost/opencost/core/pkg/opencost"
 	"github.com/opencost/opencost/core/pkg/source"
 	"github.com/opencost/opencost/core/pkg/util"
-	"github.com/opencost/opencost/core/pkg/util/promutil"
 	"github.com/opencost/opencost/core/pkg/util/timeutil"
 	"github.com/opencost/opencost/pkg/cloud/provider"
 	"github.com/opencost/opencost/pkg/env"
@@ -1012,23 +1011,11 @@ func resToNodeLabels(resNodeLabels []*source.NodeLabelsResult) map[nodeKey]map[s
 			nodeLabels[nodeKey] = map[string]string{}
 		}
 
-		for _, rawK := range env.GetAllocationNodeLabelsIncludeList() {
-			labels := res.Labels
-
-			// Sanitize the given label name to match Prometheus formatting
-			// e.g. topology.kubernetes.io/zone => topology_kubernetes_io_zone
-			k := promutil.SanitizeLabelName(rawK)
-			if v, ok := labels[k]; ok {
-				nodeLabels[nodeKey][k] = v
-				continue
-			}
-
-			// Try with the "label_" prefix, if not found
-			// e.g. topology_kubernetes_io_zone => label_topology_kubernetes_io_zone
-			k = fmt.Sprintf("label_%s", k)
-			if v, ok := labels[k]; ok {
-				nodeLabels[nodeKey][k] = v
-			}
+		labels := res.Labels
+		// labels are retrieved from prometheus here so it will be in prometheus sanitized state
+		// e.g. topology.kubernetes.io/zone => topology_kubernetes_io_zone
+		for labelKey, labelValue := range labels {
+			nodeLabels[nodeKey][labelKey] = labelValue
 		}
 	}
 
@@ -1330,7 +1317,7 @@ func resToPodDaemonSetMap(resDaemonSetLabels []*source.DaemonSetLabelsResult, po
 		}
 
 		pod := res.Pod
-		if err != nil {
+		if pod == "" {
 			log.Warnf("CostModel.ComputeAllocation: DaemonSetLabel result without pod: %s", controllerKey)
 		}
 

pkg/costmodel/costmodel.go

@@ -966,6 +966,38 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 				defaultCPUCorePrice = 0
 			}
 
+			// Some customers may want GPU pricing to be determined by the labels affixed to their nodes. GpuPricing
+			// passes the node's labels to the provider, which then cross-references them with the labels that the
+			// provider knows to have label-specific costs associated with them, and returns that cost. See CSVProvider
+			// for an example implementation.
+			var gpuPrice float64
+			gpuPricing, err := cp.GpuPricing(nodeLabels)
+			if err != nil {
+				log.Errorf("Could not determine custom GPU pricing: %s", err)
+				gpuPrice = 0
+			} else if len(gpuPricing) > 0 {
+				gpuPrice, err = strconv.ParseFloat(gpuPricing, 64)
+				if err != nil {
+					log.Errorf("Could not parse custom GPU pricing: %s", err)
+					gpuPrice = 0
+				} else if math.IsNaN(gpuPrice) {
+					log.Warnf("Custom GPU pricing parsed as NaN. Setting to 0.")
+					gpuPrice = 0
+				} else {
+					log.Infof("Using custom GPU pricing for node \"%s\": %f", name, gpuPrice)
+				}
+			} else {
+				gpuPrice, err = strconv.ParseFloat(cfg.GPU, 64)
+				if err != nil {
+					log.Errorf("Could not parse default gpu price")
+					gpuPrice = 0
+				}
+				if math.IsNaN(gpuPrice) {
+					log.Warnf("defaultGPU parsed as NaN. Setting to 0.")
+					gpuPrice = 0
+				}
+			}
+
 			defaultRAMPrice, err := strconv.ParseFloat(cfg.RAM, 64)
 			if err != nil {
 				log.Errorf("Could not parse default ram price")
@@ -987,13 +1019,13 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 			}
 			// Just say no to doing the ratios!
 			cpuCost := defaultCPUCorePrice * cpu
-			gpuCost := defaultGPUPrice * gpuc
+			gpuCost := gpuPrice * gpuc
 			ramCost := defaultRAMPrice * ram
 			nodeCost := cpuCost + gpuCost + ramCost
 
 			newCnode.Cost = fmt.Sprintf("%f", nodeCost)
 			newCnode.VCPUCost = fmt.Sprintf("%f", defaultCPUCorePrice)
-			newCnode.GPUCost = fmt.Sprintf("%f", defaultGPUPrice)
+			newCnode.GPUCost = fmt.Sprintf("%f", gpuPrice)
 			newCnode.RAMCost = fmt.Sprintf("%f", defaultRAMPrice)
 			newCnode.RAMBytes = fmt.Sprintf("%f", ram)
 
@@ -1007,95 +1039,109 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 			// cost among the CPU, RAM, and GPU.
 			log.Tracef("GPU without cost found for %s, calculating...", cp.GetKey(nodeLabels, n).Features())
 
-			defaultCPU, err := strconv.ParseFloat(cfg.CPU, 64)
+			// Some customers may want GPU pricing to be determined by the labels affixed to their nodes. GpuPricing
+			// passes the node's labels to the provider, which then cross-references them with the labels that the
+			// provider knows to have label-specific costs associated with them, and returns that cost. See CSVProvider
+			// for an example implementation.
+			gpuPricing, err := cp.GpuPricing(nodeLabels)
 			if err != nil {
-				log.Errorf("Could not parse default cpu price")
-				defaultCPU = 0
-			}
-			if math.IsNaN(defaultCPU) {
-				log.Warnf("defaultCPU parsed as NaN. Setting to 0.")
-				defaultCPU = 0
+				log.Errorf("Could not determine custom GPU pricing: %s", err)
+			} else if len(gpuPricing) > 0 {
+				newCnode.GPUCost = gpuPricing
+				log.Infof("Using custom GPU pricing for node \"%s\": %s", name, gpuPricing)
 			}
 
-			defaultRAM, err := strconv.ParseFloat(cfg.RAM, 64)
-			if err != nil {
-				log.Errorf("Could not parse default ram price")
-				defaultRAM = 0
-			}
-			if math.IsNaN(defaultRAM) {
-				log.Warnf("defaultRAM parsed as NaN. Setting to 0.")
-				defaultRAM = 0
-			}
+			if newCnode.GPUCost == "" {
+				defaultCPU, err := strconv.ParseFloat(cfg.CPU, 64)
+				if err != nil {
+					log.Errorf("Could not parse default cpu price")
+					defaultCPU = 0
+				}
+				if math.IsNaN(defaultCPU) {
+					log.Warnf("defaultCPU parsed as NaN. Setting to 0.")
+					defaultCPU = 0
+				}
 
-			defaultGPU, err := strconv.ParseFloat(cfg.GPU, 64)
-			if err != nil {
-				log.Errorf("Could not parse default gpu price")
-				defaultGPU = 0
-			}
-			if math.IsNaN(defaultGPU) {
-				log.Warnf("defaultGPU parsed as NaN. Setting to 0.")
-				defaultGPU = 0
-			}
+				defaultRAM, err := strconv.ParseFloat(cfg.RAM, 64)
+				if err != nil {
+					log.Errorf("Could not parse default ram price")
+					defaultRAM = 0
+				}
+				if math.IsNaN(defaultRAM) {
+					log.Warnf("defaultRAM parsed as NaN. Setting to 0.")
+					defaultRAM = 0
+				}
 
-			cpuToRAMRatio := defaultCPU / defaultRAM
-			if math.IsNaN(cpuToRAMRatio) {
-				log.Warnf("cpuToRAMRatio[defaultCPU: %f / defaultRAM: %f] is NaN. Setting to 10.", defaultCPU, defaultRAM)
-				cpuToRAMRatio = 10
-			}
+				defaultGPU, err := strconv.ParseFloat(cfg.GPU, 64)
+				if err != nil {
+					log.Errorf("Could not parse default gpu price")
+					defaultGPU = 0
+				}
+				if math.IsNaN(defaultGPU) {
+					log.Warnf("defaultGPU parsed as NaN. Setting to 0.")
+					defaultGPU = 0
+				}
 
-			gpuToRAMRatio := defaultGPU / defaultRAM
-			if math.IsNaN(gpuToRAMRatio) {
-				log.Warnf("gpuToRAMRatio is NaN. Setting to 100.")
-				gpuToRAMRatio = 100
-			}
+				cpuToRAMRatio := defaultCPU / defaultRAM
+				if math.IsNaN(cpuToRAMRatio) {
+					log.Warnf("cpuToRAMRatio[defaultCPU: %f / defaultRAM: %f] is NaN. Setting to 10.", defaultCPU, defaultRAM)
+					cpuToRAMRatio = 10
+				}
 
-			ramGB := ram / 1024 / 1024 / 1024
-			if math.IsNaN(ramGB) {
-				log.Warnf("ramGB is NaN. Setting to 0.")
-				ramGB = 0
-			}
+				gpuToRAMRatio := defaultGPU / defaultRAM
+				if math.IsNaN(gpuToRAMRatio) {
+					log.Warnf("gpuToRAMRatio is NaN. Setting to 100.")
+					gpuToRAMRatio = 100
+				}
 
-			ramMultiple := gpuc*gpuToRAMRatio + cpu*cpuToRAMRatio + ramGB
-			if math.IsNaN(ramMultiple) {
-				log.Warnf("ramMultiple is NaN. Setting to 0.")
-				ramMultiple = 0
-			}
+				ramGB := ram / 1024 / 1024 / 1024
+				if math.IsNaN(ramGB) {
+					log.Warnf("ramGB is NaN. Setting to 0.")
+					ramGB = 0
+				}
 
-			var nodePrice float64
-			if newCnode.Cost != "" {
-				nodePrice, err = strconv.ParseFloat(newCnode.Cost, 64)
-				if err != nil {
-					log.Errorf("Could not parse total node price")
-					return nil, err
+				ramMultiple := gpuc*gpuToRAMRatio + cpu*cpuToRAMRatio + ramGB
+				if math.IsNaN(ramMultiple) {
+					log.Warnf("ramMultiple is NaN. Setting to 0.")
+					ramMultiple = 0
 				}
-			} else if newCnode.VCPUCost != "" {
-				nodePrice, err = strconv.ParseFloat(newCnode.VCPUCost, 64) // all the price was allocated to the CPU
-				if err != nil {
-					log.Errorf("Could not parse node vcpu price")
-					return nil, err
+
+				var nodePrice float64
+				if newCnode.Cost != "" {
+					nodePrice, err = strconv.ParseFloat(newCnode.Cost, 64)
+					if err != nil {
+						log.Errorf("Could not parse total node price")
+						return nil, err
+					}
+				} else if newCnode.VCPUCost != "" {
+					nodePrice, err = strconv.ParseFloat(newCnode.VCPUCost, 64) // all the price was allocated to the CPU
+					if err != nil {
+						log.Errorf("Could not parse node vcpu price")
+						return nil, err
+					}
+				} else { // add case to use default pricing model when API data fails.
+					log.Debugf("No node price or CPUprice found, falling back to default")
+					nodePrice = defaultCPU*cpu + defaultRAM*ram + gpuc*defaultGPU
+				}
+				if math.IsNaN(nodePrice) {
+					log.Warnf("nodePrice parsed as NaN. Setting to 0.")
+					nodePrice = 0
 				}
-			} else { // add case to use default pricing model when API data fails.
-				log.Debugf("No node price or CPUprice found, falling back to default")
-				nodePrice = defaultCPU*cpu + defaultRAM*ram + gpuc*defaultGPU
-			}
-			if math.IsNaN(nodePrice) {
-				log.Warnf("nodePrice parsed as NaN. Setting to 0.")
-				nodePrice = 0
-			}
 
-			ramPrice := (nodePrice / ramMultiple)
-			if math.IsNaN(ramPrice) {
-				log.Warnf("ramPrice[nodePrice: %f / ramMultiple: %f] parsed as NaN. Setting to 0.", nodePrice, ramMultiple)
-				ramPrice = 0
-			}
+				ramPrice := (nodePrice / ramMultiple)
+				if math.IsNaN(ramPrice) {
+					log.Warnf("ramPrice[nodePrice: %f / ramMultiple: %f] parsed as NaN. Setting to 0.", nodePrice, ramMultiple)
+					ramPrice = 0
+				}
 
-			cpuPrice := ramPrice * cpuToRAMRatio
-			gpuPrice := ramPrice * gpuToRAMRatio
+				cpuPrice := ramPrice * cpuToRAMRatio
+				gpuPrice := ramPrice * gpuToRAMRatio
 
-			newCnode.VCPUCost = fmt.Sprintf("%f", cpuPrice)
-			newCnode.RAMCost = fmt.Sprintf("%f", ramPrice)
-			newCnode.RAMBytes = fmt.Sprintf("%f", ram)
-			newCnode.GPUCost = fmt.Sprintf("%f", gpuPrice)
+				newCnode.VCPUCost = fmt.Sprintf("%f", cpuPrice)
+				newCnode.RAMCost = fmt.Sprintf("%f", ramPrice)
+				newCnode.RAMBytes = fmt.Sprintf("%f", ram)
+				newCnode.GPUCost = fmt.Sprintf("%f", gpuPrice)
+			}
 		} else if newCnode.RAMCost == "" {
 			// We reach this when no RAM cost is defined in the OnDemand
 			// pricing. It calculates a cpuToRAMRatio and ramMultiple to
@@ -1271,6 +1317,37 @@ func getPodServices(cache clustercache.ClusterCache, podList []*clustercache.Pod
 	return podServicesMapping, nil
 }
 
+func getPodStatefulsets(cache clustercache.ClusterCache, podList []*clustercache.Pod, clusterID string) (map[string]map[string][]string, error) {
+	ssList := cache.GetAllStatefulSets()
+	podSSMapping := make(map[string]map[string][]string) // namespace: podName: [deploymentNames]
+	for _, ss := range ssList {
+		namespace := ss.Namespace
+		name := ss.Name
+
+		key := namespace + "," + clusterID
+		if _, ok := podSSMapping[key]; !ok {
+			podSSMapping[key] = make(map[string][]string)
+		}
+		s, err := metav1.LabelSelectorAsSelector(ss.SpecSelector)
+		if err != nil {
+			log.Errorf("Error doing deployment label conversion: " + err.Error())
+		}
+		for _, pod := range podList {
+			labelSet := labels.Set(pod.Labels)
+			if s.Matches(labelSet) && pod.Namespace == namespace {
+				sss, ok := podSSMapping[key][pod.Name]
+				if ok {
+					podSSMapping[key][pod.Name] = append(sss, name)
+				} else {
+					podSSMapping[key][pod.Name] = []string{name}
+				}
+			}
+		}
+	}
+	return podSSMapping, nil
+
+}
+
 func getPodDeployments(cache clustercache.ClusterCache, podList []*clustercache.Pod, clusterID string) (map[string]map[string][]string, error) {
 	deploymentsList := cache.GetAllDeployments()
 	podDeploymentsMapping := make(map[string]map[string][]string) // namespace: podName: [deploymentNames]

pkg/costmodel/intervals_test.go

@@ -362,7 +362,7 @@ func TestGetPVCCostCoefficients(t *testing.T) {
 				return
 			}
 
-			if err == nil && testCase.expError != nil {
+			if testCase.expError != nil {
 				t.Errorf("getPVCCostCoefficients failed: did not get expected error: %v", testCase.expError)
 			}
 

pkg/env/costmodelenv.go

@@ -76,8 +76,7 @@ const (
 
 	ETLReadOnlyMode = "ETL_READ_ONLY"
 
-	AllocationNodeLabelsEnabled     = "ALLOCATION_NODE_LABELS_ENABLED"
-	AllocationNodeLabelsIncludeList = "ALLOCATION_NODE_LABELS_INCLUDE_LIST"
+	AllocationNodeLabelsEnabled = "ALLOCATION_NODE_LABELS_ENABLED"
 
 	AssetIncludeLocalDiskCostEnvVar = "ASSET_INCLUDE_LOCAL_DISK_COST"
 
@@ -439,31 +438,6 @@ func GetAllocationNodeLabelsEnabled() bool {
 	return env.GetBool(AllocationNodeLabelsEnabled, true)
 }
 
-var defaultAllocationNodeLabelsIncludeList []string = []string{
-	"cloud.google.com/gke-nodepool",
-	"eks.amazonaws.com/nodegroup",
-	"kubernetes.azure.com/agentpool",
-	"node.kubernetes.io/instance-type",
-	"topology.kubernetes.io/region",
-	"topology.kubernetes.io/zone",
-}
-
-func GetAllocationNodeLabelsIncludeList() []string {
-	// If node labels are not enabled, return an empty list.
-	if !GetAllocationNodeLabelsEnabled() {
-		return []string{}
-	}
-
-	list := env.GetList(AllocationNodeLabelsIncludeList, ",")
-
-	// If node labels are enabled, but the white list is empty, use defaults.
-	if len(list) == 0 {
-		return defaultAllocationNodeLabelsIncludeList
-	}
-
-	return list
-}
-
 func GetAssetIncludeLocalDiskCost() bool {
 	return env.GetBool(AssetIncludeLocalDiskCostEnvVar, true)
 }

test/cloud_test.go

@@ -236,6 +236,97 @@ func TestNodePriceFromCSVWithGPU(t *testing.T) {
 
 }
 
+func TestNodePriceFromCSVWithGPULabels(t *testing.T) {
+	nameWant := "gke-standard-cluster-1-pool-1-91dc432d-cg69"
+	wantGPUCost := "0.75"
+
+	confMan := config.NewConfigFileManager(&config.ConfigFileManagerOpts{
+		LocalConfigPath: "./",
+	})
+
+	n := &clustercache.Node{}
+	n.SpecProviderID = "providerid"
+	n.Name = nameWant
+	n.Labels = make(map[string]string)
+	n.Labels["foo"] = "labelfoo"
+	n.Labels["nvidia.com/gpu_type"] = "Quadro_RTX_4000"
+	n.Status.Capacity = v1.ResourceList{"nvidia.com/gpu": *resource.NewScaledQuantity(2, 0)}
+
+	c := &provider.CSVProvider{
+		CSVLocation: "../configs/pricing_schema_gpu_labels.csv",
+		CustomProvider: &provider.CustomProvider{
+			Config: provider.NewProviderConfig(confMan, "../configs/default.json"),
+		},
+	}
+
+	c.DownloadPricingData()
+
+	fc := NewFakeNodeCache([]*clustercache.Node{n})
+	fm := FakeClusterMap{}
+	d, _ := time.ParseDuration("1m")
+
+	model := costmodel.NewCostModel(nil, nil, fc, fm, d)
+
+	nodeMap, err := model.GetNodeCost(c)
+	if err != nil {
+		t.Errorf("Error in NodePricing: %s", err.Error())
+	} else {
+		if node, ok := nodeMap[nameWant]; ok {
+			if node.GPUCost != wantGPUCost {
+				t.Errorf("Wanted gpu cost '%v' got gpu cost '%v'", wantGPUCost, node.GPUCost)
+			}
+		} else {
+			t.Errorf("Node %s not found in node map", nameWant)
+		}
+	}
+}
+
+func TestRKE2NodePriceFromCSVWithGPULabels(t *testing.T) {
+	nameWant := "gke-standard-cluster-1-pool-1-91dc432d-cg69"
+	wantGPUCost := "0.750000"
+
+	confMan := config.NewConfigFileManager(&config.ConfigFileManagerOpts{
+		LocalConfigPath: "./",
+	})
+
+	n := &clustercache.Node{}
+	n.SpecProviderID = "providerid"
+	n.Name = nameWant
+	n.Labels = make(map[string]string)
+	n.Labels["foo"] = "labelfoo"
+	n.Labels["nvidia.com/gpu_type"] = "Quadro_RTX_4000"
+	n.Labels[v1.LabelInstanceTypeStable] = "rke2"
+	n.Status.Capacity = v1.ResourceList{"nvidia.com/gpu": *resource.NewScaledQuantity(2, 0)}
+
+	c := &provider.CSVProvider{
+		CSVLocation: "../configs/pricing_schema_gpu_labels.csv",
+		CustomProvider: &provider.CustomProvider{
+			Config: provider.NewProviderConfig(confMan, "../configs/default.json"),
+		},
+	}
+
+	c.DownloadPricingData()
+
+	fc := NewFakeNodeCache([]*clustercache.Node{n})
+	fm := FakeClusterMap{}
+	d, _ := time.ParseDuration("1m")
+
+	model := costmodel.NewCostModel(nil, nil, fc, fm, d)
+
+	nodeMap, err := model.GetNodeCost(c)
+	if err != nil {
+		t.Errorf("Error in NodePricing: %s", err.Error())
+	} else {
+		if node, ok := nodeMap[nameWant]; ok {
+			if node.GPUCost != wantGPUCost {
+				t.Errorf("Wanted gpu cost '%v' got gpu cost '%v'", wantGPUCost, node.GPUCost)
+			}
+		} else {
+			t.Errorf("Node %s not found in node map", nameWant)
+		}
+	}
+}
+
 func TestNodePriceFromCSVSpecialChar(t *testing.T) {
 	nameWant := "gke-standard-cluster-1-pool-1-91dc432d-cg69"
 

test/configs/default.json

@@ -0,0 +1 @@
+{"provider":"base","description":"Default prices based on GCP us-central1","CPU":"0.021811","spotCPU":"0.006543","RAM":"0.002923","spotRAM":"0.000877","GPU":"0.95","spotGPU":"0.308","storage":"0.00005479452","zoneNetworkEgress":"0.01","regionNetworkEgress":"0.01","internetNetworkEgress":"0.12","firstFiveForwardingRulesCost":"","additionalForwardingRuleCost":"","LBIngressDataCost":"","athenaBucketName":"","athenaRegion":"","athenaDatabase":"","athenaCatalog":"","athenaTable":"","athenaWorkgroup":"","masterPayerARN":"","customPricesEnabled":"false","defaultIdle":"","azureSubscriptionID":"","azureClientID":"","azureClientSecret":"","azureTenantID":"","azureBillingRegion":"","azureBillingAccount":"","azureOfferDurableID":"","azureStorageSubscriptionID":"","azureStorageAccount":"","azureStorageAccessKey":"","azureStorageContainer":"","azureContainerPath":"","azureCloud":"","currencyCode":"","discount":"","negotiatedDiscount":"","sharedOverhead":"","clusterName":"","sharedNamespaces":"","sharedLabelNames":"","sharedLabelValues":"","shareTenancyCosts":"true","readOnly":"","editorAccess":"","kubecostToken":"","googleAnalyticsTag":"","excludeProviderID":"","defaultLBPrice":""}