Merge branch 'develop' into fix-panic-csv

pkg/filter/util/cloudcost.go

@@ -0,0 +1,119 @@
+package util
+
+import (
+	"strings"
+
+	"github.com/opencost/opencost/pkg/filter"
+	"github.com/opencost/opencost/pkg/kubecost"
+	"github.com/opencost/opencost/pkg/log"
+	"github.com/opencost/opencost/pkg/util/mapper"
+)
+
+func parseWildcardEnd(rawFilterValue string) (string, bool) {
+	return strings.TrimSuffix(rawFilterValue, "*"), strings.HasSuffix(rawFilterValue, "*")
+}
+
+func CloudCostFilterFromParams(pmr mapper.PrimitiveMapReader) filter.Filter[*kubecost.CloudCost] {
+	filter := filter.And[*kubecost.CloudCost]{
+		Filters: []filter.Filter[*kubecost.CloudCost]{},
+	}
+
+	if raw := pmr.GetList("filterInvoiceEntityIDs", ","); len(raw) > 0 {
+		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostInvoiceEntityIDProp))
+	}
+
+	if raw := pmr.GetList("filterAccountIDs", ","); len(raw) > 0 {
+		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostAccountIDProp))
+	}
+
+	if raw := pmr.GetList("filterProviders", ","); len(raw) > 0 {
+		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostProviderProp))
+	}
+
+	if raw := pmr.GetList("filterProviderIDs", ","); len(raw) > 0 {
+		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostProviderIDProp))
+	}
+
+	if raw := pmr.GetList("filterServices", ","); len(raw) > 0 {
+		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostServiceProp))
+	}
+
+	if raw := pmr.GetList("filterCategories", ","); len(raw) > 0 {
+		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostCategoryProp))
+	}
+
+	if raw := pmr.GetList("filterLabels", ","); len(raw) > 0 {
+		filter.Filters = append(filter.Filters, filterV1DoubleValueFromList(raw, kubecost.CloudCostLabelProp))
+	}
+
+	if len(filter.Filters) == 0 {
+		return nil
+	}
+
+	return filter
+}
+
+func filterV1SingleValueFromList(rawFilterValues []string, field string) filter.Filter[*kubecost.CloudCost] {
+	result := filter.Or[*kubecost.CloudCost]{
+		Filters: []filter.Filter[*kubecost.CloudCost]{},
+	}
+
+	for _, filterValue := range rawFilterValues {
+		filterValue = strings.TrimSpace(filterValue)
+		filterValue, wildcard := parseWildcardEnd(filterValue)
+
+		subFilter := filter.StringProperty[*kubecost.CloudCost]{
+			Field: field,
+			Op:    filter.StringEquals,
+			Value: filterValue,
+		}
+
+		if wildcard {
+			subFilter.Op = kubecost.FilterStartsWith
+		}
+
+		result.Filters = append(result.Filters, subFilter)
+	}
+
+	return result
+}
+
+// filterV1DoubleValueFromList creates an OR of key:value equality filters for
+// colon-split filter values.
+//
+// The v1 query language (e.g. "filterLabels=app:foo,l2:bar") uses OR within
+// a field (e.g. label[app] = foo OR label[l2] = bar)
+func filterV1DoubleValueFromList(rawFilterValuesUnsplit []string, filterField string) filter.Filter[*kubecost.CloudCost] {
+	result := filter.Or[*kubecost.CloudCost]{
+		Filters: []filter.Filter[*kubecost.CloudCost]{},
+	}
+
+	for _, unsplit := range rawFilterValuesUnsplit {
+		if unsplit != "" {
+			split := strings.Split(unsplit, ":")
+			if len(split) != 2 {
+				log.Warnf("illegal key/value filter (ignoring): %s", unsplit)
+				continue
+			}
+			labelName := strings.TrimSpace(split[0])
+			val := strings.TrimSpace(split[1])
+			val, wildcard := parseWildcardEnd(val)
+
+			subFilter := filter.StringMapProperty[*kubecost.CloudCost]{
+				Field: filterField,
+				// All v1 filters are equality comparisons
+				Op:    filter.StringMapEquals,
+				Key:   labelName,
+				Value: val,
+			}
+
+			if wildcard {
+				subFilter.Op = filter.StringMapStartsWith
+			}
+
+			result.Filters = append(result.Filters, subFilter)
+		}
+	}
+
+	return result
+}

pkg/filter/util/cloudcostaggregate.go

@@ -1,70 +0,0 @@
-package util
-
-import (
-	"strings"
-
-	"github.com/opencost/opencost/pkg/filter"
-	"github.com/opencost/opencost/pkg/kubecost"
-	"github.com/opencost/opencost/pkg/util/mapper"
-)
-
-func parseWildcardEnd(rawFilterValue string) (string, bool) {
-	return strings.TrimSuffix(rawFilterValue, "*"), strings.HasSuffix(rawFilterValue, "*")
-}
-
-func CloudCostAggregateFilterFromParams(pmr mapper.PrimitiveMapReader) filter.Filter[*kubecost.CloudCostAggregate] {
-	filter := filter.And[*kubecost.CloudCostAggregate]{
-		Filters: []filter.Filter[*kubecost.CloudCostAggregate]{},
-	}
-
-	if raw := pmr.GetList("filterBillingIDs", ","); len(raw) > 0 {
-		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostBillingIDProp))
-	}
-
-	if raw := pmr.GetList("filterWorkGroupIDs", ","); len(raw) > 0 {
-		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostWorkGroupIDProp))
-	}
-
-	if raw := pmr.GetList("filterProviders", ","); len(raw) > 0 {
-		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostProviderProp))
-	}
-
-	if raw := pmr.GetList("filterServices", ","); len(raw) > 0 {
-		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostServiceProp))
-	}
-
-	if raw := pmr.GetList("filterLabelValues", ","); len(raw) > 0 {
-		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostLabelProp))
-	}
-
-	if len(filter.Filters) == 0 {
-		return nil
-	}
-
-	return filter
-}
-
-func filterV1SingleValueFromList(rawFilterValues []string, field string) filter.Filter[*kubecost.CloudCostAggregate] {
-	result := filter.Or[*kubecost.CloudCostAggregate]{
-		Filters: []filter.Filter[*kubecost.CloudCostAggregate]{},
-	}
-
-	for _, filterValue := range rawFilterValues {
-		filterValue = strings.TrimSpace(filterValue)
-		filterValue, wildcard := parseWildcardEnd(filterValue)
-
-		subFilter := filter.StringProperty[*kubecost.CloudCostAggregate]{
-			Field: field,
-			Op:    filter.StringEquals,
-			Value: filterValue,
-		}
-
-		if wildcard {
-			subFilter.Op = kubecost.FilterStartsWith
-		}
-
-		result.Filters = append(result.Filters, subFilter)
-	}
-
-	return result
-}

pkg/kubecost/allocation.go

@@ -86,7 +86,7 @@ type Allocation struct {
 	// allocation as a percentage of the per-resource total cost of the
 	// asset on which the allocation was run. It is optionally computed
 	// and appended to an Allocation, and so by default is is nil.
-	ProportionalAssetResourceCosts ProportionalAssetResourceCosts `json:"proportionalAssetResourceCosts"`
+	ProportionalAssetResourceCosts ProportionalAssetResourceCosts `json:"proportionalAssetResourceCosts"` //@bingen:field[ignore]
 }
 
 // RawAllocationOnlyData is information that only belong in "raw" Allocations,

pkg/kubecost/asset.go

@@ -2158,22 +2158,6 @@ func (n *Node) GPUs() float64 {
 	return n.GPUHours * (60.0 / n.Minutes())
 }
 
-func (n *Node) MonitoringKey() string {
-	nodeProps := n.GetProperties()
-	if nodeProps == nil {
-		return ""
-	}
-	//TO-DO: For Alibaba investigate why cloudCost ProviderID doesnt match Kubecost ProviderID via Kubernetes API
-	if nodeProps.Provider == AlibabaProvider {
-		aliProviderID := strings.Split(nodeProps.ProviderID, ".")
-		if len(aliProviderID) != 2 {
-			return ""
-		}
-		return nodeProps.Provider + "/" + aliProviderID[1]
-	}
-	return nodeProps.Provider + "/" + nodeProps.ProviderID
-}
-
 // LoadBalancer is an Asset representing a single load balancer in a cluster
 // TODO: add GB of ingress processed, numForwardingRules once we start recording those to prometheus metric
 type LoadBalancer struct {
@@ -3180,19 +3164,6 @@ func (as *AssetSet) accumulate(that *AssetSet) (*AssetSet, error) {
 	return acc, nil
 }
 
-func (as *AssetSet) MonitoredNodeForCloudCostItem(cci *CloudCostItem) *Node {
-	for _, node := range as.Nodes {
-		if node.MonitoringKey() == cci.MonitoringKey() {
-			props := node.GetProperties()
-			if props == nil {
-				continue
-			}
-			return node
-		}
-	}
-	return nil
-}
-
 type DiffKind string
 
 const (

pkg/kubecost/bingen.go

@@ -73,20 +73,13 @@ package kubecost
 // @bingen:generate:AuditSetRange
 // @bingen:end
 
-// @bingen:set[name=CloudCostAggregate,version=2]
-// @bingen:generate:CloudCostAggregate
-// @bingen:generate[stringtable]:CloudCostAggregateSet
-// @bingen:generate:CloudCostAggregateSetRange
-// @bingen:generate:CloudCostAggregateProperties
-// @bingen:generate:CloudCostAggregateLabels
-// @bingen:end
-
-// @bingen:set[name=CloudCostItem,version=2]
-// @bingen:generate:CloudCostItem
-// @bingen:generate[stringtable]:CloudCostItemSet
-// @bingen:generate:CloudCostItemSetRange
-// @bingen:generate:CloudCostItemProperties
-// @bingen:generate:CloudCostItemLabels
+// @bingen:set[name=CloudCost,version=1]
+// @bingen:generate:CloudCost
+// @bingen:generate:CostMetric
+// @bingen:generate[stringtable]:CloudCostSet
+// @bingen:generate:CloudCostSetRange
+// @bingen:generate:CloudCostProperties
+// @bingen:generate:CloudCostLabels
 // @bingen:end
 
 //go:generate bingen -package=kubecost -version=17 -buffer=github.com/opencost/opencost/pkg/util

pkg/kubecost/cloudcost.go

@@ -0,0 +1,550 @@
+package kubecost
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/opencost/opencost/pkg/filter"
+	"github.com/opencost/opencost/pkg/log"
+)
+
+// CloudCost represents a CUR line item, identifying a cloud resource and
+// its cost over some period of time.
+type CloudCost struct {
+	Properties       *CloudCostProperties `json:"properties"`
+	Window           Window               `json:"window"`
+	ListCost         CostMetric           `json:"listCost"`
+	NetCost          CostMetric           `json:"netCost"`
+	AmortizedNetCost CostMetric           `json:"amortizedNetCost"`
+	InvoicedCost     CostMetric           `json:"invoicedCost"`
+}
+
+// NewCloudCost instantiates a new CloudCost
+func NewCloudCost(start, end time.Time, ccProperties *CloudCostProperties, kubernetesPercent, listCost, netCost, amortizedNetCost, invoicedCost float64) *CloudCost {
+	return &CloudCost{
+		Properties: ccProperties,
+		Window:     NewWindow(&start, &end),
+		ListCost: CostMetric{
+			Cost:              listCost,
+			KubernetesPercent: kubernetesPercent,
+		},
+		NetCost: CostMetric{
+			Cost:              netCost,
+			KubernetesPercent: kubernetesPercent,
+		},
+		AmortizedNetCost: CostMetric{
+			Cost:              amortizedNetCost,
+			KubernetesPercent: kubernetesPercent,
+		},
+		InvoicedCost: CostMetric{
+			Cost:              listCost,
+			KubernetesPercent: kubernetesPercent,
+		},
+	}
+}
+
+func (cc *CloudCost) Clone() *CloudCost {
+	return &CloudCost{
+		Properties:       cc.Properties.Clone(),
+		Window:           cc.Window.Clone(),
+		ListCost:         cc.ListCost.Clone(),
+		NetCost:          cc.NetCost.Clone(),
+		AmortizedNetCost: cc.AmortizedNetCost.Clone(),
+		InvoicedCost:     cc.InvoicedCost.Clone(),
+	}
+}
+
+func (cc *CloudCost) Equal(that *CloudCost) bool {
+	if that == nil {
+		return false
+	}
+
+	return cc.Properties.Equal(that.Properties) &&
+		cc.Window.Equal(that.Window) &&
+		cc.ListCost.Equal(that.ListCost) &&
+		cc.NetCost.Equal(that.NetCost) &&
+		cc.AmortizedNetCost.Equal(that.AmortizedNetCost) &&
+		cc.InvoicedCost.Equal(that.InvoicedCost)
+}
+
+func (cc *CloudCost) add(that *CloudCost) {
+	if cc == nil {
+		log.Warnf("cannot add to nil CloudCost")
+		return
+	}
+
+	// Preserve properties of cloud cost  that are matching between the two CloudCost
+	cc.Properties = cc.Properties.Intersection(that.Properties)
+
+	cc.ListCost = cc.ListCost.add(that.ListCost)
+	cc.NetCost = cc.NetCost.add(that.NetCost)
+	cc.AmortizedNetCost = cc.AmortizedNetCost.add(that.AmortizedNetCost)
+	cc.InvoicedCost = cc.InvoicedCost.add(that.InvoicedCost)
+
+	cc.Window = cc.Window.Expand(that.Window)
+}
+
+func (cc *CloudCost) StringProperty(prop string) (string, error) {
+	if cc == nil {
+		return "", nil
+	}
+
+	switch prop {
+	case CloudCostInvoiceEntityIDProp:
+		return cc.Properties.InvoiceEntityID, nil
+	case CloudCostAccountIDProp:
+		return cc.Properties.AccountID, nil
+	case CloudCostProviderProp:
+		return cc.Properties.Provider, nil
+	case CloudCostProviderIDProp:
+		return cc.Properties.ProviderID, nil
+	case CloudCostServiceProp:
+		return cc.Properties.Service, nil
+	case CloudCostCategoryProp:
+		return cc.Properties.Category, nil
+	default:
+		return "", fmt.Errorf("invalid property name: %s", prop)
+	}
+}
+
+func (cc *CloudCost) StringMapProperty(property string) (map[string]string, error) {
+	switch property {
+	case CloudCostLabelProp:
+		if cc.Properties == nil {
+			return nil, nil
+		}
+		return cc.Properties.Labels, nil
+
+	default:
+		return nil, fmt.Errorf("CloudCost: StringMapProperty: invalid property name: %s", property)
+	}
+}
+
+func (cc *CloudCost) GetCostMetric(costMetricName string) (CostMetric, error) {
+	switch costMetricName {
+	case ListCostMetric:
+		return cc.ListCost, nil
+	case NetCostMetric:
+		return cc.NetCost, nil
+	case AmortizedNetCostMetric:
+		return cc.AmortizedNetCost, nil
+	case InvoicedCostMetric:
+		return cc.InvoicedCost, nil
+	}
+	return CostMetric{}, fmt.Errorf("invalid Cost Metric: %s", costMetricName)
+}
+
+// CloudCostSet follows the established set pattern of windowed data types. It has addition metadata types that can be
+// used to preserve data consistency and be used for validation.
+// - Integration is the ID for the integration that a CloudCostSet was sourced from, this value is cleared if when a
+// set is joined with another with a different key
+// - AggregationProperties is set by the Aggregate function and ensures that any additional inserts are keyed correctly
+type CloudCostSet struct {
+	CloudCosts            map[string]*CloudCost `json:"cloudCosts"`
+	Window                Window                `json:"window"`
+	Integration           string                `json:"-"`
+	AggregationProperties []string              `json:"aggregationProperties"`
+}
+
+// NewCloudCostSet instantiates a new CloudCostSet and, optionally, inserts
+// the given list of CloudCosts
+func NewCloudCostSet(start, end time.Time, cloudCosts ...*CloudCost) *CloudCostSet {
+	ccs := &CloudCostSet{
+		CloudCosts: map[string]*CloudCost{},
+		Window:     NewWindow(&start, &end),
+	}
+
+	for _, cc := range cloudCosts {
+		ccs.Insert(cc)
+	}
+
+	return ccs
+}
+
+func (ccs *CloudCostSet) Aggregate(props []string) (*CloudCostSet, error) {
+	if ccs == nil {
+		return nil, errors.New("cannot aggregate a nil CloudCostSet")
+	}
+
+	if ccs.Window.IsOpen() {
+		return nil, fmt.Errorf("cannot aggregate a CloudCostSet with an open window: %s", ccs.Window)
+	}
+
+	// Create a new result set, with the given aggregation property
+	result := ccs.cloneSet()
+	result.AggregationProperties = props
+
+	// Insert clones of each item in the set, keyed by the given property.
+	// The underlying insert logic will add binned items together.
+	for name, cc := range ccs.CloudCosts {
+		ccClone := cc.Clone()
+		err := result.Insert(ccClone)
+		if err != nil {
+			return nil, fmt.Errorf("error aggregating %s by %v: %s", name, props, err)
+		}
+	}
+
+	return result, nil
+}
+
+func (ccs *CloudCostSet) Accumulate(that *CloudCostSet) (*CloudCostSet, error) {
+	if ccs.IsEmpty() {
+		return that.Clone(), nil
+	}
+	acc := ccs.Clone()
+	err := acc.accumulateInto(that)
+	if err == nil {
+		return nil, err
+	}
+	return acc, nil
+}
+
+// accumulateInto accumulates a the arg CloudCostSet Into the receiver
+func (ccs *CloudCostSet) accumulateInto(that *CloudCostSet) error {
+	if ccs == nil {
+		return fmt.Errorf("CloudCost: cannot accumulate into nil set")
+	}
+
+	if that.IsEmpty() {
+		return nil
+	}
+
+	if ccs.Integration != that.Integration {
+		ccs.Integration = ""
+	}
+
+	ccs.Window.Expand(that.Window)
+
+	for _, cc := range that.CloudCosts {
+		err := ccs.Insert(cc)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (ccs *CloudCostSet) Equal(that *CloudCostSet) bool {
+	if ccs.Integration != that.Integration {
+		return false
+	}
+
+	if !ccs.Window.Equal(that.Window) {
+		return false
+	}
+
+	// Check Aggregation Properties, slice order is grounds for inequality
+	if len(ccs.AggregationProperties) != len(that.AggregationProperties) {
+		return false
+	}
+	for i, prop := range ccs.AggregationProperties {
+		if that.AggregationProperties[i] != prop {
+			return false
+		}
+	}
+
+	if len(ccs.CloudCosts) != len(that.CloudCosts) {
+		return false
+	}
+
+	for k, cc := range ccs.CloudCosts {
+		if tcc, ok := that.CloudCosts[k]; !ok || !cc.Equal(tcc) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (ccs *CloudCostSet) Filter(filters filter.Filter[*CloudCost]) *CloudCostSet {
+	if ccs == nil {
+		return nil
+	}
+
+	if filters == nil {
+		return ccs.Clone()
+	}
+
+	result := ccs.cloneSet()
+
+	for _, cc := range ccs.CloudCosts {
+		if filters.Matches(cc) {
+			result.Insert(cc.Clone())
+		}
+	}
+
+	return result
+}
+
+// Insert adds a CloudCost to a CloudCostSet using its AggregationProperties and LabelConfig
+// to determine the key where it will be inserted
+func (ccs *CloudCostSet) Insert(cc *CloudCost) error {
+	if ccs == nil {
+		return fmt.Errorf("cannot insert into nil CloudCostSet")
+	}
+
+	if cc == nil {
+		return fmt.Errorf("cannot insert nil CloudCost into CloudCostSet")
+	}
+
+	if ccs.CloudCosts == nil {
+		ccs.CloudCosts = map[string]*CloudCost{}
+	}
+
+	ccKey := cc.Properties.GenerateKey(ccs.AggregationProperties)
+
+	// Add the given CloudCost to the existing entry, if there is one;
+	// otherwise just set directly into allocations
+	if _, ok := ccs.CloudCosts[ccKey]; !ok {
+		ccs.CloudCosts[ccKey] = cc.Clone()
+	} else {
+		ccs.CloudCosts[ccKey].add(cc)
+	}
+
+	return nil
+}
+
+func (ccs *CloudCostSet) Clone() *CloudCostSet {
+	cloudCosts := make(map[string]*CloudCost, len(ccs.CloudCosts))
+	for k, v := range ccs.CloudCosts {
+		cloudCosts[k] = v.Clone()
+	}
+
+	cloneCCS := ccs.cloneSet()
+	cloneCCS.CloudCosts = cloudCosts
+
+	return cloneCCS
+}
+
+// cloneSet creates a copy of the receiver without any of its CloudCosts
+func (ccs *CloudCostSet) cloneSet() *CloudCostSet {
+	aggProps := make([]string, len(ccs.AggregationProperties))
+	for i, v := range ccs.AggregationProperties {
+		aggProps[i] = v
+	}
+	return &CloudCostSet{
+		CloudCosts:            make(map[string]*CloudCost),
+		Integration:           ccs.Integration,
+		AggregationProperties: aggProps,
+		Window:                ccs.Window.Clone(),
+	}
+}
+
+func (ccs *CloudCostSet) IsEmpty() bool {
+	if ccs == nil {
+		return true
+	}
+
+	if len(ccs.CloudCosts) == 0 {
+		return true
+	}
+
+	return false
+}
+
+func (ccs *CloudCostSet) Length() int {
+	if ccs == nil {
+		return 0
+	}
+	return len(ccs.CloudCosts)
+}
+
+func (ccs *CloudCostSet) GetWindow() Window {
+	return ccs.Window
+}
+
+func (ccs *CloudCostSet) Merge(that *CloudCostSet) (*CloudCostSet, error) {
+	if ccs == nil {
+		return nil, fmt.Errorf("cannot merge nil CloudCostSets")
+	}
+
+	if that.IsEmpty() {
+		return ccs.Clone(), nil
+	}
+
+	if !ccs.Window.Equal(that.Window) {
+		return nil, fmt.Errorf("cannot merge CloudCostSets with different windows")
+	}
+
+	result := ccs.cloneSet()
+	// clear integration if it is not equal
+	if ccs.Integration != that.Integration {
+		result.Integration = ""
+	}
+
+	for _, cc := range ccs.CloudCosts {
+		result.Insert(cc)
+	}
+
+	for _, cc := range that.CloudCosts {
+		result.Insert(cc)
+	}
+
+	return result, nil
+}
+
+type CloudCostSetRange struct {
+	CloudCostSets []*CloudCostSet `json:"sets"`
+	Window        Window          `json:"window"`
+}
+
+// NewCloudCostSetRange create a CloudCostSetRange containing CloudCostSets with windows of equal duration
+// the duration between start and end must be divisible by the window duration argument
+func NewCloudCostSetRange(start time.Time, end time.Time, window time.Duration, integration string) (*CloudCostSetRange, error) {
+	windows, err := GetWindows(start, end, window)
+	if err != nil {
+		return nil, err
+	}
+
+	// Build slice of CloudCostSet to cover the range
+	cloudCostItemSets := make([]*CloudCostSet, len(windows))
+	for i, w := range windows {
+		ccs := NewCloudCostSet(*w.Start(), *w.End())
+		ccs.Integration = integration
+		cloudCostItemSets[i] = ccs
+	}
+	return &CloudCostSetRange{
+		Window:        NewWindow(&start, &end),
+		CloudCostSets: cloudCostItemSets,
+	}, nil
+}
+
+func (ccsr *CloudCostSetRange) Clone() *CloudCostSetRange {
+	ccsSlice := make([]*CloudCostSet, len(ccsr.CloudCostSets))
+	for i, ccs := range ccsr.CloudCostSets {
+		ccsSlice[i] = ccs.Clone()
+	}
+	return &CloudCostSetRange{
+		Window:        ccsr.Window.Clone(),
+		CloudCostSets: ccsSlice,
+	}
+}
+
+func (ccsr *CloudCostSetRange) IsEmpty() bool {
+	for _, ccs := range ccsr.CloudCostSets {
+		if !ccs.IsEmpty() {
+			return false
+		}
+	}
+	return true
+}
+
+// Accumulate sums each CloudCostSet in the given range, returning a single cumulative
+// CloudCostSet for the entire range.
+func (ccsr *CloudCostSetRange) Accumulate() (*CloudCostSet, error) {
+	var cloudCostSet *CloudCostSet
+	var err error
+
+	for _, ccs := range ccsr.CloudCostSets {
+		if cloudCostSet == nil {
+			cloudCostSet = ccs.Clone()
+			continue
+		}
+		err = cloudCostSet.accumulateInto(ccs)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return cloudCostSet, nil
+}
+
+// LoadCloudCost loads CloudCosts into existing CloudCostSets of the CloudCostSetRange.
+// This function service to aggregate and distribute costs over predefined windows
+// are accumulated here so that the resulting CloudCost with the 1d window has the correct price for the entire day.
+// If all or a portion of the window of the CloudCost is outside of the windows of the existing CloudCostSets,
+// that portion of the CloudCost's cost will not be inserted
+func (ccsr *CloudCostSetRange) LoadCloudCost(cloudCost *CloudCost) {
+	window := cloudCost.Window
+	if window.IsOpen() {
+		log.Errorf("CloudCostSetRange: LoadCloudCost: invalid window %s", window.String())
+		return
+	}
+
+	totalPct := 0.0
+
+	// Distribute cost of the current item across one or more CloudCosts in
+	// across each relevant CloudCostSet. Stop when the end of the current
+	// block reaches the item's end time or the end of the range.
+	for _, ccs := range ccsr.CloudCostSets {
+		setWindow := ccs.Window
+
+		// get percent of item window contained in set window
+		pct := setWindow.GetPercentInWindow(window)
+		if pct == 0 {
+			continue
+		}
+
+		cc := cloudCost
+		// If the current set Window only contains a portion of the CloudCost Window, insert costs relative to that portion
+		if pct < 1.0 {
+			cc = &CloudCost{
+				Properties:       cloudCost.Properties,
+				Window:           window.Contract(setWindow),
+				ListCost:         cloudCost.ListCost.percent(pct),
+				NetCost:          cloudCost.NetCost.percent(pct),
+				AmortizedNetCost: cloudCost.AmortizedNetCost.percent(pct),
+				InvoicedCost:     cloudCost.InvoicedCost.percent(pct),
+			}
+		}
+
+		err := ccs.Insert(cc)
+		if err != nil {
+			log.Errorf("CloudCostSetRange: LoadCloudCost: failed to load CloudCost with window %s: %s", setWindow.String(), err.Error())
+		}
+
+		// If all cost has been inserted, then there is no need to check later days in the range
+		totalPct += pct
+		if totalPct >= 1.0 {
+			return
+		}
+	}
+}
+
+const (
+	ListCostMetric         string = "ListCost"
+	NetCostMetric          string = "NetCost"
+	AmortizedNetCostMetric string = "AmortizedNetCost"
+	InvoicedCostMetric     string = "InvoicedCost"
+)
+
+type CostMetric struct {
+	Cost              float64 `json:"cost"`
+	KubernetesPercent float64 `json:"kubernetesPercent"`
+}
+
+func (cm CostMetric) Equal(that CostMetric) bool {
+	return cm.Cost == that.Cost && cm.KubernetesPercent == that.KubernetesPercent
+}
+
+func (cm CostMetric) Clone() CostMetric {
+	return CostMetric{
+		Cost:              cm.Cost,
+		KubernetesPercent: cm.KubernetesPercent,
+	}
+}
+
+func (cm CostMetric) add(that CostMetric) CostMetric {
+	// Compute KubernetesPercent for sum
+	k8sPct := 0.0
+	sumCost := cm.Cost + that.Cost
+	if sumCost > 0.0 {
+		thisK8sCost := cm.Cost * cm.KubernetesPercent
+		thatK8sCost := that.Cost * that.KubernetesPercent
+		k8sPct = (thisK8sCost + thatK8sCost) / sumCost
+	}
+
+	return CostMetric{
+		Cost:              sumCost,
+		KubernetesPercent: k8sPct,
+	}
+}
+
+// percent returns the product of the given percent and the cost, KubernetesPercent remains the same
+func (cm CostMetric) percent(pct float64) CostMetric {
+	return CostMetric{
+		Cost:              cm.Cost * pct,
+		KubernetesPercent: cm.KubernetesPercent,
+	}
+}

pkg/kubecost/cloudcost_test.go

@@ -0,0 +1,270 @@
+package kubecost
+
+import (
+	"testing"
+	"time"
+
+	"github.com/opencost/opencost/pkg/util/timeutil"
+)
+
+var ccProperties1 = &CloudCostProperties{
+	ProviderID:      "providerid1",
+	Provider:        "provider1",
+	AccountID:       "workgroup1",
+	InvoiceEntityID: "billing1",
+	Service:         "service1",
+	Category:        "category1",
+	Labels: map[string]string{
+		"label1": "value1",
+		"label2": "value2",
+	},
+}
+
+// TestCloudCost_LoadCloudCost checks that loaded CloudCosts end up in the correct set in the
+// correct proportions
+func TestCloudCost_LoadCloudCost(t *testing.T) {
+	cc1Key := ccProperties1.GenerateKey(nil)
+	// create values for 3 day Range tests
+	end := RoundBack(time.Now().UTC(), timeutil.Day)
+	start := end.Add(-3 * timeutil.Day)
+	dayWindows, _ := GetWindows(start, end, timeutil.Day)
+	emtpyCCSR, _ := NewCloudCostSetRange(start, end, timeutil.Day, "integration")
+	testCases := map[string]struct {
+		cc       []*CloudCost
+		ccsr     *CloudCostSetRange
+		expected []*CloudCostSet
+	}{
+		"Load Single Day On Grid": {
+			cc: []*CloudCost{
+				{
+					Properties:       ccProperties1,
+					Window:           dayWindows[0],
+					ListCost:         CostMetric{Cost: 100, KubernetesPercent: 1},
+					NetCost:          CostMetric{Cost: 80, KubernetesPercent: 1},
+					AmortizedNetCost: CostMetric{Cost: 90, KubernetesPercent: 1},
+					InvoicedCost:     CostMetric{Cost: 95, KubernetesPercent: 1},
+				},
+			},
+			ccsr: emtpyCCSR.Clone(),
+			expected: []*CloudCostSet{
+				{
+					Integration: "integration",
+					Window:      dayWindows[0],
+					CloudCosts: map[string]*CloudCost{
+						cc1Key: {
+							Properties:       ccProperties1,
+							Window:           dayWindows[0],
+							ListCost:         CostMetric{Cost: 100, KubernetesPercent: 1},
+							NetCost:          CostMetric{Cost: 80, KubernetesPercent: 1},
+							AmortizedNetCost: CostMetric{Cost: 90, KubernetesPercent: 1},
+							InvoicedCost:     CostMetric{Cost: 95, KubernetesPercent: 1},
+						},
+					},
+				},
+				{
+					Integration: "integration",
+					Window:      dayWindows[1],
+					CloudCosts:  map[string]*CloudCost{},
+				},
+				{
+					Integration: "integration",
+					Window:      dayWindows[2],
+					CloudCosts:  map[string]*CloudCost{},
+				},
+			},
+		},
+		"Load Single Day Off Grid": {
+			cc: []*CloudCost{
+				{
+					Properties:       ccProperties1,
+					Window:           NewClosedWindow(start.Add(12*time.Hour), start.Add(36*time.Hour)),
+					ListCost:         CostMetric{Cost: 100, KubernetesPercent: 1},
+					NetCost:          CostMetric{Cost: 80, KubernetesPercent: 1},
+					AmortizedNetCost: CostMetric{Cost: 90, KubernetesPercent: 1},
+					InvoicedCost:     CostMetric{Cost: 95, KubernetesPercent: 1},
+				},
+			},
+			ccsr: emtpyCCSR.Clone(),
+			expected: []*CloudCostSet{
+				{
+					Integration: "integration",
+					Window:      dayWindows[0],
+					CloudCosts: map[string]*CloudCost{
+						cc1Key: {
+							Properties:       ccProperties1,
+							Window:           NewClosedWindow(start.Add(12*time.Hour), start.Add(24*time.Hour)),
+							ListCost:         CostMetric{Cost: 50, KubernetesPercent: 1},
+							NetCost:          CostMetric{Cost: 40, KubernetesPercent: 1},
+							AmortizedNetCost: CostMetric{Cost: 45, KubernetesPercent: 1},
+							InvoicedCost:     CostMetric{Cost: 47.5, KubernetesPercent: 1},
+						},
+					},
+				},
+				{
+					Integration: "integration",
+					Window:      dayWindows[1],
+					CloudCosts: map[string]*CloudCost{
+						cc1Key: {
+							Properties:       ccProperties1,
+							Window:           NewClosedWindow(start.Add(24*time.Hour), start.Add(36*time.Hour)),
+							ListCost:         CostMetric{Cost: 50, KubernetesPercent: 1},
+							NetCost:          CostMetric{Cost: 40, KubernetesPercent: 1},
+							AmortizedNetCost: CostMetric{Cost: 45, KubernetesPercent: 1},
+							InvoicedCost:     CostMetric{Cost: 47.5, KubernetesPercent: 1},
+						},
+					},
+				},
+				{
+					Integration: "integration",
+					Window:      dayWindows[2],
+					CloudCosts:  map[string]*CloudCost{},
+				},
+			},
+		},
+		"Load Single Day Off Grid Before Range Window": {
+			cc: []*CloudCost{
+				{
+					Properties:       ccProperties1,
+					Window:           NewClosedWindow(start.Add(-12*time.Hour), start.Add(12*time.Hour)),
+					ListCost:         CostMetric{Cost: 100, KubernetesPercent: 1},
+					NetCost:          CostMetric{Cost: 80, KubernetesPercent: 1},
+					AmortizedNetCost: CostMetric{Cost: 90, KubernetesPercent: 1},
+					InvoicedCost:     CostMetric{Cost: 95, KubernetesPercent: 1},
+				},
+			},
+			ccsr: emtpyCCSR.Clone(),
+			expected: []*CloudCostSet{
+				{
+					Integration: "integration",
+					Window:      dayWindows[0],
+					CloudCosts: map[string]*CloudCost{
+						cc1Key: {
+							Properties:       ccProperties1,
+							Window:           NewClosedWindow(start, start.Add(12*time.Hour)),
+							ListCost:         CostMetric{Cost: 50, KubernetesPercent: 1},
+							NetCost:          CostMetric{Cost: 40, KubernetesPercent: 1},
+							AmortizedNetCost: CostMetric{Cost: 45, KubernetesPercent: 1},
+							InvoicedCost:     CostMetric{Cost: 47.5, KubernetesPercent: 1},
+						},
+					},
+				},
+				{
+					Integration: "integration",
+					Window:      dayWindows[1],
+					CloudCosts:  map[string]*CloudCost{},
+				},
+				{
+					Integration: "integration",
+					Window:      dayWindows[2],
+					CloudCosts:  map[string]*CloudCost{},
+				},
+			},
+		},
+		"Load Single Day Off Grid After Range Window": {
+			cc: []*CloudCost{
+				{
+					Properties:       ccProperties1,
+					Window:           NewClosedWindow(end.Add(-12*time.Hour), end.Add(12*time.Hour)),
+					ListCost:         CostMetric{Cost: 100, KubernetesPercent: 1},
+					NetCost:          CostMetric{Cost: 80, KubernetesPercent: 1},
+					AmortizedNetCost: CostMetric{Cost: 90, KubernetesPercent: 1},
+					InvoicedCost:     CostMetric{Cost: 95, KubernetesPercent: 1},
+				},
+			},
+			ccsr: emtpyCCSR.Clone(),
+			expected: []*CloudCostSet{
+				{
+					Integration: "integration",
+					Window:      dayWindows[0],
+					CloudCosts:  map[string]*CloudCost{},
+				},
+				{
+					Integration: "integration",
+					Window:      dayWindows[1],
+					CloudCosts:  map[string]*CloudCost{},
+				},
+				{
+					Integration: "integration",
+					Window:      dayWindows[2],
+					CloudCosts: map[string]*CloudCost{
+						cc1Key: {
+							Properties:       ccProperties1,
+							Window:           NewClosedWindow(end.Add(-12*time.Hour), end),
+							ListCost:         CostMetric{Cost: 50, KubernetesPercent: 1},
+							NetCost:          CostMetric{Cost: 40, KubernetesPercent: 1},
+							AmortizedNetCost: CostMetric{Cost: 45, KubernetesPercent: 1},
+							InvoicedCost:     CostMetric{Cost: 47.5, KubernetesPercent: 1},
+						},
+					},
+				},
+			},
+		},
+		"Single Day Kubernetes Percent": {
+			cc: []*CloudCost{
+				{
+					Properties:       ccProperties1,
+					Window:           dayWindows[1],
+					ListCost:         CostMetric{Cost: 75, KubernetesPercent: 1},
+					NetCost:          CostMetric{Cost: 40, KubernetesPercent: 1},
+					AmortizedNetCost: CostMetric{Cost: 60, KubernetesPercent: 1},
+					InvoicedCost:     CostMetric{Cost: 50, KubernetesPercent: 1},
+				},
+				{
+					Properties:       ccProperties1,
+					Window:           dayWindows[1],
+					ListCost:         CostMetric{Cost: 25, KubernetesPercent: 0},
+					NetCost:          CostMetric{Cost: 60, KubernetesPercent: 0},
+					AmortizedNetCost: CostMetric{Cost: 40, KubernetesPercent: 0},
+					InvoicedCost:     CostMetric{Cost: 50, KubernetesPercent: 0},
+				},
+			},
+			ccsr: emtpyCCSR.Clone(),
+			expected: []*CloudCostSet{
+				{
+					Integration: "integration",
+					Window:      dayWindows[0],
+					CloudCosts:  map[string]*CloudCost{},
+				},
+				{
+					Integration: "integration",
+					Window:      dayWindows[1],
+					CloudCosts: map[string]*CloudCost{
+						cc1Key: {
+							Properties:       ccProperties1,
+							Window:           dayWindows[1],
+							ListCost:         CostMetric{Cost: 100, KubernetesPercent: 0.75},
+							NetCost:          CostMetric{Cost: 100, KubernetesPercent: 0.4},
+							AmortizedNetCost: CostMetric{Cost: 100, KubernetesPercent: 0.6},
+							InvoicedCost:     CostMetric{Cost: 100, KubernetesPercent: 0.5},
+						},
+					},
+				},
+				{
+					Integration: "integration",
+					Window:      dayWindows[2],
+					CloudCosts:  map[string]*CloudCost{},
+				},
+			},
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			// load Cloud Costs
+			for _, cc := range tc.cc {
+				tc.ccsr.LoadCloudCost(cc)
+			}
+
+			if len(tc.ccsr.CloudCostSets) != len(tc.expected) {
+				t.Errorf("the CloudCostSetRanges did not have the expected length")
+			}
+
+			for i, ccs := range tc.ccsr.CloudCostSets {
+				if !ccs.Equal(tc.expected[i]) {
+					t.Errorf("CloudCostSet at index: %d did not match expected", i)
+				}
+			}
+		})
+	}
+
+}

pkg/kubecost/cloudcostaggregate.go

@@ -1,504 +0,0 @@
-package kubecost
-
-import (
-	"errors"
-	"fmt"
-	"strings"
-	"time"
-
-	"github.com/opencost/opencost/pkg/filter"
-	"github.com/opencost/opencost/pkg/log"
-)
-
-const (
-	CloudCostBillingIDProp   string = "billingID"
-	CloudCostWorkGroupIDProp string = "workGroupID"
-	CloudCostProviderProp    string = "provider"
-	CloudCostServiceProp     string = "service"
-	CloudCostLabelProp       string = "label"
-)
-
-// CloudCostAggregateProperties unique property set for CloudCostAggregate within a window
-type CloudCostAggregateProperties struct {
-	Provider    string `json:"provider"`
-	WorkGroupID string `json:"workGroupID"`
-	BillingID   string `json:"billingID"`
-	Service     string `json:"service"`
-	LabelValue  string `json:"label"`
-}
-
-func (ccap CloudCostAggregateProperties) Equal(that CloudCostAggregateProperties) bool {
-	return ccap.Provider == that.Provider &&
-		ccap.WorkGroupID == that.WorkGroupID &&
-		ccap.BillingID == that.BillingID &&
-		ccap.Service == that.Service &&
-		ccap.LabelValue == that.LabelValue
-}
-
-// Intersection ensure the values of two CloudCostAggregateProperties are maintain only if they are equal
-func (ccap CloudCostAggregateProperties) Intersection(that CloudCostAggregateProperties) CloudCostAggregateProperties {
-	if ccap.Equal(that) {
-		return ccap
-	}
-	intersectionCCAP := CloudCostAggregateProperties{}
-	if ccap == intersectionCCAP || that == intersectionCCAP {
-		return intersectionCCAP
-	}
-
-	if ccap.Provider == that.Provider {
-		intersectionCCAP.Provider = ccap.Provider
-	}
-	if ccap.WorkGroupID == that.WorkGroupID {
-		intersectionCCAP.WorkGroupID = ccap.WorkGroupID
-	}
-	if ccap.BillingID == that.BillingID {
-		intersectionCCAP.BillingID = ccap.BillingID
-	}
-	if ccap.Service == that.Service {
-		intersectionCCAP.Service = ccap.Service
-	}
-	if ccap.LabelValue == that.LabelValue {
-		intersectionCCAP.LabelValue = ccap.LabelValue
-	}
-	return intersectionCCAP
-}
-func (ccap CloudCostAggregateProperties) Key(props []string) string {
-	if len(props) == 0 {
-		return fmt.Sprintf("%s/%s/%s/%s/%s", ccap.Provider, ccap.BillingID, ccap.WorkGroupID, ccap.Service, ccap.LabelValue)
-	}
-
-	keys := make([]string, len(props))
-	for i, prop := range props {
-		key := UnallocatedSuffix
-
-		switch prop {
-		case CloudCostProviderProp:
-			if ccap.Provider != "" {
-				key = ccap.Provider
-			}
-		case CloudCostBillingIDProp:
-			if ccap.BillingID != "" {
-				key = ccap.BillingID
-			}
-		case CloudCostWorkGroupIDProp:
-			if ccap.WorkGroupID != "" {
-				key = ccap.WorkGroupID
-			}
-		case CloudCostServiceProp:
-			if ccap.Service != "" {
-				key = ccap.Service
-			}
-		case CloudCostLabelProp:
-			if ccap.LabelValue != "" {
-				key = ccap.LabelValue
-			}
-		}
-
-		keys[i] = key
-	}
-
-	return strings.Join(keys, "/")
-}
-
-// CloudCostAggregate represents an aggregation of Billing Integration data on the properties listed
-// - KubernetesPercent is the percent of the CloudCostAggregates cost which was from an item which could be identified
-//   as coming from a kubernetes resources.
-// - Cost is the sum of the cost of each item in the CloudCostAggregate
-// - Credit is the sum of credits applied to each item in the CloudCostAggregate
-
-type CloudCostAggregate struct {
-	Properties        CloudCostAggregateProperties `json:"properties"`
-	KubernetesPercent float64                      `json:"kubernetesPercent"`
-	Cost              float64                      `json:"cost"`
-	NetCost           float64                      `json:"netCost"`
-}
-
-func NewCloudCostAggregate(properties CloudCostAggregateProperties, kubernetesPercent, cost, netCost float64) *CloudCostAggregate {
-	return &CloudCostAggregate{
-		Properties:        properties,
-		KubernetesPercent: kubernetesPercent,
-		Cost:              cost,
-		NetCost:           netCost,
-	}
-}
-
-func (cca *CloudCostAggregate) Clone() *CloudCostAggregate {
-	return &CloudCostAggregate{
-		Properties:        cca.Properties,
-		KubernetesPercent: cca.KubernetesPercent,
-		Cost:              cca.Cost,
-		NetCost:           cca.NetCost,
-	}
-}
-
-func (cca *CloudCostAggregate) Equal(that *CloudCostAggregate) bool {
-	if that == nil {
-		return false
-	}
-
-	return cca.Cost == that.Cost &&
-		cca.NetCost == that.NetCost &&
-		cca.Properties.Equal(that.Properties)
-}
-
-func (cca *CloudCostAggregate) Key(props []string) string {
-	return cca.Properties.Key(props)
-}
-
-func (cca *CloudCostAggregate) StringProperty(prop string) (string, error) {
-	if cca == nil {
-		return "", nil
-	}
-
-	switch prop {
-	case CloudCostBillingIDProp:
-		return cca.Properties.BillingID, nil
-	case CloudCostWorkGroupIDProp:
-		return cca.Properties.WorkGroupID, nil
-	case CloudCostProviderProp:
-		return cca.Properties.Provider, nil
-	case CloudCostServiceProp:
-		return cca.Properties.Service, nil
-	case CloudCostLabelProp:
-		return cca.Properties.LabelValue, nil
-	default:
-		return "", fmt.Errorf("invalid property name: %s", prop)
-	}
-}
-
-func (cca *CloudCostAggregate) add(that *CloudCostAggregate) {
-	if cca == nil {
-		log.Warnf("cannot add to nil CloudCostAggregate")
-		return
-	}
-
-	// Preserve string properties of cloud cost aggregates that are matching between the two CloudCostAggregate
-	cca.Properties = cca.Properties.Intersection(that.Properties)
-
-	// Compute KubernetesPercent for sum
-	k8sPct := 0.0
-	sumCost := cca.Cost + that.Cost
-	if sumCost > 0.0 {
-		thisK8sCost := cca.Cost * cca.KubernetesPercent
-		thatK8sCost := that.Cost * that.KubernetesPercent
-		k8sPct = (thisK8sCost + thatK8sCost) / sumCost
-	}
-
-	cca.Cost = sumCost
-	cca.NetCost += that.NetCost
-	cca.KubernetesPercent = k8sPct
-}
-
-type CloudCostAggregateSet struct {
-	CloudCostAggregates   map[string]*CloudCostAggregate `json:"aggregates"`
-	AggregationProperties []string                       `json:"-"`
-	Integration           string                         `json:"-"`
-	LabelName             string                         `json:"labelName,omitempty"`
-	Window                Window                         `json:"window"`
-}
-
-func NewCloudCostAggregateSet(start, end time.Time, cloudCostAggregates ...*CloudCostAggregate) *CloudCostAggregateSet {
-	ccas := &CloudCostAggregateSet{
-		CloudCostAggregates: map[string]*CloudCostAggregate{},
-		Window:              NewWindow(&start, &end),
-	}
-
-	for _, cca := range cloudCostAggregates {
-		ccas.insertByProperty(cca, nil)
-	}
-
-	return ccas
-}
-
-func (ccas *CloudCostAggregateSet) Aggregate(props []string) (*CloudCostAggregateSet, error) {
-	if ccas == nil {
-		return nil, errors.New("cannot aggregate a nil CloudCostAggregateSet")
-	}
-
-	if ccas.Window.IsOpen() {
-		return nil, fmt.Errorf("cannot aggregate a CloudCostAggregateSet with an open window: %s", ccas.Window)
-	}
-
-	// Create a new result set, with the given aggregation property
-	result := NewCloudCostAggregateSet(*ccas.Window.Start(), *ccas.Window.End())
-	result.AggregationProperties = props
-	result.LabelName = ccas.LabelName
-	result.Integration = ccas.Integration
-
-	// Insert clones of each item in the set, keyed by the given property.
-	// The underlying insert logic will add binned items together.
-	for name, cca := range ccas.CloudCostAggregates {
-		ccaClone := cca.Clone()
-		err := result.insertByProperty(ccaClone, props)
-		if err != nil {
-			return nil, fmt.Errorf("error aggregating %s by %v: %s", name, props, err)
-		}
-	}
-
-	return result, nil
-}
-
-func (ccas *CloudCostAggregateSet) Filter(filters filter.Filter[*CloudCostAggregate]) *CloudCostAggregateSet {
-	if ccas == nil {
-		return nil
-	}
-
-	result := ccas.Clone()
-	result.filter(filters)
-
-	return result
-}
-
-func (ccas *CloudCostAggregateSet) filter(filters filter.Filter[*CloudCostAggregate]) {
-	if ccas == nil {
-		return
-	}
-
-	if filters == nil {
-		return
-	}
-
-	for name, cca := range ccas.CloudCostAggregates {
-		if !filters.Matches(cca) {
-			delete(ccas.CloudCostAggregates, name)
-		}
-	}
-}
-
-func (ccas *CloudCostAggregateSet) Insert(that *CloudCostAggregate) error {
-	// Publicly, only allow Inserting as a basic operation (i.e. without causing
-	// an aggregation on a property).
-	return ccas.insertByProperty(that, nil)
-}
-
-func (ccas *CloudCostAggregateSet) insertByProperty(that *CloudCostAggregate, props []string) error {
-	if ccas == nil {
-		return fmt.Errorf("cannot insert into nil CloudCostAggregateSet")
-	}
-
-	if ccas.CloudCostAggregates == nil {
-		ccas.CloudCostAggregates = map[string]*CloudCostAggregate{}
-	}
-
-	// Add the given CloudCostAggregate to the existing entry, if there is one;
-	// otherwise just set directly into allocations
-	if _, ok := ccas.CloudCostAggregates[that.Key(props)]; !ok {
-		ccas.CloudCostAggregates[that.Key(props)] = that
-	} else {
-		ccas.CloudCostAggregates[that.Key(props)].add(that)
-	}
-
-	return nil
-}
-
-func (ccas *CloudCostAggregateSet) Clone() *CloudCostAggregateSet {
-	aggs := make(map[string]*CloudCostAggregate, len(ccas.CloudCostAggregates))
-	for k, v := range ccas.CloudCostAggregates {
-		aggs[k] = v.Clone()
-	}
-
-	return &CloudCostAggregateSet{
-		CloudCostAggregates: aggs,
-		Integration:         ccas.Integration,
-		LabelName:           ccas.LabelName,
-		Window:              ccas.Window.Clone(),
-	}
-}
-
-func (ccas *CloudCostAggregateSet) Equal(that *CloudCostAggregateSet) bool {
-	if ccas.Integration != that.Integration {
-		return false
-	}
-
-	if ccas.LabelName != that.LabelName {
-		return false
-	}
-
-	if !ccas.Window.Equal(that.Window) {
-		return false
-	}
-
-	if len(ccas.CloudCostAggregates) != len(that.CloudCostAggregates) {
-		return false
-	}
-
-	for k, cca := range ccas.CloudCostAggregates {
-		tcca, ok := that.CloudCostAggregates[k]
-		if !ok {
-			return false
-		}
-		if !cca.Equal(tcca) {
-			return false
-		}
-	}
-
-	return true
-}
-
-func (ccas *CloudCostAggregateSet) IsEmpty() bool {
-	if ccas == nil {
-		return true
-	}
-
-	if len(ccas.CloudCostAggregates) == 0 {
-		return true
-	}
-
-	return false
-}
-
-func (ccas *CloudCostAggregateSet) Length() int {
-	if ccas == nil {
-		return 0
-	}
-	return len(ccas.CloudCostAggregates)
-}
-
-func (ccas *CloudCostAggregateSet) GetWindow() Window {
-	return ccas.Window
-}
-
-func (ccas *CloudCostAggregateSet) Merge(that *CloudCostAggregateSet) (*CloudCostAggregateSet, error) {
-	if ccas == nil || that == nil {
-		return nil, fmt.Errorf("cannot merge nil CloudCostAggregateSets")
-	}
-
-	if that.IsEmpty() {
-		return ccas.Clone(), nil
-	}
-
-	if !ccas.Window.Equal(that.Window) {
-		return nil, fmt.Errorf("cannot merge CloudCostAggregateSets with different windows")
-	}
-
-	if ccas.LabelName != that.LabelName {
-		return nil, fmt.Errorf("cannot merge CloudCostAggregateSets with different label names: '%s' != '%s'", ccas.LabelName, that.LabelName)
-	}
-
-	start, end := *ccas.Window.Start(), *ccas.Window.End()
-	result := NewCloudCostAggregateSet(start, end)
-	result.LabelName = ccas.LabelName
-
-	for _, cca := range ccas.CloudCostAggregates {
-		result.insertByProperty(cca, nil)
-	}
-
-	for _, cca := range that.CloudCostAggregates {
-		result.insertByProperty(cca, nil)
-	}
-
-	return result, nil
-}
-
-type CloudCostAggregateSetRange struct {
-	CloudCostAggregateSets []*CloudCostAggregateSet `json:"sets"`
-	Window                 Window                   `json:"window"`
-}
-
-// NewCloudCostAggregateSetRange create a CloudCostAggregateSetRange containing CloudCostItemSets with windows of equal duration
-// the duration between start and end must be divisible by the window duration argument
-func NewCloudCostAggregateSetRange(start, end time.Time, window time.Duration, integration string, labelName string) (*CloudCostAggregateSetRange, error) {
-	windows, err := GetWindows(start, end, window)
-	if err != nil {
-		return nil, err
-	}
-
-	// Build slice of CloudCostAggregateSet to cover the range
-	cloudCostAggregateSets := make([]*CloudCostAggregateSet, len(windows))
-	for i, w := range windows {
-		ccas := NewCloudCostAggregateSet(*w.Start(), *w.End())
-		ccas.Integration = integration
-		ccas.LabelName = labelName
-		cloudCostAggregateSets[i] = ccas
-	}
-	return &CloudCostAggregateSetRange{
-		Window:                 NewWindow(&start, &end),
-		CloudCostAggregateSets: cloudCostAggregateSets,
-	}, nil
-}
-
-// LoadCloudCostAggregate loads CloudCostAggregates into existing CloudCostAggregateSets of the CloudCostAggregateSetRange.
-// This function service to aggregate and distribute costs over predefined windows
-// If all or a portion of the window of the CloudCostAggregate is outside of the windows of the existing CloudCostAggregateSets,
-// that portion of the CloudCostAggregate's cost will not be inserted
-func (ccasr *CloudCostAggregateSetRange) LoadCloudCostAggregate(window Window, cloudCostAggregate *CloudCostAggregate) {
-	if window.IsOpen() {
-		log.Errorf("CloudCostItemSetRange: LoadCloudCostItem: invalid window %s", window.String())
-		return
-	}
-
-	totalPct := 0.0
-
-	// Distribute cost of the current item across one or more CloudCostAggregates in
-	// across each relevant CloudCostAggregateSet. Stop when the end of the current
-	// block reaches the item's end time or the end of the range.
-	for _, ccas := range ccasr.CloudCostAggregateSets {
-		pct := ccas.GetWindow().GetPercentInWindow(window)
-		if pct == 0 {
-			continue
-		}
-		cca := cloudCostAggregate
-		// If the current set Window only contains a portion of the CloudCostItem Window, insert costs relative to that portion
-		if pct < 1.0 {
-			cca = &CloudCostAggregate{
-				Properties:        cloudCostAggregate.Properties,
-				KubernetesPercent: cloudCostAggregate.KubernetesPercent * pct,
-				Cost:              cloudCostAggregate.Cost * pct,
-				NetCost:           cloudCostAggregate.NetCost * pct,
-			}
-		}
-		err := ccas.insertByProperty(cca, nil)
-		if err != nil {
-			log.Errorf("LoadCloudCostAggregateSets: failed to load CloudCostAggregate with key %s and window %s", cca.Key(nil), ccas.GetWindow().String())
-		}
-
-		// If all cost has been inserted then finish
-		totalPct += pct
-		if totalPct >= 1.0 {
-			return
-		}
-	}
-}
-
-func (ccasr *CloudCostAggregateSetRange) Clone() *CloudCostAggregateSetRange {
-	ccasSlice := make([]*CloudCostAggregateSet, len(ccasr.CloudCostAggregateSets))
-	for i, ccas := range ccasr.CloudCostAggregateSets {
-		ccasSlice[i] = ccas.Clone()
-	}
-	return &CloudCostAggregateSetRange{
-		Window:                 ccasr.Window.Clone(),
-		CloudCostAggregateSets: ccasSlice,
-	}
-}
-
-func (ccasr *CloudCostAggregateSetRange) IsEmpty() bool {
-	for _, ccas := range ccasr.CloudCostAggregateSets {
-		if !ccas.IsEmpty() {
-			return false
-		}
-	}
-	return true
-}
-
-func (ccasr *CloudCostAggregateSetRange) Accumulate() (*CloudCostAggregateSet, error) {
-	if ccasr == nil {
-		return nil, errors.New("cannot accumulate a nil CloudCostAggregateSetRange")
-	}
-
-	if ccasr.Window.IsOpen() {
-		return nil, fmt.Errorf("cannot accumulate a CloudCostAggregateSetRange with an open window: %s", ccasr.Window)
-	}
-
-	result := NewCloudCostAggregateSet(*ccasr.Window.Start(), *ccasr.Window.End())
-
-	for _, ccas := range ccasr.CloudCostAggregateSets {
-		for name, cca := range ccas.CloudCostAggregates {
-			err := result.insertByProperty(cca.Clone(), ccas.AggregationProperties)
-			if err != nil {
-				return nil, fmt.Errorf("error accumulating CloudCostAggregateSetRange[%s][%s]: %s", ccas.Window.String(), name, err)
-			}
-		}
-	}
-
-	return result, nil
-}

pkg/kubecost/cloudcostaggregate_test.go

@@ -1,370 +0,0 @@
-package kubecost
-
-import (
-	"testing"
-	"time"
-
-	"github.com/opencost/opencost/pkg/util/timeutil"
-)
-
-var ccaProperties1 = CloudCostAggregateProperties{
-	Provider:    "provider1",
-	WorkGroupID: "workgroup1",
-	BillingID:   "billing1",
-	Service:     "service1",
-	LabelValue:  "labelValue1",
-}
-
-func TestCloudCostAggregatePropertiesIntersection(t *testing.T) {
-	testCases := map[string]struct {
-		baseCCAP     CloudCostAggregateProperties
-		intCCAP      CloudCostAggregateProperties
-		expectedCCAP CloudCostAggregateProperties
-	}{
-		"When properties match between both CloudCostAggregateProperties": {
-			baseCCAP: CloudCostAggregateProperties{
-				Provider:    "CustomProvider",
-				WorkGroupID: "WorkGroupID1",
-				BillingID:   "BillingID1",
-				Service:     "Service1",
-				LabelValue:  "Label1",
-			},
-			intCCAP: CloudCostAggregateProperties{
-				Provider:    "CustomProvider",
-				WorkGroupID: "WorkGroupID1",
-				BillingID:   "BillingID1",
-				Service:     "Service1",
-				LabelValue:  "Label1",
-			},
-			expectedCCAP: CloudCostAggregateProperties{
-				Provider:    "CustomProvider",
-				WorkGroupID: "WorkGroupID1",
-				BillingID:   "BillingID1",
-				Service:     "Service1",
-				LabelValue:  "Label1",
-			},
-		},
-		"When one of the properties differ in the two CloudCostAggregateProperties": {
-			baseCCAP: CloudCostAggregateProperties{
-				Provider:    "CustomProvider",
-				WorkGroupID: "WorkGroupID1",
-				BillingID:   "BillingID1",
-				Service:     "Service1",
-				LabelValue:  "Label1",
-			},
-			intCCAP: CloudCostAggregateProperties{
-				Provider:    "CustomProvider",
-				WorkGroupID: "WorkGroupID1",
-				BillingID:   "BillingID1",
-				Service:     "Service2",
-				LabelValue:  "Label1",
-			},
-			expectedCCAP: CloudCostAggregateProperties{
-				Provider:    "CustomProvider",
-				WorkGroupID: "WorkGroupID1",
-				BillingID:   "BillingID1",
-				Service:     "",
-				LabelValue:  "Label1",
-			},
-		},
-		"When two of the properties differ in the two CloudCostAggregateProperties": {
-			baseCCAP: CloudCostAggregateProperties{
-				Provider:    "CustomProvider",
-				WorkGroupID: "WorkGroupID1",
-				BillingID:   "BillingID1",
-				Service:     "Service1",
-				LabelValue:  "Label1",
-			},
-			intCCAP: CloudCostAggregateProperties{
-				Provider:    "CustomProvider",
-				WorkGroupID: "WorkGroupID2",
-				BillingID:   "BillingID1",
-				Service:     "Service2",
-				LabelValue:  "Label1",
-			},
-			expectedCCAP: CloudCostAggregateProperties{
-				Provider:    "CustomProvider",
-				WorkGroupID: "",
-				BillingID:   "BillingID1",
-				Service:     "",
-				LabelValue:  "Label1",
-			},
-		},
-	}
-	for name, tc := range testCases {
-		t.Run(name, func(t *testing.T) {
-			actualCCAP := tc.baseCCAP.Intersection(tc.intCCAP)
-			if actualCCAP.Provider != tc.expectedCCAP.Provider {
-				t.Errorf("Case %s: Provider properties dont match with expected CloudCostAggregateProperties: %v actual %v", name, tc.expectedCCAP, actualCCAP)
-			}
-			if actualCCAP.WorkGroupID != tc.expectedCCAP.WorkGroupID {
-				t.Errorf("Case %s: WorkGroupID properties dont match with expected CloudCostAggregateProperties: %v actual %v", name, tc.expectedCCAP, actualCCAP)
-			}
-			if actualCCAP.BillingID != tc.expectedCCAP.BillingID {
-				t.Errorf("Case %s: BillingID properties dont match with expected CloudCostAggregateProperties: %v actual %v", name, tc.expectedCCAP, actualCCAP)
-			}
-			if actualCCAP.Service != tc.expectedCCAP.Service {
-				t.Errorf("Case %s: Service properties dont match with expected CloudCostAggregateProperties: %v actual %v", name, tc.expectedCCAP, actualCCAP)
-			}
-			if actualCCAP.LabelValue != tc.expectedCCAP.LabelValue {
-				t.Errorf("Case %s: LabelValue properties dont match with expected CloudCostAggregateProperties: %v actual %v", name, tc.expectedCCAP, actualCCAP)
-			}
-		})
-	}
-}
-
-// TestCloudCostAggregate_LoadCloudCostAggregate checks that loaded CloudCostAggregates end up in the correct set in the
-// correct proportions
-func TestCloudCostAggregate_LoadCloudCostAggregate(t *testing.T) {
-	// create values for 3 day Range tests
-	end := RoundBack(time.Now().UTC(), timeutil.Day)
-	start := end.Add(-3 * timeutil.Day)
-	dayWindows, _ := GetWindows(start, end, timeutil.Day)
-	emtpyCASSR, _ := NewCloudCostAggregateSetRange(start, end, timeutil.Day, "integration", "label")
-	testCases := map[string]struct {
-		cca      []*CloudCostAggregate
-		windows  []Window
-		ccasr    *CloudCostAggregateSetRange
-		expected []*CloudCostAggregateSet
-	}{
-		"Load Single Day On Grid": {
-			cca: []*CloudCostAggregate{
-				{
-					Properties:        ccaProperties1,
-					KubernetesPercent: 1,
-					Cost:              100,
-					NetCost:           80,
-				},
-			},
-			windows: []Window{
-				dayWindows[0],
-			},
-			ccasr: emtpyCASSR.Clone(),
-			expected: []*CloudCostAggregateSet{
-				{
-					Integration: "integration",
-					LabelName:   "label",
-					Window:      dayWindows[0],
-					CloudCostAggregates: map[string]*CloudCostAggregate{
-						ccaProperties1.Key(nil): {
-							Properties:        ccaProperties1,
-							KubernetesPercent: 1,
-							Cost:              100,
-							NetCost:           80,
-						},
-					},
-				},
-				{
-					Integration:         "integration",
-					LabelName:           "label",
-					Window:              dayWindows[1],
-					CloudCostAggregates: map[string]*CloudCostAggregate{},
-				},
-				{
-					Integration:         "integration",
-					LabelName:           "label",
-					Window:              dayWindows[2],
-					CloudCostAggregates: map[string]*CloudCostAggregate{},
-				},
-			},
-		},
-		"Load Single Day Off Grid": {
-			cca: []*CloudCostAggregate{
-				{
-					Properties:        ccaProperties1,
-					KubernetesPercent: 1,
-					Cost:              100,
-					NetCost:           80,
-				},
-			},
-			windows: []Window{
-				NewClosedWindow(start.Add(12*time.Hour), start.Add(36*time.Hour)),
-			},
-			ccasr: emtpyCASSR.Clone(),
-			expected: []*CloudCostAggregateSet{
-				{
-					Integration: "integration",
-					LabelName:   "label",
-					Window:      dayWindows[0],
-					CloudCostAggregates: map[string]*CloudCostAggregate{
-						ccaProperties1.Key(nil): {
-							Properties:        ccaProperties1,
-							KubernetesPercent: 1,
-							Cost:              50,
-							NetCost:           40,
-						},
-					},
-				},
-				{
-					Integration: "integration",
-					LabelName:   "label",
-					Window:      dayWindows[1],
-					CloudCostAggregates: map[string]*CloudCostAggregate{
-						ccaProperties1.Key(nil): {
-							Properties:        ccaProperties1,
-							KubernetesPercent: 1,
-							Cost:              50,
-							NetCost:           40,
-						},
-					},
-				},
-				{
-					Integration:         "integration",
-					LabelName:           "label",
-					Window:              dayWindows[2],
-					CloudCostAggregates: map[string]*CloudCostAggregate{},
-				},
-			},
-		},
-		"Load Single Day Off Grid Before Range Window": {
-			cca: []*CloudCostAggregate{
-				{
-					Properties:        ccaProperties1,
-					KubernetesPercent: 1,
-					Cost:              100,
-					NetCost:           80,
-				},
-			},
-			windows: []Window{
-				NewClosedWindow(start.Add(-12*time.Hour), start.Add(12*time.Hour)),
-			},
-			ccasr: emtpyCASSR.Clone(),
-			expected: []*CloudCostAggregateSet{
-				{
-					Integration: "integration",
-					LabelName:   "label",
-					Window:      dayWindows[0],
-					CloudCostAggregates: map[string]*CloudCostAggregate{
-						ccaProperties1.Key(nil): {
-							Properties:        ccaProperties1,
-							KubernetesPercent: 1,
-							Cost:              50,
-							NetCost:           40,
-						},
-					},
-				},
-				{
-					Integration:         "integration",
-					LabelName:           "label",
-					Window:              dayWindows[1],
-					CloudCostAggregates: map[string]*CloudCostAggregate{},
-				},
-				{
-					Integration:         "integration",
-					LabelName:           "label",
-					Window:              dayWindows[2],
-					CloudCostAggregates: map[string]*CloudCostAggregate{},
-				},
-			},
-		},
-		"Load Single Day Off Grid After Range Window": {
-			cca: []*CloudCostAggregate{
-				{
-					Properties:        ccaProperties1,
-					KubernetesPercent: 1,
-					Cost:              100,
-					NetCost:           80,
-				},
-			},
-			windows: []Window{
-				NewClosedWindow(end.Add(-12*time.Hour), end.Add(12*time.Hour)),
-			},
-			ccasr: emtpyCASSR.Clone(),
-			expected: []*CloudCostAggregateSet{
-				{
-					Integration:         "integration",
-					LabelName:           "label",
-					Window:              dayWindows[0],
-					CloudCostAggregates: map[string]*CloudCostAggregate{},
-				},
-				{
-					Integration:         "integration",
-					LabelName:           "label",
-					Window:              dayWindows[1],
-					CloudCostAggregates: map[string]*CloudCostAggregate{},
-				},
-				{
-					Integration: "integration",
-					LabelName:   "label",
-					Window:      dayWindows[2],
-					CloudCostAggregates: map[string]*CloudCostAggregate{
-						ccaProperties1.Key(nil): {
-							Properties:        ccaProperties1,
-							KubernetesPercent: 1,
-							Cost:              50,
-							NetCost:           40,
-						},
-					},
-				},
-			},
-		},
-		"Single Day Kubecost Percent": {
-			cca: []*CloudCostAggregate{
-				{
-					Properties:        ccaProperties1,
-					KubernetesPercent: 1,
-					Cost:              75,
-					NetCost:           60,
-				},
-				{
-					Properties:        ccaProperties1,
-					KubernetesPercent: 0,
-					Cost:              25,
-					NetCost:           20,
-				},
-			},
-			windows: []Window{
-				dayWindows[1],
-				dayWindows[1],
-			},
-			ccasr: emtpyCASSR.Clone(),
-			expected: []*CloudCostAggregateSet{
-				{
-					Integration:         "integration",
-					LabelName:           "label",
-					Window:              dayWindows[0],
-					CloudCostAggregates: map[string]*CloudCostAggregate{},
-				},
-				{
-					Integration: "integration",
-					LabelName:   "label",
-					Window:      dayWindows[1],
-					CloudCostAggregates: map[string]*CloudCostAggregate{
-						ccaProperties1.Key(nil): {
-							Properties:        ccaProperties1,
-							KubernetesPercent: 0.75,
-							Cost:              100,
-							NetCost:           80,
-						},
-					},
-				},
-				{
-					Integration:         "integration",
-					LabelName:           "label",
-					Window:              dayWindows[2],
-					CloudCostAggregates: map[string]*CloudCostAggregate{},
-				},
-			},
-		},
-	}
-
-	for name, tc := range testCases {
-		t.Run(name, func(t *testing.T) {
-			// load Cloud Cost Aggregates
-			for i, cca := range tc.cca {
-				tc.ccasr.LoadCloudCostAggregate(tc.windows[i], cca)
-			}
-
-			if len(tc.ccasr.CloudCostAggregateSets) != len(tc.expected) {
-				t.Errorf("the CloudCostAggregateSetRanges did not have the expected length")
-			}
-
-			for i, ccas := range tc.ccasr.CloudCostAggregateSets {
-				if !ccas.Equal(tc.expected[i]) {
-					t.Errorf("CloudCostAggregateSet at index: %d did not match expected", i)
-				}
-			}
-		})
-	}
-
-}

pkg/kubecost/cloudcostitem.go

@@ -1,519 +0,0 @@
-package kubecost
-
-import (
-	"fmt"
-	"strings"
-	"time"
-
-	"github.com/opencost/opencost/pkg/filter"
-	"github.com/opencost/opencost/pkg/log"
-)
-
-// These contain some labels that can be used on Cloud cost
-// item to get the corresponding cluster its associated.
-const (
-	AWSMatchLabel1     = "eks_cluster_name"
-	AWSMatchLabel2     = "alpha_eksctl_io_cluster_name"
-	AlibabaMatchLabel1 = "ack.aliyun.com"
-	GCPMatchLabel1     = "goog-k8s-cluster-name"
-)
-
-type CloudCostItemLabels map[string]string
-
-func (ccil CloudCostItemLabels) Clone() CloudCostItemLabels {
-	result := make(map[string]string, len(ccil))
-	for k, v := range ccil {
-		result[k] = v
-	}
-	return result
-}
-
-func (ccil CloudCostItemLabels) Equal(that CloudCostItemLabels) bool {
-	if len(ccil) != len(that) {
-		return false
-	}
-
-	// Maps are of equal length, so if all keys are in both maps, we don't
-	// have to check the keys of the other map.
-	for k, v := range ccil {
-		if tv, ok := that[k]; !ok || v != tv {
-			return false
-		}
-	}
-
-	return true
-}
-
-type CloudCostItemProperties struct {
-	ProviderID  string              `json:"providerID,omitempty"`
-	Provider    string              `json:"provider,omitempty"`
-	WorkGroupID string              `json:"workGroupID,omitempty"`
-	BillingID   string              `json:"billingID,omitempty"`
-	Service     string              `json:"service,omitempty"`
-	Category    string              `json:"category,omitempty"`
-	Labels      CloudCostItemLabels `json:"labels,omitempty"`
-}
-
-func (ccip CloudCostItemProperties) Equal(that CloudCostItemProperties) bool {
-	return ccip.ProviderID == that.ProviderID &&
-		ccip.Provider == that.Provider &&
-		ccip.WorkGroupID == that.WorkGroupID &&
-		ccip.BillingID == that.BillingID &&
-		ccip.Service == that.Service &&
-		ccip.Category == that.Category &&
-		ccip.Labels.Equal(that.Labels)
-}
-
-func (ccip CloudCostItemProperties) Clone() CloudCostItemProperties {
-	return CloudCostItemProperties{
-		ProviderID:  ccip.ProviderID,
-		Provider:    ccip.Provider,
-		WorkGroupID: ccip.WorkGroupID,
-		BillingID:   ccip.BillingID,
-		Service:     ccip.Service,
-		Category:    ccip.Category,
-		Labels:      ccip.Labels.Clone(),
-	}
-}
-
-func (ccip CloudCostItemProperties) Key() string {
-	return fmt.Sprintf("%s/%s/%s/%s/%s/%s", ccip.Provider, ccip.BillingID, ccip.WorkGroupID, ccip.Category, ccip.Service, ccip.ProviderID)
-}
-
-func (ccip CloudCostItemProperties) MonitoringKey() string {
-	return fmt.Sprintf("%s/%s", ccip.Provider, ccip.ProviderID)
-}
-
-// CloudCostItem represents a CUR line item, identifying a cloud resource and
-// its cost over some period of time.
-type CloudCostItem struct {
-	Properties   CloudCostItemProperties `json:"properties"`
-	IsKubernetes bool                    `json:"isKubernetes"`
-	Window       Window                  `json:"window"`
-	Cost         float64                 `json:"cost"`
-	NetCost      float64                 `json:"netCost"`
-}
-
-// NewCloudCostItem instantiates a new CloudCostItem asset
-func NewCloudCostItem(start, end time.Time, cciProperties CloudCostItemProperties, isKubernetes bool, cost, netcost float64) *CloudCostItem {
-	return &CloudCostItem{
-		Properties:   cciProperties,
-		IsKubernetes: isKubernetes,
-		Window:       NewWindow(&start, &end),
-		Cost:         cost,
-		NetCost:      netcost,
-	}
-}
-
-func (cci *CloudCostItem) Clone() *CloudCostItem {
-	return &CloudCostItem{
-		Properties:   cci.Properties.Clone(),
-		IsKubernetes: cci.IsKubernetes,
-		Window:       cci.Window.Clone(),
-		Cost:         cci.Cost,
-		NetCost:      cci.NetCost,
-	}
-}
-
-func (cci *CloudCostItem) Equal(that *CloudCostItem) bool {
-	if that == nil {
-		return false
-	}
-
-	return cci.Properties.Equal(that.Properties) &&
-		cci.IsKubernetes == that.IsKubernetes &&
-		cci.Window.Equal(that.Window) &&
-		cci.Cost == that.Cost &&
-		cci.NetCost == that.NetCost
-}
-
-func (cci *CloudCostItem) Key() string {
-	return cci.Properties.Key()
-}
-
-func (cci *CloudCostItem) add(that *CloudCostItem) {
-	if cci == nil {
-		log.Warnf("cannot add to nil CloudCostItem")
-		return
-	}
-
-	cci.Cost += that.Cost
-	cci.NetCost += that.NetCost
-	cci.Window = cci.Window.Expand(that.Window)
-}
-
-func (cci *CloudCostItem) MonitoringKey() string {
-	return cci.Properties.MonitoringKey()
-}
-
-// Ony use compute resources to get Cluster names
-func (cci *CloudCostItem) GetCluster() string {
-	switch provider := cci.Properties.Provider; provider {
-	case AWSProvider:
-		return cci.GetAWSCluster()
-	case AzureProvider:
-		return cci.GetAzureCluster()
-	case GCPProvider:
-		return cci.GetGCPCluster()
-	case AlibabaProvider:
-		return cci.GetAlibabaCluster()
-	default:
-		log.Warnf("unsupported CloudCostItem found for a provider: %s", provider)
-		return ""
-	}
-}
-
-// Add any new ways of finding GCP cluster from Cloud cost Item
-func (cci *CloudCostItem) GetGCPCluster() string {
-	// currently from Cloud cost compute unable to get cluster name so returning empty
-	return ""
-}
-
-// Add any new ways of finding AWS cluster from Cloud cost Item
-func (cci *CloudCostItem) GetAWSCluster() string {
-	if cci == nil {
-		return ""
-	}
-
-	// This flag should be removed with filters in the compute query
-	if cci.Properties.Provider != AWSProvider || cci.Properties.Category != ComputeCategory {
-		return ""
-	}
-	// cn be either of these two labels to distinguish cluster name for a given providerID
-	if val, ok := cci.Properties.Labels[AWSMatchLabel1]; ok {
-		return val
-	}
-	if val, ok := cci.Properties.Labels[AWSMatchLabel2]; ok {
-		return val
-	}
-	return ""
-}
-
-// Add any new ways of finding Azure cluster from Cloud cost Item
-func (cci *CloudCostItem) GetAzureCluster() string {
-	if cci == nil {
-		return ""
-	}
-
-	// This flag should be removed with filters in the compute query
-	if cci.Properties.Provider != AzureProvider || cci.Properties.Category != ComputeCategory {
-		return ""
-	}
-
-	providerIDSplit := strings.Split(cci.Properties.ProviderID, "/")
-	// ensure this is actually returnable before return
-	if len(providerIDSplit) < 6 {
-		return ""
-	}
-	return strings.Split(cci.Properties.ProviderID, "/")[6]
-}
-
-// Add any new ways of finding Alibaba cluster from Cloud cost Item
-func (cci *CloudCostItem) GetAlibabaCluster() string {
-	if cci == nil {
-		return ""
-	}
-
-	// This flag should be removed with filters in the compute query
-	if cci.Properties.Provider != AlibabaProvider || cci.Properties.Category != ComputeCategory {
-		return ""
-	}
-	if val, ok := cci.Properties.Labels[AlibabaMatchLabel1]; ok {
-		return val
-	}
-	return ""
-}
-
-type CloudCostItemSet struct {
-	CloudCostItems map[string]*CloudCostItem `json:"items"`
-	Window         Window                    `json:"window"`
-	Integration    string                    `json:"-"`
-}
-
-// NewAssetSet instantiates a new AssetSet and, optionally, inserts
-// the given list of Assets
-func NewCloudCostItemSet(start, end time.Time, cloudCostItems ...*CloudCostItem) *CloudCostItemSet {
-	ccis := &CloudCostItemSet{
-		CloudCostItems: map[string]*CloudCostItem{},
-		Window:         NewWindow(&start, &end),
-	}
-
-	for _, cci := range cloudCostItems {
-		ccis.Insert(cci)
-	}
-
-	return ccis
-}
-
-func (ccis *CloudCostItemSet) Accumulate(that *CloudCostItemSet) (*CloudCostItemSet, error) {
-	if ccis.IsEmpty() {
-		return that.Clone(), nil
-	}
-
-	if that.IsEmpty() {
-		return ccis.Clone(), nil
-	}
-	// Set start, end to min(start), max(end)
-	start := ccis.Window.Start()
-	end := ccis.Window.End()
-	if that.Window.Start().Before(*start) {
-		start = that.Window.Start()
-	}
-	if that.Window.End().After(*end) {
-		end = that.Window.End()
-	}
-
-	acc := NewCloudCostItemSet(*start, *end)
-
-	for _, cci := range ccis.CloudCostItems {
-		err := acc.Insert(cci)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	for _, cci := range that.CloudCostItems {
-		err := acc.Insert(cci)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return acc, nil
-}
-
-func (ccis *CloudCostItemSet) Equal(that *CloudCostItemSet) bool {
-	if ccis.Integration != that.Integration {
-		return false
-	}
-
-	if !ccis.Window.Equal(that.Window) {
-		return false
-	}
-
-	if len(ccis.CloudCostItems) != len(that.CloudCostItems) {
-		return false
-	}
-
-	for k, cci := range ccis.CloudCostItems {
-		tcci, ok := that.CloudCostItems[k]
-		if !ok {
-			return false
-		}
-		if !cci.Equal(tcci) {
-			return false
-		}
-	}
-
-	return true
-}
-
-func (ccis *CloudCostItemSet) Filter(filters filter.Filter[*CloudCostItem]) *CloudCostItemSet {
-	if ccis == nil {
-		return nil
-	}
-
-	if filters == nil {
-		return ccis.Clone()
-	}
-
-	result := NewCloudCostItemSet(*ccis.Window.start, *ccis.Window.end)
-
-	for _, cci := range ccis.CloudCostItems {
-		if filters.Matches(cci) {
-			result.Insert(cci.Clone())
-		}
-	}
-
-	return result
-}
-
-func (ccis *CloudCostItemSet) Insert(that *CloudCostItem) error {
-	if ccis == nil {
-		return fmt.Errorf("cannot insert into nil CloudCostItemSet")
-	}
-
-	if that == nil {
-		return fmt.Errorf("cannot insert nil CloudCostItem into CloudCostItemSet")
-	}
-
-	if ccis.CloudCostItems == nil {
-		ccis.CloudCostItems = map[string]*CloudCostItem{}
-	}
-
-	// Add the given CloudCostItem to the existing entry, if there is one;
-	// otherwise just set directly into allocations
-	if _, ok := ccis.CloudCostItems[that.Key()]; !ok {
-		ccis.CloudCostItems[that.Key()] = that.Clone()
-	} else {
-		ccis.CloudCostItems[that.Key()].add(that)
-	}
-
-	return nil
-}
-
-func (ccis *CloudCostItemSet) Clone() *CloudCostItemSet {
-	items := make(map[string]*CloudCostItem, len(ccis.CloudCostItems))
-	for k, v := range ccis.CloudCostItems {
-		items[k] = v.Clone()
-	}
-
-	return &CloudCostItemSet{
-		CloudCostItems: items,
-		Integration:    ccis.Integration,
-		Window:         ccis.Window.Clone(),
-	}
-}
-
-func (ccis *CloudCostItemSet) IsEmpty() bool {
-	if ccis == nil {
-		return true
-	}
-
-	if len(ccis.CloudCostItems) == 0 {
-		return true
-	}
-
-	return false
-}
-
-func (ccis *CloudCostItemSet) Length() int {
-	if ccis == nil {
-		return 0
-	}
-	return len(ccis.CloudCostItems)
-}
-
-func (ccis *CloudCostItemSet) GetWindow() Window {
-	return ccis.Window
-}
-
-func (ccis *CloudCostItemSet) Merge(that *CloudCostItemSet) (*CloudCostItemSet, error) {
-	if ccis == nil {
-		return nil, fmt.Errorf("cannot merge nil CloudCostItemSets")
-	}
-
-	if that.IsEmpty() {
-		return ccis.Clone(), nil
-	}
-
-	if !ccis.Window.Equal(that.Window) {
-		return nil, fmt.Errorf("cannot merge CloudCostItemSets with different windows")
-	}
-
-	start, end := *ccis.Window.Start(), *ccis.Window.End()
-	result := NewCloudCostItemSet(start, end)
-
-	for _, cci := range ccis.CloudCostItems {
-		result.Insert(cci)
-	}
-
-	for _, cci := range that.CloudCostItems {
-		result.Insert(cci)
-	}
-
-	return result, nil
-}
-
-type CloudCostItemSetRange struct {
-	CloudCostItemSets []*CloudCostItemSet `json:"sets"`
-	Window            Window              `json:"window"`
-}
-
-// NewCloudCostItemSetRange create a CloudCostItemSetRange containing CloudCostItemSets with windows of equal duration
-// the duration between start and end must be divisible by the window duration argument
-func NewCloudCostItemSetRange(start time.Time, end time.Time, window time.Duration, integration string) (*CloudCostItemSetRange, error) {
-	windows, err := GetWindows(start, end, window)
-	if err != nil {
-		return nil, err
-	}
-
-	// Build slice of CloudCostItemSet to cover the range
-	cloudCostItemSets := make([]*CloudCostItemSet, len(windows))
-	for i, w := range windows {
-		ccis := NewCloudCostItemSet(*w.Start(), *w.End())
-		ccis.Integration = integration
-		cloudCostItemSets[i] = ccis
-	}
-	return &CloudCostItemSetRange{
-		Window:            NewWindow(&start, &end),
-		CloudCostItemSets: cloudCostItemSets,
-	}, nil
-}
-
-func (ccisr *CloudCostItemSetRange) Clone() *CloudCostItemSetRange {
-	ccisSlice := make([]*CloudCostItemSet, len(ccisr.CloudCostItemSets))
-	for i, ccis := range ccisr.CloudCostItemSets {
-		ccisSlice[i] = ccis.Clone()
-	}
-	return &CloudCostItemSetRange{
-		Window:            ccisr.Window.Clone(),
-		CloudCostItemSets: ccisSlice,
-	}
-}
-
-// Accumulate sums each CloudCostItemSet in the given range, returning a single cumulative
-// CloudCostItemSet for the entire range.
-func (ccisr *CloudCostItemSetRange) Accumulate() (*CloudCostItemSet, error) {
-	var cloudCostItemSet *CloudCostItemSet
-	var err error
-
-	for _, ccis := range ccisr.CloudCostItemSets {
-		cloudCostItemSet, err = cloudCostItemSet.Accumulate(ccis)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return cloudCostItemSet, nil
-}
-
-// LoadCloudCostItem loads CloudCostItems into existing CloudCostItemSets of the CloudCostItemSetRange.
-// This function service to aggregate and distribute costs over predefined windows
-// are accumulated here so that the resulting CloudCostItem with the 1d window has the correct price for the entire day.
-// If all or a portion of the window of the CloudCostItem is outside of the windows of the existing CloudCostItemSets,
-// that portion of the CloudCostItem's cost will not be inserted
-func (ccisr *CloudCostItemSetRange) LoadCloudCostItem(cloudCostItem *CloudCostItem) {
-	window := cloudCostItem.Window
-	if window.IsOpen() {
-		log.Errorf("CloudCostItemSetRange: LoadCloudCostItem: invalid window %s", window.String())
-		return
-	}
-
-	totalPct := 0.0
-
-	// Distribute cost of the current item across one or more CloudCostItems in
-	// across each relevant CloudCostItemSet. Stop when the end of the current
-	// block reaches the item's end time or the end of the range.
-	for _, ccis := range ccisr.CloudCostItemSets {
-		setWindow := ccis.Window
-
-		// get percent of item window contained in set window
-		pct := setWindow.GetPercentInWindow(window)
-		if pct == 0 {
-			continue
-		}
-
-		cci := cloudCostItem
-		// If the current set Window only contains a portion of the CloudCostItem Window, insert costs relative to that portion
-		if pct < 1.0 {
-			cci = &CloudCostItem{
-				Properties:   cloudCostItem.Properties,
-				IsKubernetes: cloudCostItem.IsKubernetes,
-				Window:       window.Contract(setWindow),
-				Cost:         cloudCostItem.Cost * pct,
-				NetCost:      cloudCostItem.NetCost * pct,
-			}
-		}
-
-		err := ccis.Insert(cci)
-		if err != nil {
-			log.Errorf("CloudCostItemSetRange: LoadCloudCostItem: failed to load CloudCostItem with key %s and window %s: %s", cci.Key(), ccis.GetWindow().String(), err.Error())
-		}
-
-		// If all cost has been inserted then finish
-		totalPct += pct
-		if totalPct >= 1.0 {
-			return
-		}
-	}
-}

pkg/kubecost/cloudcostitem_test.go

@@ -1,420 +0,0 @@
-package kubecost
-
-import (
-	"testing"
-	"time"
-
-	"github.com/opencost/opencost/pkg/util/timeutil"
-)
-
-var cciProperties1 = CloudCostItemProperties{
-	ProviderID:  "providerid1",
-	Provider:    "provider1",
-	WorkGroupID: "workgroup1",
-	BillingID:   "billing1",
-	Service:     "service1",
-	Category:    "category1",
-	Labels: map[string]string{
-		"label1": "value1",
-		"label2": "value2",
-	},
-}
-
-// TestCloudCostItem_LoadCloudCostItem checks that loaded CloudCostItems end up in the correct set in the
-// correct proportions
-func TestCloudCostItem_LoadCloudCostItem(t *testing.T) {
-	// create values for 3 day Range tests
-	end := RoundBack(time.Now().UTC(), timeutil.Day)
-	start := end.Add(-3 * timeutil.Day)
-	dayWindows, _ := GetWindows(start, end, timeutil.Day)
-	emtpyCCISR, _ := NewCloudCostItemSetRange(start, end, timeutil.Day, "integration")
-	testCases := map[string]struct {
-		cci      []*CloudCostItem
-		ccisr    *CloudCostItemSetRange
-		expected []*CloudCostItemSet
-	}{
-		"Load Single Day On Grid": {
-			cci: []*CloudCostItem{
-				{
-					Properties:   cciProperties1,
-					Window:       dayWindows[0],
-					IsKubernetes: true,
-					Cost:         100,
-					NetCost:      80,
-				},
-			},
-			ccisr: emtpyCCISR.Clone(),
-			expected: []*CloudCostItemSet{
-				{
-					Integration: "integration",
-					Window:      dayWindows[0],
-					CloudCostItems: map[string]*CloudCostItem{
-						cciProperties1.Key(): {
-							Properties:   cciProperties1,
-							Window:       dayWindows[0],
-							IsKubernetes: true,
-							Cost:         100,
-							NetCost:      80,
-						},
-					},
-				},
-				{
-					Integration:    "integration",
-					Window:         dayWindows[1],
-					CloudCostItems: map[string]*CloudCostItem{},
-				},
-				{
-					Integration:    "integration",
-					Window:         dayWindows[2],
-					CloudCostItems: map[string]*CloudCostItem{},
-				},
-			},
-		},
-		"Load Single Day Off Grid": {
-			cci: []*CloudCostItem{
-				{
-					Properties:   cciProperties1,
-					Window:       NewClosedWindow(start.Add(12*time.Hour), start.Add(36*time.Hour)),
-					IsKubernetes: true,
-					Cost:         100,
-					NetCost:      80,
-				},
-			},
-			ccisr: emtpyCCISR.Clone(),
-			expected: []*CloudCostItemSet{
-				{
-					Integration: "integration",
-					Window:      dayWindows[0],
-					CloudCostItems: map[string]*CloudCostItem{
-						cciProperties1.Key(): {
-							Properties:   cciProperties1,
-							Window:       NewClosedWindow(start.Add(12*time.Hour), start.Add(24*time.Hour)),
-							IsKubernetes: true,
-							Cost:         50,
-							NetCost:      40,
-						},
-					},
-				},
-				{
-					Integration: "integration",
-					Window:      dayWindows[1],
-					CloudCostItems: map[string]*CloudCostItem{
-						cciProperties1.Key(): {
-							Properties:   cciProperties1,
-							Window:       NewClosedWindow(start.Add(24*time.Hour), start.Add(36*time.Hour)),
-							IsKubernetes: true,
-							Cost:         50,
-							NetCost:      40,
-						},
-					},
-				},
-				{
-					Integration:    "integration",
-					Window:         dayWindows[2],
-					CloudCostItems: map[string]*CloudCostItem{},
-				},
-			},
-		},
-		"Load Single Day Off Grid Before Range Window": {
-			cci: []*CloudCostItem{
-				{
-					Properties:   cciProperties1,
-					Window:       NewClosedWindow(start.Add(-12*time.Hour), start.Add(12*time.Hour)),
-					IsKubernetes: true,
-					Cost:         100,
-					NetCost:      80,
-				},
-			},
-			ccisr: emtpyCCISR.Clone(),
-			expected: []*CloudCostItemSet{
-				{
-					Integration: "integration",
-					Window:      dayWindows[0],
-					CloudCostItems: map[string]*CloudCostItem{
-						cciProperties1.Key(): {
-							Properties:   cciProperties1,
-							Window:       NewClosedWindow(start, start.Add(12*time.Hour)),
-							IsKubernetes: true,
-							Cost:         50,
-							NetCost:      40,
-						},
-					},
-				},
-				{
-					Integration:    "integration",
-					Window:         dayWindows[1],
-					CloudCostItems: map[string]*CloudCostItem{},
-				},
-				{
-					Integration:    "integration",
-					Window:         dayWindows[2],
-					CloudCostItems: map[string]*CloudCostItem{},
-				},
-			},
-		},
-		"Load Single Day Off Grid After Range Window": {
-			cci: []*CloudCostItem{
-				{
-					Properties:   cciProperties1,
-					Window:       NewClosedWindow(end.Add(-12*time.Hour), end.Add(12*time.Hour)),
-					IsKubernetes: true,
-					Cost:         100,
-					NetCost:      80,
-				},
-			},
-			ccisr: emtpyCCISR.Clone(),
-			expected: []*CloudCostItemSet{
-				{
-					Integration:    "integration",
-					Window:         dayWindows[0],
-					CloudCostItems: map[string]*CloudCostItem{},
-				},
-				{
-					Integration:    "integration",
-					Window:         dayWindows[1],
-					CloudCostItems: map[string]*CloudCostItem{},
-				},
-				{
-					Integration: "integration",
-					Window:      dayWindows[2],
-					CloudCostItems: map[string]*CloudCostItem{
-						cciProperties1.Key(): {
-							Properties:   cciProperties1,
-							Window:       NewClosedWindow(end.Add(-12*time.Hour), end),
-							IsKubernetes: true,
-							Cost:         50,
-							NetCost:      40,
-						},
-					},
-				},
-			},
-		},
-		"Single Day Kubecost Percent": {
-			cci: []*CloudCostItem{
-				{
-					Properties:   cciProperties1,
-					Window:       dayWindows[1],
-					IsKubernetes: true,
-					Cost:         75,
-					NetCost:      60,
-				},
-				{
-					Properties:   cciProperties1,
-					Window:       dayWindows[1],
-					IsKubernetes: true,
-					Cost:         25,
-					NetCost:      20,
-				},
-			},
-			ccisr: emtpyCCISR.Clone(),
-			expected: []*CloudCostItemSet{
-				{
-					Integration:    "integration",
-					Window:         dayWindows[0],
-					CloudCostItems: map[string]*CloudCostItem{},
-				},
-				{
-					Integration: "integration",
-					Window:      dayWindows[1],
-					CloudCostItems: map[string]*CloudCostItem{
-						cciProperties1.Key(): {
-							Properties:   cciProperties1,
-							Window:       dayWindows[1],
-							IsKubernetes: true,
-							Cost:         100,
-							NetCost:      80,
-						},
-					},
-				},
-				{
-					Integration:    "integration",
-					Window:         dayWindows[2],
-					CloudCostItems: map[string]*CloudCostItem{},
-				},
-			},
-		},
-	}
-
-	for name, tc := range testCases {
-		t.Run(name, func(t *testing.T) {
-			// load Cloud Cost Items
-			for _, cci := range tc.cci {
-				tc.ccisr.LoadCloudCostItem(cci)
-			}
-
-			if len(tc.ccisr.CloudCostItemSets) != len(tc.expected) {
-				t.Errorf("the CloudCostItemSetRanges did not have the expected length")
-			}
-
-			for i, ccis := range tc.ccisr.CloudCostItemSets {
-				if !ccis.Equal(tc.expected[i]) {
-					t.Errorf("CloudCostItemSet at index: %d did not match expected", i)
-				}
-			}
-		})
-	}
-
-}
-
-func TestGetAWSClusterFromCCI(t *testing.T) {
-	awsCCIWithLabeleksClusterName, eksClusterName := GenerateAWSMockCCIAndPID(1, 1, AWSMatchLabel1, ComputeCategory)
-	awsCCIWithLabeleksCtlClusterName, eksCtlClusterName := GenerateAWSMockCCIAndPID(2, 2, AWSMatchLabel2, ComputeCategory)
-	awsCCIWithLabelWithRandomLabel, _ := GenerateAWSMockCCIAndPID(1, 1, "randomLabel", ComputeCategory)
-	awsCCINetworkCategory, _ := GenerateAWSMockCCIAndPID(1, 1, AWSMatchLabel1, NetworkCategory)
-	alibabaCCI, _ := GenerateAlibabaMockCCIAndPID(4, 4, AlibabaMatchLabel1, ComputeCategory)
-	testCases := map[string]struct {
-		testcci  *CloudCostItem
-		expected string
-	}{
-		"cluster in label eks_cluster_name": {
-			testcci:  awsCCIWithLabeleksClusterName,
-			expected: eksClusterName,
-		},
-		"cluster in label alpha_eksctl_io_cluster_name": {
-			testcci:  awsCCIWithLabeleksCtlClusterName,
-			expected: eksCtlClusterName,
-		},
-		"cluster name in random label either not eks_cluster_name or eks_cluster_name": {
-			testcci:  awsCCIWithLabelWithRandomLabel,
-			expected: "",
-		},
-		"Not a AWS provider": {
-			testcci:  alibabaCCI,
-			expected: "",
-		},
-		"Not a compute resource": {
-			testcci:  awsCCINetworkCategory,
-			expected: "",
-		},
-	}
-	for name, testCase := range testCases {
-		t.Run(name, func(t *testing.T) {
-			actual := testCase.testcci.GetAWSCluster()
-			if actual != testCase.expected {
-				t.Errorf("incorrect result: Actual: '%s', Expected: '%s", actual, testCase.expected)
-			}
-		})
-	}
-}
-
-func TestGetAzureClusterFromCCI(t *testing.T) {
-	testCases := map[string]struct {
-		testcci  *CloudCostItem
-		expected string
-	}{
-		"cluster in ProviderID complete": {
-			testcci: &CloudCostItem{
-				IsKubernetes: true,
-				Window:       Window{},
-				Properties: CloudCostItemProperties{
-					Labels: map[string]string{
-						"randomLabel": "value1",
-					},
-					Provider:   AzureProvider,
-					Category:   ComputeCategory,
-					ProviderID: "azure:///subscriptions/0bd50fdf-c923-4e1e-850c-196dd3dcc5d3/resourceGroups/mc_dev_dev-1_eastus/providers/Microsoft.Compute/virtualMachineScaleSets/aks-devsysz1-24570986-vmss/virtualMachines/0",
-				},
-			},
-			expected: "mc_dev_dev-1_eastus",
-		},
-		"cluster in ProviderID complete but missing some values": {
-			testcci: &CloudCostItem{
-				IsKubernetes: true,
-				Window:       Window{},
-				Properties: CloudCostItemProperties{
-					Labels: map[string]string{
-						"randomLabel": "value1",
-					},
-					Provider:   AzureProvider,
-					Category:   ComputeCategory,
-					ProviderID: "azure:///subscriptions//resourceGroups/mc_dev_dev-1_eastus/providers/Microsoft.Compute/virtualMachineScaleSets/aks-devsysz1-XXXXX-vmss/virtualMachines/0",
-				},
-			},
-			expected: "mc_dev_dev-1_eastus",
-		},
-		"Not having enough split content in providerID": {
-			testcci: &CloudCostItem{
-				IsKubernetes: true,
-				Window:       Window{},
-				Properties: CloudCostItemProperties{
-					Labels: map[string]string{
-						"randomLabel": "value1",
-					},
-					Provider:   AzureProvider,
-					Category:   ComputeCategory,
-					ProviderID: "test1",
-				},
-			},
-			expected: "",
-		},
-		"Not a Azure provider": {
-			testcci: &CloudCostItem{
-				IsKubernetes: true,
-				Window:       Window{},
-				Properties: CloudCostItemProperties{
-					Labels: map[string]string{
-						"randomLabel": "value1",
-					},
-					Provider:   AWSProvider,
-					Category:   ComputeCategory,
-					ProviderID: "test1",
-				},
-			},
-			expected: "",
-		},
-		"Not a compute resource": {
-			testcci: &CloudCostItem{
-				IsKubernetes: true,
-				Window:       Window{},
-				Properties: CloudCostItemProperties{
-					Labels: map[string]string{
-						"randomLabel": "value1",
-					},
-					Provider:   AzureProvider,
-					Category:   StorageCategory,
-					ProviderID: "pvc-xyz",
-				},
-			},
-			expected: "",
-		},
-	}
-	for name, testCase := range testCases {
-		t.Run(name, func(t *testing.T) {
-			actual := testCase.testcci.GetAzureCluster()
-			if actual != testCase.expected {
-				t.Errorf("incorrect result: Actual: '%s', Expected: '%s", actual, testCase.expected)
-			}
-		})
-	}
-}
-
-func TestGetAlibabaClusterFromCCI(t *testing.T) {
-	alibabaCCIWithACKAliyunCom, clusterName1 := GenerateAlibabaMockCCIAndPID(4, 4, AlibabaMatchLabel1, ComputeCategory)
-	awsCCI, _ := GenerateAWSMockCCIAndPID(1, 1, AWSMatchLabel1, ComputeCategory)
-	alibabaCCINetworkCategory, clusterName1 := GenerateAlibabaMockCCIAndPID(4, 4, AlibabaMatchLabel1, NetworkCategory)
-	testCases := map[string]struct {
-		testcci  *CloudCostItem
-		expected string
-	}{
-		"cluster in label ack.aliyun.com": {
-			testcci:  alibabaCCIWithACKAliyunCom,
-			expected: clusterName1,
-		},
-		"Not a Alibaba provider": {
-			testcci:  awsCCI,
-			expected: "",
-		},
-		"Not a compute resource": {
-			testcci:  alibabaCCINetworkCategory,
-			expected: "",
-		},
-	}
-	for name, testCase := range testCases {
-		t.Run(name, func(t *testing.T) {
-			actual := testCase.testcci.GetAlibabaCluster()
-			if actual != testCase.expected {
-				t.Errorf("incorrect result: Actual: '%s', Expected: '%s", actual, testCase.expected)
-			}
-		})
-	}
-}

pkg/kubecost/cloudcostprops.go

@@ -0,0 +1,214 @@
+package kubecost
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/opencost/opencost/pkg/log"
+)
+
+const (
+	CloudCostInvoiceEntityIDProp string = "invoiceEntityID"
+	CloudCostAccountIDProp       string = "accountID"
+	CloudCostProviderProp        string = "provider"
+	CloudCostProviderIDProp      string = "providerID"
+	CloudCostCategoryProp        string = "category"
+	CloudCostServiceProp         string = "service"
+	CloudCostLabelProp           string = "label"
+)
+
+const (
+	// CloudCostClusterManagementCategory describes CloudCost representing Hosted Kubernetes Fees
+	CloudCostClusterManagementCategory string = "Cluster Management"
+
+	// CloudCostDiskCategory describes CloudCost representing Disk usage
+	CloudCostDiskCategory string = "Disk"
+
+	// CloudCostLoadBalancerCategory describes CloudCost representing Load Balancer usage
+	CloudCostLoadBalancerCategory string = "Load Balancer"
+
+	// CloudCostNetworkCategory describes CloudCost representing Network usage
+	CloudCostNetworkCategory string = "Network"
+
+	// CloudCostVirtualMachineCategory describes CloudCost representing VM usage
+	CloudCostVirtualMachineCategory string = "Virtual Machine"
+
+	// CloudCostOtherCategory describes CloudCost that do not belong to a defined category
+	CloudCostOtherCategory string = "Other"
+)
+
+type CloudCostLabels map[string]string
+
+func (ccl CloudCostLabels) Clone() CloudCostLabels {
+	result := make(map[string]string, len(ccl))
+	for k, v := range ccl {
+		result[k] = v
+	}
+	return result
+}
+
+func (ccl CloudCostLabels) Equal(that CloudCostLabels) bool {
+	if len(ccl) != len(that) {
+		return false
+	}
+
+	// Maps are of equal length, so if all keys are in both maps, we don't
+	// have to check the keys of the other map.
+	for k, val := range ccl {
+		if thatVal, ok := that[k]; !ok || val != thatVal {
+			return false
+		}
+	}
+
+	return true
+}
+
+// Intersection returns the set of labels that have the same key and value in the receiver and arg
+func (ccl CloudCostLabels) Intersection(that CloudCostLabels) CloudCostLabels {
+	intersection := make(map[string]string)
+	if len(ccl) == 0 || len(that) == 0 {
+		return intersection
+	}
+
+	// Pick the smaller of the two label sets
+	smallerLabels := ccl
+	largerLabels := that
+	if len(ccl) > len(that) {
+		smallerLabels = that
+		largerLabels = ccl
+	}
+
+	// Loop through the smaller label set
+	for k, sVal := range smallerLabels {
+		if lVal, ok := largerLabels[k]; ok && sVal == lVal {
+			intersection[k] = sVal
+		}
+	}
+	return intersection
+}
+
+type CloudCostProperties struct {
+	ProviderID      string          `json:"providerID,omitempty"`
+	Provider        string          `json:"provider,omitempty"`
+	AccountID       string          `json:"accountID,omitempty"`
+	InvoiceEntityID string          `json:"invoiceEntityID,omitempty"`
+	Service         string          `json:"service,omitempty"`
+	Category        string          `json:"category,omitempty"`
+	Labels          CloudCostLabels `json:"labels,omitempty"`
+}
+
+func (ccp *CloudCostProperties) Equal(that *CloudCostProperties) bool {
+	return ccp.ProviderID == that.ProviderID &&
+		ccp.Provider == that.Provider &&
+		ccp.AccountID == that.AccountID &&
+		ccp.InvoiceEntityID == that.InvoiceEntityID &&
+		ccp.Service == that.Service &&
+		ccp.Category == that.Category &&
+		ccp.Labels.Equal(that.Labels)
+}
+
+func (ccp *CloudCostProperties) Clone() *CloudCostProperties {
+	return &CloudCostProperties{
+		ProviderID:      ccp.ProviderID,
+		Provider:        ccp.Provider,
+		AccountID:       ccp.AccountID,
+		InvoiceEntityID: ccp.InvoiceEntityID,
+		Service:         ccp.Service,
+		Category:        ccp.Category,
+		Labels:          ccp.Labels.Clone(),
+	}
+}
+
+// Intersection ensure the values of two CloudCostAggregateProperties are maintain only if they are equal
+func (ccp *CloudCostProperties) Intersection(that *CloudCostProperties) *CloudCostProperties {
+	if ccp == nil || that == nil {
+		return nil
+	}
+
+	if ccp.Equal(that) {
+		return ccp
+	}
+	intersectionCCP := &CloudCostProperties{}
+	if ccp.Equal(intersectionCCP) || that.Equal(intersectionCCP) {
+		return intersectionCCP
+	}
+
+	if ccp.Provider == that.Provider {
+		intersectionCCP.Provider = ccp.Provider
+	}
+	if ccp.ProviderID == that.ProviderID {
+		intersectionCCP.ProviderID = ccp.ProviderID
+	}
+	if ccp.AccountID == that.AccountID {
+		intersectionCCP.AccountID = ccp.AccountID
+	}
+	if ccp.InvoiceEntityID == that.InvoiceEntityID {
+		intersectionCCP.InvoiceEntityID = ccp.InvoiceEntityID
+	}
+	if ccp.Service == that.Service {
+		intersectionCCP.Service = ccp.Service
+	}
+	if ccp.Category == that.Category {
+		intersectionCCP.Category = ccp.Category
+	}
+	intersectionCCP.Labels = ccp.Labels.Intersection(that.Labels)
+
+	return intersectionCCP
+}
+
+func (ccp *CloudCostProperties) GenerateKey(props []string) string {
+
+	if len(props) == 0 {
+		return fmt.Sprintf("%s/%s/%s/%s/%s/%s", ccp.Provider, ccp.InvoiceEntityID, ccp.AccountID, ccp.Category, ccp.Service, ccp.ProviderID)
+	}
+
+	values := make([]string, len(props))
+	for i, prop := range props {
+		propVal := UnallocatedSuffix
+
+		switch true {
+		case prop == CloudCostProviderProp:
+			if ccp.Provider != "" {
+				propVal = ccp.Provider
+			}
+		case prop == CloudCostProviderIDProp:
+			if ccp.ProviderID != "" {
+				propVal = ccp.ProviderID
+			}
+		case prop == CloudCostCategoryProp:
+			if ccp.Category != "" {
+				propVal = ccp.Category
+			}
+		case prop == CloudCostInvoiceEntityIDProp:
+			if ccp.InvoiceEntityID != "" {
+				propVal = ccp.InvoiceEntityID
+			}
+		case prop == CloudCostAccountIDProp:
+			if ccp.AccountID != "" {
+				propVal = ccp.AccountID
+			}
+		case prop == CloudCostServiceProp:
+			if ccp.Service != "" {
+				propVal = ccp.Service
+			}
+		case strings.HasPrefix(prop, "label:"):
+			labels := ccp.Labels
+			if labels != nil {
+				labelName := strings.TrimPrefix(prop, "label:")
+				if labelValue, ok := labels[labelName]; ok && labelValue != "" {
+					propVal = labelValue
+				}
+			}
+		default:
+			// This case should never be reached, as input up until this point
+			// should be checked and rejected if invalid. But if we do get a
+			// value we don't recognize, log a warning.
+			log.Warnf("CloudCost: GenerateKey: illegal aggregation parameter: %s", prop)
+
+		}
+
+		values[i] = propVal
+	}
+
+	return strings.Join(values, "/")
+}

pkg/kubecost/cloudcostprops_test.go

@@ -0,0 +1,165 @@
+package kubecost
+
+import "testing"
+
+func TestCloudCostPropertiesIntersection(t *testing.T) {
+	testCases := map[string]struct {
+		baseCCP     *CloudCostProperties
+		intCCP      *CloudCostProperties
+		expectedCCP *CloudCostProperties
+	}{
+		"When properties match between both CloudCostProperties": {
+			baseCCP: &CloudCostProperties{
+				Provider:        "CustomProvider",
+				ProviderID:      "ProviderID1",
+				AccountID:       "WorkGroupID1",
+				InvoiceEntityID: "InvoiceEntityID1",
+				Service:         "Service1",
+				Category:        "Category1",
+				Labels: map[string]string{
+					"key1": "value1",
+				},
+			},
+			intCCP: &CloudCostProperties{
+				Provider:        "CustomProvider",
+				ProviderID:      "ProviderID1",
+				AccountID:       "WorkGroupID1",
+				InvoiceEntityID: "InvoiceEntityID1",
+				Service:         "Service1",
+				Category:        "Category1",
+				Labels: map[string]string{
+					"key1": "value1",
+				},
+			},
+			expectedCCP: &CloudCostProperties{
+				Provider:        "CustomProvider",
+				ProviderID:      "ProviderID1",
+				AccountID:       "WorkGroupID1",
+				InvoiceEntityID: "InvoiceEntityID1",
+				Service:         "Service1",
+				Category:        "Category1",
+				Labels: map[string]string{
+					"key1": "value1",
+				},
+			},
+		},
+		"When one of the properties differ in the two CloudCostProperties": {
+			baseCCP: &CloudCostProperties{
+				Provider:        "CustomProvider",
+				ProviderID:      "ProviderID1",
+				AccountID:       "WorkGroupID1",
+				InvoiceEntityID: "InvoiceEntityID1",
+				Service:         "Service1",
+				Category:        "Category1",
+				Labels: map[string]string{
+					"key1": "value1",
+				},
+			},
+			intCCP: &CloudCostProperties{
+				Provider:        "CustomProvider",
+				ProviderID:      "ProviderID1",
+				AccountID:       "WorkGroupID1",
+				InvoiceEntityID: "InvoiceEntityID1",
+				Service:         "Service2",
+				Category:        "Category1",
+				Labels: map[string]string{
+					"key1": "value1",
+				},
+			},
+			expectedCCP: &CloudCostProperties{
+				Provider:        "CustomProvider",
+				ProviderID:      "ProviderID1",
+				AccountID:       "WorkGroupID1",
+				InvoiceEntityID: "InvoiceEntityID1",
+				Service:         "",
+				Category:        "Category1",
+				Labels: map[string]string{
+					"key1": "value1",
+				},
+			},
+		},
+		"When two of the properties differ in the two CloudCostProperties": {
+			baseCCP: &CloudCostProperties{
+				Provider:        "CustomProvider",
+				ProviderID:      "ProviderID1",
+				AccountID:       "WorkGroupID1",
+				InvoiceEntityID: "InvoiceEntityID1",
+				Service:         "Service1",
+				Category:        "Category1",
+				Labels: map[string]string{
+					"key1": "value1",
+				},
+			},
+			intCCP: &CloudCostProperties{
+				Provider:        "CustomProvider",
+				ProviderID:      "ProviderID1",
+				AccountID:       "WorkGroupID2",
+				InvoiceEntityID: "InvoiceEntityID1",
+				Service:         "Service2",
+				Category:        "Category1",
+				Labels: map[string]string{
+					"key1": "value1",
+				},
+			},
+			expectedCCP: &CloudCostProperties{
+				Provider:        "CustomProvider",
+				ProviderID:      "ProviderID1",
+				AccountID:       "",
+				InvoiceEntityID: "InvoiceEntityID1",
+				Service:         "",
+				Category:        "Category1",
+				Labels: map[string]string{
+					"key1": "value1",
+				},
+			},
+		},
+		"When labels differ": {
+			baseCCP: &CloudCostProperties{
+				Provider:        "CustomProvider",
+				ProviderID:      "ProviderID1",
+				AccountID:       "WorkGroupID1",
+				InvoiceEntityID: "InvoiceEntityID1",
+				Service:         "Service1",
+				Category:        "Category1",
+				Labels: map[string]string{
+					"key1": "value1",
+					"key2": "value2",
+					"key3": "value3",
+				},
+			},
+			intCCP: &CloudCostProperties{
+				Provider:        "CustomProvider",
+				ProviderID:      "ProviderID1",
+				AccountID:       "WorkGroupID1",
+				InvoiceEntityID: "InvoiceEntityID1",
+				Service:         "Service1",
+				Category:        "Category1",
+				Labels: map[string]string{
+					"key1": "value2",
+					"key2": "value2",
+					"key4": "value4",
+				},
+			},
+			expectedCCP: &CloudCostProperties{
+				Provider:        "CustomProvider",
+				ProviderID:      "ProviderID1",
+				AccountID:       "WorkGroupID1",
+				InvoiceEntityID: "InvoiceEntityID1",
+				Service:         "Service1",
+				Category:        "Category1",
+				Labels: map[string]string{
+					"key2": "value2",
+				},
+			},
+		},
+	}
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actualCCP := tc.baseCCP.Intersection(tc.intCCP)
+
+			if !actualCCP.Equal(tc.expectedCCP) {
+				t.Errorf("Case %s: properties dont match with expected CloudCostProperties: %v actual %v", name, tc.expectedCCP, actualCCP)
+			}
+		})
+	}
+}

pkg/kubecost/mock.go

@@ -2,7 +2,6 @@ package kubecost
 
 import (
 	"fmt"
-	"strconv"
 	"time"
 )
 
@@ -699,53 +698,6 @@ func GenerateMockAssetSet(start time.Time, duration time.Duration) *AssetSet {
 	)
 }
 
-func GenerateKubecostNodeAndPID(mockProviderIDInt int, provider string, mockClusterID int, setEndTime time.Time) (*Node, string) {
-	providerID := "PID" + strconv.FormatInt(int64(mockProviderIDInt), 10)
-	return &Node{
-		Properties: &AssetProperties{
-			Provider:   provider,
-			ProviderID: providerID,
-			Cluster:    "cluster" + strconv.FormatInt(int64(mockClusterID), 10),
-		},
-		End: setEndTime,
-	}, providerID
-}
-func GenerateAWSMockCCIAndPID(mockProviderIDInt int, mockCloudIDInt int, labelKey string, resourceCategory string) (*CloudCostItem, string) {
-	return &CloudCostItem{
-		Properties: CloudCostItemProperties{
-			ProviderID: "PID" + strconv.FormatInt(int64(mockProviderIDInt), 10),
-			Provider:   AWSProvider,
-			Category:   resourceCategory,
-			Labels: map[string]string{
-				labelKey: "cluster" + strconv.FormatInt(int64(mockCloudIDInt), 10),
-			},
-		},
-	}, "cluster" + strconv.FormatInt(int64(mockCloudIDInt), 10)
-}
-
-func GenerateAlibabaMockCCIAndPID(mockProviderIDInt int, mockCloudIDInt int, labelKey string, resourceCategory string) (*CloudCostItem, string) {
-	return &CloudCostItem{
-		Properties: CloudCostItemProperties{
-			ProviderID: "PID" + strconv.FormatInt(int64(mockProviderIDInt), 10),
-			Provider:   AlibabaProvider,
-			Category:   resourceCategory,
-			Labels: map[string]string{
-				labelKey: "cluster" + strconv.FormatInt(int64(mockCloudIDInt), 10),
-			},
-		},
-	}, "cluster" + strconv.FormatInt(int64(mockCloudIDInt), 10)
-}
-
-func GenerateGCPMockCCIAndPID(mockProviderIDInt int, mockCloudIDInt int, labelKey string, resourceCategory string) (*CloudCostItem, string) {
-	return &CloudCostItem{
-		Properties: CloudCostItemProperties{
-			ProviderID: "PID" + strconv.FormatInt(int64(mockProviderIDInt), 10),
-			Provider:   GCPProvider,
-			Category:   resourceCategory,
-		},
-	}, ""
-}
-
 // NewMockUnitSummaryAllocation creates an *SummaryAllocation with all of its float64 values set to 1 and generic properties if not provided in arg
 func NewMockUnitSummaryAllocation(name string, start time.Time, resolution time.Duration, props *AllocationProperties) *SummaryAllocation {
 	if name == "" {

pkg/kubecost/window.go

@@ -746,14 +746,14 @@ func (w Window) DurationOffsetStrings() (string, string) {
 // e.g. here are the two possible scenarios as simplidied
 // 10m windows with dashes representing item's time running:
 //
-//  1. item falls entirely within one CloudCostItemSet window
+//  1. item falls entirely within one CloudCostSet window
 //     |     ---- |          |          |
 //     totalMins = 4.0
 //     pct := 4.0 / 4.0 = 1.0 for window 1
 //     pct := 0.0 / 4.0 = 0.0 for window 2
 //     pct := 0.0 / 4.0 = 0.0 for window 3
 //
-//  2. item overlaps multiple CloudCostItemSet windows
+//  2. item overlaps multiple CloudCostSet windows
 //     |      ----|----------|--        |
 //     totalMins = 16.0
 //     pct :=  4.0 / 16.0 = 0.250 for window 1
@@ -810,7 +810,7 @@ func GetWindows(start time.Time, end time.Time, windowSize time.Duration) ([]Win
 		return nil, fmt.Errorf("range timezone doesn't match configured timezone: expected %s; found %ds", env.GetParsedUTCOffset(), sz)
 	}
 
-	// Build array of windows to cover the CloudCostItemSetRange
+	// Build array of windows to cover the CloudCostSetRange
 	windows := []Window{}
 	s, e := start, start.Add(windowSize)
 	for !e.After(end) {
@@ -836,7 +836,7 @@ func GetWindowsForQueryWindow(start time.Time, end time.Time, queryWindow time.D
 		return nil, fmt.Errorf("range timezone doesn't match configured timezone: expected %s; found %ds", env.GetParsedUTCOffset(), sz)
 	}
 
-	// Build array of windows to cover the CloudCostItemSetRange
+	// Build array of windows to cover the CloudCostSetRange
 	windows := []Window{}
 	s, e := start, start.Add(queryWindow)
 	for s.Before(end) {