Merge pull request #1857 from ameijer/core-140-attempt-III

Core 140 attempt iii

pkg/costmodel/aggregation.go

@@ -2250,7 +2250,10 @@ func (a *Accesses) ComputeAllocationHandler(w http.ResponseWriter, r *http.Reque
 	// IncludeProportionalAssetResourceCosts, if true,
 	includeProportionalAssetResourceCosts := qp.GetBool("includeProportionalAssetResourceCosts", false)
 
-	asr, err := a.Model.QueryAllocation(window, resolution, step, aggregateBy, includeIdle, idleByNode, includeProportionalAssetResourceCosts)
+	// include aggregated labels/annotations if true
+	includeAggregatedMetadata := qp.GetBool("includeAggregatedMetadata", true)
+
+	asr, err := a.Model.QueryAllocation(window, resolution, step, aggregateBy, includeIdle, idleByNode, includeProportionalAssetResourceCosts, includeAggregatedMetadata)
 	if err != nil {
 		if strings.Contains(strings.ToLower(err.Error()), "bad request") {
 			WriteError(w, BadRequest(err.Error()))

pkg/costmodel/costmodel.go

@@ -2295,7 +2295,7 @@ func measureTimeAsync(start time.Time, threshold time.Duration, name string, ch
 	}
 }
 
-func (cm *CostModel) QueryAllocation(window kubecost.Window, resolution, step time.Duration, aggregate []string, includeIdle, idleByNode, includeProportionalAssetResourceCosts bool) (*kubecost.AllocationSetRange, error) {
+func (cm *CostModel) QueryAllocation(window kubecost.Window, resolution, step time.Duration, aggregate []string, includeIdle, idleByNode, includeProportionalAssetResourceCosts, includeAggregatedMetadata bool) (*kubecost.AllocationSetRange, error) {
 	// Validate window is legal
 	if window.IsOpen() || window.IsNegative() {
 		return nil, fmt.Errorf("illegal window: %s", window)
@@ -2347,6 +2347,7 @@ func (cm *CostModel) QueryAllocation(window kubecost.Window, resolution, step ti
 	opts := &kubecost.AllocationAggregationOptions{
 		IncludeProportionalAssetResourceCosts: includeProportionalAssetResourceCosts,
 		IdleByNode:                            idleByNode,
+		IncludeAggregatedMetadata:             includeAggregatedMetadata,
 	}
 
 	// Aggregate

pkg/kubecost/allocation.go

@@ -921,6 +921,7 @@ type AllocationAggregationOptions struct {
 	ShareSplit                            string
 	SharedHourlyCosts                     map[string]float64
 	SplitIdle                             bool
+	IncludeAggregatedMetadata             bool
 }
 
 // AggregateBy aggregates the Allocations in the given AllocationSet by the given
@@ -1041,6 +1042,10 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 	// them to their respective sets, removing them from the set of allocations
 	// to aggregate.
 	for _, alloc := range as.Allocations {
+		// if the user does not want any aggregated labels/annotations returned
+		// set the properties accordingly
+		alloc.Properties.AggregatedMetadata = options.IncludeAggregatedMetadata
+
 		// External allocations get aggregated post-hoc (see step 6) and do
 		// not necessarily contain complete sets of properties, so they are
 		// moved to a separate AllocationSet.

pkg/kubecost/allocationprops.go

@@ -103,6 +103,9 @@ type AllocationProperties struct {
 	ProviderID     string                `json:"providerID,omitempty"`
 	Labels         AllocationLabels      `json:"labels,omitempty"`
 	Annotations    AllocationAnnotations `json:"annotations,omitempty"`
+	// When set to true, maintain the intersection of all labels + annotations
+	// in the aggregated AllocationProperties object
+	AggregatedMetadata bool `json:"-"`
 }
 
 // AllocationLabels is a schema-free mapping of key/value pairs that can be
@@ -439,7 +442,31 @@ func (p *AllocationProperties) Intersection(that *AllocationProperties) *Allocat
 		intersectionProps.ControllerKind = p.ControllerKind
 	}
 	if p.Namespace == that.Namespace {
+
 		intersectionProps.Namespace = p.Namespace
+		// ignore the incoming labels from unallocated or unmounted special case pods
+		if p.AggregatedMetadata || that.AggregatedMetadata {
+			intersectionProps.AggregatedMetadata = true
+
+			// When aggregating by metadata, we maintain the intersection of the labels/annotations
+			// of the two AllocationProperties objects being intersected here.
+			// Special case unallocated/unmounted Allocations never have any labels or annotations.
+			// As a result, they have the effect of always clearing out the intersection,
+			// regardless if all the other actual allocations/etc have them.
+			// This logic is designed to effectively ignore the unmounted/unallocated objects
+			// and just copy over the labels from the other object - we only take the intersection
+			// of 'legitimate' allocations.
+			if p.Container == UnmountedSuffix {
+				intersectionProps.Annotations = that.Annotations
+				intersectionProps.Labels = that.Labels
+			} else if that.Container == UnmountedSuffix {
+				intersectionProps.Annotations = p.Annotations
+				intersectionProps.Labels = p.Labels
+			} else {
+				intersectionProps.Annotations = mapIntersection(p.Annotations, that.Annotations)
+				intersectionProps.Labels = mapIntersection(p.Labels, that.Labels)
+			}
+		}
 	}
 	if p.Pod == that.Pod {
 		intersectionProps.Pod = p.Pod
@@ -450,6 +477,20 @@ func (p *AllocationProperties) Intersection(that *AllocationProperties) *Allocat
 	return intersectionProps
 }
 
+func mapIntersection(map1, map2 map[string]string) map[string]string {
+	result := make(map[string]string)
+	for key, value := range map1 {
+		if value2, ok := map2[key]; ok {
+			if value2 == value {
+				result[key] = value
+			}
+		}
+
+	}
+
+	return result
+}
+
 func (p *AllocationProperties) String() string {
 	if p == nil {
 		return "<nil>"

pkg/kubecost/allocationprops_test.go

@@ -5,6 +5,110 @@ import (
 	"testing"
 )
 
+func TestAllocationPropsIntersection(t *testing.T) {
+	cases := map[string]struct {
+		allocationProps1 *AllocationProperties
+		allocationProps2 *AllocationProperties
+		expected         *AllocationProperties
+	}{
+		"intersection two allocation properties with empty labels/annotations": {
+			allocationProps1: &AllocationProperties{
+				Labels:      map[string]string{},
+				Annotations: map[string]string{},
+			},
+			allocationProps2: &AllocationProperties{
+				Labels:      map[string]string{},
+				Annotations: map[string]string{},
+			},
+			expected: &AllocationProperties{
+				Labels:      nil,
+				Annotations: nil,
+			},
+		},
+		"nil intersection": {
+			allocationProps1: nil,
+			allocationProps2: nil,
+			expected:         nil,
+		},
+		"intersection, with labels/annotations, no aggregated metdata": {
+			allocationProps1: &AllocationProperties{
+				AggregatedMetadata: false,
+				Node:               "node1",
+				Labels:             map[string]string{"key1": "val1"},
+				Annotations:        map[string]string{"key2": "val2"},
+			},
+			allocationProps2: &AllocationProperties{
+				AggregatedMetadata: false,
+				Node:               "node1",
+				Labels:             map[string]string{"key3": "val3"},
+				Annotations:        map[string]string{"key4": "val4"},
+			},
+			expected: &AllocationProperties{
+				AggregatedMetadata: false,
+				Node:               "node1",
+				Labels:             nil,
+				Annotations:        nil,
+			},
+		},
+		"intersection, with labels/annotations, with aggregated metdata": {
+			allocationProps1: &AllocationProperties{
+				AggregatedMetadata: false,
+				ControllerKind:     "controller1",
+				Namespace:          "ns1",
+				Labels:             map[string]string{"key1": "val1"},
+				Annotations:        map[string]string{"key2": "val2"},
+			},
+			allocationProps2: &AllocationProperties{
+				AggregatedMetadata: true,
+				ControllerKind:     "controller2",
+				Namespace:          "ns1",
+				Labels:             map[string]string{"key1": "val1"},
+				Annotations:        map[string]string{"key2": "val2"},
+			},
+			expected: &AllocationProperties{
+				AggregatedMetadata: true,
+				Namespace:          "ns1",
+				ControllerKind:     "",
+				Labels:             map[string]string{"key1": "val1"},
+				Annotations:        map[string]string{"key2": "val2"},
+			},
+		},
+		"intersection, with labels/annotations, special case container": {
+			allocationProps1: &AllocationProperties{
+				AggregatedMetadata: false,
+				Container:          UnmountedSuffix,
+				Namespace:          "ns1",
+				Labels:             map[string]string{},
+				Annotations:        map[string]string{},
+			},
+			allocationProps2: &AllocationProperties{
+				AggregatedMetadata: true,
+				Container:          "container3",
+				Namespace:          "ns1",
+				Labels:             map[string]string{"key1": "val1"},
+				Annotations:        map[string]string{"key2": "val2"},
+			},
+			expected: &AllocationProperties{
+				AggregatedMetadata: true,
+				Namespace:          "ns1",
+				ControllerKind:     "",
+				Labels:             map[string]string{"key1": "val1"},
+				Annotations:        map[string]string{"key2": "val2"},
+			},
+		},
+	}
+
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+
+			actual := tc.allocationProps1.Intersection(tc.allocationProps2)
+
+			if !reflect.DeepEqual(actual, tc.expected) {
+				t.Fatalf("test case %s: expected %+v; got %+v", name, tc.expected, actual)
+			}
+		})
+	}
+}
 func TestGenerateKey(t *testing.T) {
 
 	customOwnerLabelConfig := NewLabelConfig()