generate shared cost breakdown and add to allocation struct (#1925)

* generate shared cost breakdown and add to allocation struct

Signed-off-by: nickcurie <nick.curie64@gmail.com>

* add breakdown to json

Signed-off-by: nickcurie <nick.curie64@gmail.com>

* breakdown clone func

Signed-off-by: nickcurie <nick.curie64@gmail.com>

* total cost field and overhead

Signed-off-by: nickcurie <nick.curie64@gmail.com>

* insert directly into allocation

Signed-off-by: nickcurie <nick.curie64@gmail.com>

* PR feedback

Signed-off-by: nickcurie <nick.curie64@gmail.com>

* prevent nil panic + unit tests

Signed-off-by: nickcurie <nick.curie64@gmail.com>

---------

Signed-off-by: nickcurie <nick.curie64@gmail.com>

pkg/kubecost/allocation.go

@@ -87,6 +87,7 @@ type Allocation struct {
 	// asset on which the allocation was run. It is optionally computed
 	// and appended to an Allocation, and so by default is is nil.
 	ProportionalAssetResourceCosts ProportionalAssetResourceCosts `json:"proportionalAssetResourceCosts"` //@bingen:field[ignore]
+	SharedCostBreakdown            SharedCostBreakdowns           `json:"sharedCostBreakdown"`            //@bingen:field[ignore]
 }
 
 // RawAllocationOnlyData is information that only belong in "raw" Allocations,
@@ -356,6 +357,53 @@ func (parcs ProportionalAssetResourceCosts) Add(that ProportionalAssetResourceCo
 	}
 }
 
+type SharedCostBreakdown struct {
+	Name         string  `json:"name"`
+	TotalCost    float64 `json:"totalCost"`
+	CPUCost      float64 `json:"cpuCost,omitempty"`
+	GPUCost      float64 `json:"gpuCost,omitempty"`
+	RAMCost      float64 `json:"ramCost,omitempty"`
+	PVCost       float64 `json:"pvCost,omitempty"`
+	NetworkCost  float64 `json:"networkCost,omitempty"`
+	LBCost       float64 `json:"loadBalancerCost,omitempty"`
+	ExternalCost float64 `json:"externalCost,omitempty"`
+}
+
+type SharedCostBreakdowns map[string]SharedCostBreakdown
+
+func (scbs SharedCostBreakdowns) Clone() SharedCostBreakdowns {
+	cloned := SharedCostBreakdowns{}
+
+	for key, scb := range scbs {
+		cloned[key] = scb
+	}
+	return cloned
+}
+
+func (scbs SharedCostBreakdowns) Insert(scb SharedCostBreakdown) {
+	if curr, ok := scbs[scb.Name]; ok {
+		scbs[scb.Name] = SharedCostBreakdown{
+			Name:         curr.Name,
+			TotalCost:    curr.TotalCost + scb.TotalCost,
+			CPUCost:      curr.CPUCost + scb.CPUCost,
+			GPUCost:      curr.GPUCost + scb.GPUCost,
+			RAMCost:      curr.RAMCost + scb.RAMCost,
+			PVCost:       curr.PVCost + scb.PVCost,
+			NetworkCost:  curr.NetworkCost + scb.NetworkCost,
+			LBCost:       curr.LBCost + scb.LBCost,
+			ExternalCost: curr.ExternalCost + scb.ExternalCost,
+		}
+	} else {
+		scbs[scb.Name] = scb
+	}
+}
+
+func (scbs SharedCostBreakdowns) Add(that SharedCostBreakdowns) {
+	for _, scb := range that {
+		scbs.Insert(scb)
+	}
+}
+
 // GetWindow returns the window of the struct
 func (a *Allocation) GetWindow() Window {
 	return a.Window
@@ -424,6 +472,7 @@ func (a *Allocation) Clone() *Allocation {
 		ExternalCost:                   a.ExternalCost,
 		RawAllocationOnly:              a.RawAllocationOnly.Clone(),
 		ProportionalAssetResourceCosts: a.ProportionalAssetResourceCosts.Clone(),
+		SharedCostBreakdown:            a.SharedCostBreakdown.Clone(),
 	}
 }
 
@@ -844,6 +893,18 @@ func (a *Allocation) add(that *Allocation) {
 		a.ProportionalAssetResourceCosts.Add(that.ProportionalAssetResourceCosts)
 	}
 
+	// If both Allocations have SharedCostBreakdowns, then
+	// add those from the given Allocation into the receiver.
+	if a.SharedCostBreakdown != nil || that.SharedCostBreakdown != nil {
+		if a.SharedCostBreakdown == nil {
+			a.SharedCostBreakdown = SharedCostBreakdowns{}
+		}
+		if that.SharedCostBreakdown == nil {
+			that.SharedCostBreakdown = SharedCostBreakdowns{}
+		}
+		a.SharedCostBreakdown.Add(that.SharedCostBreakdown)
+	}
+
 	// Overwrite regular intersection logic for the controller name property in the
 	// case that the Allocation keys are the same but the controllers are not.
 	if leftKey == rightKey &&
@@ -987,6 +1048,7 @@ type AllocationAggregationOptions struct {
 	ShareIdle                             string
 	ShareSplit                            string
 	SharedHourlyCosts                     map[string]float64
+	IncludeSharedCostBreakdown            bool
 	SplitIdle                             bool
 	IncludeAggregatedMetadata             bool
 }
@@ -1473,6 +1535,31 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 					continue
 				}
 
+				if options.IncludeSharedCostBreakdown {
+					if alloc.SharedCostBreakdown == nil {
+						alloc.SharedCostBreakdown = map[string]SharedCostBreakdown{}
+					}
+					sharedCostName := sharedAlloc.generateKey(aggregateBy, options.LabelConfig)
+					// check if current allocation is a shared flat overhead cost
+					if strings.Contains(sharedAlloc.Name, SharedSuffix) {
+						sharedCostName = "overheadCost"
+					}
+
+					scb := SharedCostBreakdown{
+						Name:         sharedCostName,
+						TotalCost:    sharedAlloc.TotalCost() * shareCoefficients[alloc.Name],
+						CPUCost:      sharedAlloc.CPUTotalCost() * shareCoefficients[alloc.Name],
+						GPUCost:      sharedAlloc.GPUTotalCost() * shareCoefficients[alloc.Name],
+						RAMCost:      sharedAlloc.RAMTotalCost() * shareCoefficients[alloc.Name],
+						PVCost:       sharedAlloc.PVCost() * shareCoefficients[alloc.Name],
+						NetworkCost:  sharedAlloc.NetworkTotalCost() * shareCoefficients[alloc.Name],
+						LBCost:       sharedAlloc.LBTotalCost() * shareCoefficients[alloc.Name],
+						ExternalCost: sharedAlloc.ExternalCost * shareCoefficients[alloc.Name],
+					}
+					// fmt.Printf("shared cost: %+v", scb)
+					alloc.SharedCostBreakdown.Insert(scb)
+				}
+
 				alloc.SharedCost += sharedAlloc.TotalCost() * shareCoefficients[alloc.Name]
 			}
 		}

pkg/kubecost/allocation_json.go

@@ -55,6 +55,7 @@ type AllocationJSON struct {
 	TotalEfficiency                *float64                        `json:"totalEfficiency"`
 	RawAllocationOnly              *RawAllocationOnlyData          `json:"rawAllocationOnly,omitempty"`
 	ProportionalAssetResourceCosts *ProportionalAssetResourceCosts `json:"proportionalAssetResourceCosts,omitempty"`
+	SharedCostBreakdown            *SharedCostBreakdowns           `json:"sharedCostBreakdown,omitempty"`
 }
 
 func (aj *AllocationJSON) BuildFromAllocation(a *Allocation) {
@@ -105,6 +106,7 @@ func (aj *AllocationJSON) BuildFromAllocation(a *Allocation) {
 	aj.TotalEfficiency = formatFloat64ForResponse(a.TotalEfficiency())
 	aj.RawAllocationOnly = a.RawAllocationOnly
 	aj.ProportionalAssetResourceCosts = &a.ProportionalAssetResourceCosts
+	aj.SharedCostBreakdown = &a.SharedCostBreakdown
 
 }
 

pkg/kubecost/allocation_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/opencost/opencost/pkg/log"
 	"github.com/opencost/opencost/pkg/util"
 	"github.com/opencost/opencost/pkg/util/json"
+	"github.com/opencost/opencost/pkg/util/timeutil"
 )
 
 func TestAllocation_Add(t *testing.T) {
@@ -1723,6 +1724,126 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 	}
 }
 
+func TestAllocationSet_AggregateBy_SharedCostBreakdown(t *testing.T) {
+	// Set generated by GenerateMockAllocationSet
+	// | Hierarchy                              | Cost |  CPU |  RAM |  GPU |   PV |  Net |  LB  |
+	// +----------------------------------------+------+------+------+------+------+------+------+
+	//   cluster1:
+	//     idle:                                  20.00   5.00  15.00   0.00   0.00   0.00   0.00
+	//     namespace1:
+	//       pod1:
+	//         container1: [app1, env1]   16.00   1.00  11.00   1.00   1.00   1.00   1.00
+	//       pod-abc: (deployment1)
+	//         container2:                         6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	//       pod-def: (deployment1)
+	//         container3:                         6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	//     namespace2:
+	//       pod-ghi: (deployment2)
+	//         container4: [app2, env2]    6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	//         container5: [app2, env2]    6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	//       pod-jkl: (daemonset1)
+	//         container6: {service1}              6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	// +-----------------------------------------+------+------+------+------+------+------+------+
+	//   cluster1 subtotal                        66.00  11.00  31.00   6.00   6.00   6.00   6.00
+	// +-----------------------------------------+------+------+------+------+------+------+------+
+	//   cluster2:
+	//     idle:                                  10.00   5.00   5.00   0.00   0.00   0.00   0.00
+	//     namespace2:
+	//       pod-mno: (deployment2)
+	//         container4: [app2]              6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	//         container5: [app2]              6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	//       pod-pqr: (daemonset1)
+	//         container6: {service1}              6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	//     namespace3:
+	//       pod-stu: (deployment3)
+	//         container7: an[team1]          6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	//       pod-vwx: (statefulset1)
+	//         container8: an[team2]          6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	//         container9: an[team1]          6.00   1.00   1.00   1.00   1.00   1.00   1.00
+	// +----------------------------------------+------+------+------+------+------+------+------+
+	//   cluster2 subtotal                        46.00  11.00  11.00   6.00   6.00   6.00   6.00
+	// +----------------------------------------+------+------+------+------+------+------+------+
+	//   total                                   112.00  22.00  42.00  12.00  12.00  12.00  12.00
+	// +----------------------------------------+------+------+------+------+------+------+------+
+	end := time.Now().UTC().Truncate(day)
+	start := end.Add(-day)
+
+	isNamespace1 := func(a *Allocation) bool {
+		ns := a.Properties.Namespace
+		return ns == "namespace1"
+	}
+
+	isNamespace3 := func(a *Allocation) bool {
+		ns := a.Properties.Namespace
+		return ns == "namespace3"
+	}
+
+	cases := map[string]struct {
+		start   time.Time
+		aggBy   []string
+		aggOpts *AllocationAggregationOptions
+	}{
+		"agg cluster, flat shared cost": {
+			start: start,
+			aggBy: []string{"cluster"},
+			aggOpts: &AllocationAggregationOptions{
+				SharedHourlyCosts:          map[string]float64{"share_hourly": 10.0 / timeutil.HoursPerDay},
+				IncludeSharedCostBreakdown: true,
+			},
+		},
+		"agg namespace, shared namespace: namespace1": {
+			start: start,
+			aggBy: []string{"namespace"},
+			aggOpts: &AllocationAggregationOptions{
+				ShareFuncs: []AllocationMatchFunc{
+					isNamespace1,
+				},
+				IncludeSharedCostBreakdown: true,
+			},
+		},
+		"agg namespace, shared namespace: namespace3": {
+			start: start,
+			aggBy: []string{"namespace"},
+			aggOpts: &AllocationAggregationOptions{
+				ShareFuncs: []AllocationMatchFunc{
+					isNamespace3,
+				},
+				IncludeSharedCostBreakdown: true,
+			},
+		},
+	}
+
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			as := GenerateMockAllocationSetClusterIdle(tc.start)
+			err := as.AggregateBy(tc.aggBy, tc.aggOpts)
+			if err != nil {
+				t.Fatalf("error aggregating: %s", err)
+			}
+			for _, alloc := range as.Allocations {
+				var breakdownTotal float64
+				// ignore idle since it should never have shared costs
+				if strings.Contains(alloc.Name, IdleSuffix) {
+					continue
+				}
+				for _, sharedAlloc := range alloc.SharedCostBreakdown {
+					breakdownTotal += sharedAlloc.TotalCost
+					totalInternal := sharedAlloc.CPUCost + sharedAlloc.GPUCost + sharedAlloc.RAMCost + sharedAlloc.NetworkCost + sharedAlloc.LBCost + sharedAlloc.PVCost + sharedAlloc.ExternalCost
+					// check that the total cost of a single item in the breakdown equals the sum of its parts
+					// we can ignore the overheadCost breakdown since it only has a total
+					if totalInternal != sharedAlloc.TotalCost && sharedAlloc.Name != "overheadCost" {
+						t.Errorf("expected internal total: %f; got %f", sharedAlloc.TotalCost, totalInternal)
+					}
+				}
+				// check that the totals of all shared cost breakdowns equal the allocation's SharedCost
+				if breakdownTotal != alloc.SharedCost {
+					t.Errorf("expected breakdown total: %f; got %f", alloc.SharedCost, breakdownTotal)
+				}
+			}
+		})
+	}
+}
+
 // TODO niko/etl
 //func TestAllocationSet_Clone(t *testing.T) {}