[CORE-129] updates to get idle coeffs surfaced, also surface provider ID

Signed-off-by: Alex Meijer <ameijer@kubecost.com>

pkg/costmodel/costmodel.go

@@ -2343,6 +2343,7 @@ func (cm *CostModel) QueryAllocation(window kubecost.Window, resolution, step ti
 	// Set aggregation options and aggregate
 	opts := &kubecost.AllocationAggregationOptions{
 		IncludeProportionalAssetResourceCosts: includeProportionalAssetResourceCosts,
+		IdleByNode:                            idleByNode,
 	}
 
 	// Aggregate

pkg/kubecost/allocation.go

@@ -246,39 +246,53 @@ func (pva *PVAllocation) Equal(that *PVAllocation) bool {
 }
 
 type ProportionalAssetResourceCost struct {
-	ProportionalAssetResourceCostKey
+	Cluster       string  `json:"cluster"`
+	Node          string  `json:"node,omitempty"`
+	ProviderID    string  `json:"providerID,omitempty"`
 	CPUPercentage float64 `json:"cpuPercentage"`
 	GPUPercentage float64 `json:"gpuPercentage"`
 	RAMPercentage float64 `json:"ramPercentage"`
 }
 
-func (parc ProportionalAssetResourceCost) Key() ProportionalAssetResourceCostKey {
-	return parc.ProportionalAssetResourceCostKey
-}
+func (parc ProportionalAssetResourceCost) Key(insertByNode bool) string {
+	if insertByNode {
+		return parc.Cluster + "," + parc.Node
+	} else {
+		return parc.Cluster
+	}
 
-type ProportionalAssetResourceCostKey struct {
-	Cluster string `json:"cluster"`
-	Node    string `json:"node"`
 }
 
-type ProportionalAssetResourceCosts map[ProportionalAssetResourceCostKey]ProportionalAssetResourceCost
+type ProportionalAssetResourceCosts map[string]ProportionalAssetResourceCost
 
-func (parcs ProportionalAssetResourceCosts) Insert(parc ProportionalAssetResourceCost) {
-	if curr, ok := parcs[parc.Key()]; ok {
-		parcs[parc.Key()] = ProportionalAssetResourceCost{
-			ProportionalAssetResourceCostKey: parc.Key(),
-			CPUPercentage:                    curr.CPUPercentage + parc.CPUPercentage,
-			GPUPercentage:                    curr.GPUPercentage + parc.GPUPercentage,
-			RAMPercentage:                    curr.RAMPercentage + parc.RAMPercentage,
+func (parcs ProportionalAssetResourceCosts) Insert(parc ProportionalAssetResourceCost, insertByNode bool) {
+	if !insertByNode {
+		parc.Node = ""
+		parc.ProviderID = ""
+	}
+	if curr, ok := parcs[parc.Key(insertByNode)]; ok {
+		parcs[parc.Key(insertByNode)] = ProportionalAssetResourceCost{
+			Node:          curr.Node,
+			Cluster:       curr.Cluster,
+			ProviderID:    curr.ProviderID,
+			CPUPercentage: curr.CPUPercentage + parc.CPUPercentage,
+			GPUPercentage: curr.GPUPercentage + parc.GPUPercentage,
+			RAMPercentage: curr.RAMPercentage + parc.RAMPercentage,
 		}
 	} else {
-		parcs[parc.Key()] = parc
+		parcs[parc.Key(insertByNode)] = parc
 	}
 }
 
 func (parcs ProportionalAssetResourceCosts) Add(that ProportionalAssetResourceCosts) {
+
 	for _, parc := range that {
-		parcs.Insert(parc)
+		// if node field is empty, we know this is a cluster level PARC aggregation
+		insertByNode := true
+		if parc.Node == "" {
+			insertByNode = false
+		}
+		parcs.Insert(parc, insertByNode)
 	}
 }
 
@@ -1010,6 +1024,12 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 		Window: as.Window.Clone(),
 	}
 
+	// parcSet is used to compute proportionalAssetResourceCosts
+	// for surfacing in the API
+	parcSet := &AllocationSet{
+		Window: as.Window.Clone(),
+	}
+
 	// shareSet will be shared among aggSet after initial aggregation
 	// is complete
 	shareSet := &AllocationSet{
@@ -1043,6 +1063,12 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 				aggSet.Insert(alloc)
 			}
 
+			// build a parallel set of allocations to only be used
+			// for computing PARCs
+			if options.IncludeProportionalAssetResourceCosts {
+				parcSet.Insert(alloc.Clone())
+			}
+
 			continue
 		}
 
@@ -1110,14 +1136,23 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 		}
 	}
 
+	var parcCoefficients map[string]map[string]map[string]float64
+	if parcSet.Length() > 0 {
+		parcCoefficients, allocatedTotalsMap, err = computeIdleCoeffs(options, as, shareSet)
+		if err != nil {
+			log.Warnf("AllocationSet.AggregateBy: compute parc idle coeff: %s", err)
+			return fmt.Errorf("error computing parc coefficients: %s", err)
+		}
+	}
+
 	log.Infof("[PARCS] idleSet.Length(): %d", idleSet.Length())
-	log.Infof("[PARCS] idleCoefficients: nil:%t len:%d", idleCoefficients == nil, len(idleCoefficients))
+	log.Infof("[PARCS] parcCoefficients: nil:%t len:%d", parcCoefficients == nil, len(parcCoefficients))
 
 	// (2b) If proportional asset resource costs are to be included, derive them
 	// from idle coefficients and add them to the allocations.
 	if options.IncludeProportionalAssetResourceCosts {
-		if idleCoefficients == nil {
-			return fmt.Errorf("cannot include proportional resource costs because idle coefficients are nil")
+		if parcCoefficients == nil {
+			return fmt.Errorf("cannot include proportional resource costs because parc coefficients are nil")
 		}
 
 		for _, alloc := range as.Allocations {
@@ -1127,12 +1162,12 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 
 			// Attempt to derive proportional asset resource costs from idle
 			// coefficients, and insert them into the set if successful.
-			parc, err := deriveProportionalAssetResourceCostsFromIdleCoefficients(idleCoefficients, alloc, options)
+			parc, err := deriveProportionalAssetResourceCostsFromIdleCoefficients(parcCoefficients, alloc, options)
 			if err != nil {
 				log.Debugf("AggregateBy: failed to derive proportional asset resource costs from idle coefficients for %s: %s", alloc.Name, err)
 				continue
 			}
-			alloc.ProportionalAssetResourceCosts.Insert(parc)
+			alloc.ProportionalAssetResourceCosts.Insert(parc, options.IdleByNode)
 		}
 	}
 
@@ -1720,16 +1755,13 @@ func deriveProportionalAssetResourceCostsFromIdleCoefficients(idleCoeffs map[str
 	gpuPct := idleCoeffs[idleId][allocation.Name]["gpu"]
 	ramPct := idleCoeffs[idleId][allocation.Name]["ram"]
 
-	key := ProportionalAssetResourceCostKey{
-		Cluster: allocation.Properties.Cluster,
-		Node:    allocation.Properties.Node,
-	}
-
 	return ProportionalAssetResourceCost{
-		ProportionalAssetResourceCostKey: key,
-		CPUPercentage:                    cpuPct,
-		GPUPercentage:                    gpuPct,
-		RAMPercentage:                    ramPct,
+		Cluster:       allocation.Properties.Cluster,
+		Node:          allocation.Properties.Node,
+		ProviderID:    allocation.Properties.ProviderID,
+		CPUPercentage: cpuPct,
+		GPUPercentage: gpuPct,
+		RAMPercentage: ramPct,
 	}, nil
 }
 

pkg/kubecost/allocation_json.go

@@ -2,57 +2,59 @@ package kubecost
 
 import (
 	"fmt"
-	"github.com/opencost/opencost/pkg/util/json"
 	"math"
 	"time"
+
+	"github.com/opencost/opencost/pkg/util/json"
 )
 
 // AllocationJSON  exists because there are expected JSON response fields
 // that are calculated values from methods on an annotation
 type AllocationJSON struct {
-	Name                       string                 `json:"name"`
-	Properties                 *AllocationProperties  `json:"properties"`
-	Window                     Window                 `json:"window"`
-	Start                      string                 `json:"start"`
-	End                        string                 `json:"end"`
-	Minutes                    *float64               `json:"minutes"`
-	CPUCores                   *float64               `json:"cpuCores"`
-	CPUCoreRequestAverage      *float64               `json:"cpuCoreRequestAverage"`
-	CPUCoreUsageAverage        *float64               `json:"cpuCoreUsageAverage"`
-	CPUCoreHours               *float64               `json:"cpuCoreHours"`
-	CPUCost                    *float64               `json:"cpuCost"`
-	CPUCostAdjustment          *float64               `json:"cpuCostAdjustment"`
-	CPUEfficiency              *float64               `json:"cpuEfficiency"`
-	GPUCount                   *float64               `json:"gpuCount"`
-	GPUHours                   *float64               `json:"gpuHours"`
-	GPUCost                    *float64               `json:"gpuCost"`
-	GPUCostAdjustment          *float64               `json:"gpuCostAdjustment"`
-	NetworkTransferBytes       *float64               `json:"networkTransferBytes"`
-	NetworkReceiveBytes        *float64               `json:"networkReceiveBytes"`
-	NetworkCost                *float64               `json:"networkCost"`
-	NetworkCrossZoneCost       *float64               `json:"networkCrossZoneCost"`
-	NetworkCrossRegionCost     *float64               `json:"networkCrossRegionCost"`
-	NetworkInternetCost        *float64               `json:"networkInternetCost"`
-	NetworkCostAdjustment      *float64               `json:"networkCostAdjustment"`
-	LoadBalancerCost           *float64               `json:"loadBalancerCost"`
-	LoadBalancerCostAdjustment *float64               `json:"loadBalancerCostAdjustment"`
-	PVBytes                    *float64               `json:"pvBytes"`
-	PVByteHours                *float64               `json:"pvByteHours"`
-	PVCost                     *float64               `json:"pvCost"`
-	PVs                        PVAllocations          `json:"pvs"`
-	PVCostAdjustment           *float64               `json:"pvCostAdjustment"`
-	RAMBytes                   *float64               `json:"ramBytes"`
-	RAMByteRequestAverage      *float64               `json:"ramByteRequestAverage"`
-	RAMByteUsageAverage        *float64               `json:"ramByteUsageAverage"`
-	RAMByteHours               *float64               `json:"ramByteHours"`
-	RAMCost                    *float64               `json:"ramCost"`
-	RAMCostAdjustment          *float64               `json:"ramCostAdjustment"`
-	RAMEfficiency              *float64               `json:"ramEfficiency"`
-	ExternalCost               *float64               `json:"externalCost"`
-	SharedCost                 *float64               `json:"sharedCost"`
-	TotalCost                  *float64               `json:"totalCost"`
-	TotalEfficiency            *float64               `json:"totalEfficiency"`
-	RawAllocationOnly          *RawAllocationOnlyData `json:"rawAllocationOnly"`
+	Name                           string                          `json:"name"`
+	Properties                     *AllocationProperties           `json:"properties"`
+	Window                         Window                          `json:"window"`
+	Start                          string                          `json:"start"`
+	End                            string                          `json:"end"`
+	Minutes                        *float64                        `json:"minutes"`
+	CPUCores                       *float64                        `json:"cpuCores"`
+	CPUCoreRequestAverage          *float64                        `json:"cpuCoreRequestAverage"`
+	CPUCoreUsageAverage            *float64                        `json:"cpuCoreUsageAverage"`
+	CPUCoreHours                   *float64                        `json:"cpuCoreHours"`
+	CPUCost                        *float64                        `json:"cpuCost"`
+	CPUCostAdjustment              *float64                        `json:"cpuCostAdjustment"`
+	CPUEfficiency                  *float64                        `json:"cpuEfficiency"`
+	GPUCount                       *float64                        `json:"gpuCount"`
+	GPUHours                       *float64                        `json:"gpuHours"`
+	GPUCost                        *float64                        `json:"gpuCost"`
+	GPUCostAdjustment              *float64                        `json:"gpuCostAdjustment"`
+	NetworkTransferBytes           *float64                        `json:"networkTransferBytes"`
+	NetworkReceiveBytes            *float64                        `json:"networkReceiveBytes"`
+	NetworkCost                    *float64                        `json:"networkCost"`
+	NetworkCrossZoneCost           *float64                        `json:"networkCrossZoneCost"`
+	NetworkCrossRegionCost         *float64                        `json:"networkCrossRegionCost"`
+	NetworkInternetCost            *float64                        `json:"networkInternetCost"`
+	NetworkCostAdjustment          *float64                        `json:"networkCostAdjustment"`
+	LoadBalancerCost               *float64                        `json:"loadBalancerCost"`
+	LoadBalancerCostAdjustment     *float64                        `json:"loadBalancerCostAdjustment"`
+	PVBytes                        *float64                        `json:"pvBytes"`
+	PVByteHours                    *float64                        `json:"pvByteHours"`
+	PVCost                         *float64                        `json:"pvCost"`
+	PVs                            PVAllocations                   `json:"pvs"`
+	PVCostAdjustment               *float64                        `json:"pvCostAdjustment"`
+	RAMBytes                       *float64                        `json:"ramBytes"`
+	RAMByteRequestAverage          *float64                        `json:"ramByteRequestAverage"`
+	RAMByteUsageAverage            *float64                        `json:"ramByteUsageAverage"`
+	RAMByteHours                   *float64                        `json:"ramByteHours"`
+	RAMCost                        *float64                        `json:"ramCost"`
+	RAMCostAdjustment              *float64                        `json:"ramCostAdjustment"`
+	RAMEfficiency                  *float64                        `json:"ramEfficiency"`
+	ExternalCost                   *float64                        `json:"externalCost"`
+	SharedCost                     *float64                        `json:"sharedCost"`
+	TotalCost                      *float64                        `json:"totalCost"`
+	TotalEfficiency                *float64                        `json:"totalEfficiency"`
+	RawAllocationOnly              *RawAllocationOnlyData          `json:"rawAllocationOnly,omitEmpty"`
+	ProportionalAssetResourceCosts *ProportionalAssetResourceCosts `json:"proportionalAssetResourceCosts,omitEmpty"`
 }
 
 func (aj *AllocationJSON) BuildFromAllocation(a *Allocation) {
@@ -64,7 +66,7 @@ func (aj *AllocationJSON) BuildFromAllocation(a *Allocation) {
 	aj.Window = a.Window
 	aj.Start = a.Start.UTC().Format(time.RFC3339)
 	aj.End = a.End.UTC().Format(time.RFC3339)
-  aj.Minutes = formatFloat64ForResponse(a.Minutes())
+	aj.Minutes = formatFloat64ForResponse(a.Minutes())
 	aj.CPUCores = formatFloat64ForResponse(a.CPUCores())
 	aj.CPUCoreRequestAverage = formatFloat64ForResponse(a.CPUCoreRequestAverage)
 	aj.CPUCoreUsageAverage = formatFloat64ForResponse(a.CPUCoreUsageAverage)
@@ -102,6 +104,8 @@ func (aj *AllocationJSON) BuildFromAllocation(a *Allocation) {
 	aj.TotalCost = formatFloat64ForResponse(a.TotalCost())
 	aj.TotalEfficiency = formatFloat64ForResponse(a.TotalEfficiency())
 	aj.RawAllocationOnly = a.RawAllocationOnly
+	aj.ProportionalAssetResourceCosts = &a.ProportionalAssetResourceCosts
+
 }
 
 // formatFloat64ForResponse - take an existing float64, round it to 6 decimal places and return is possible, or return nil if invalid