re-write to pass through allocations, not assets

Signed-off-by: Alex Meijer <ameijer@kubecost.com>
(cherry picked from commit a1440622c6fb4462c11369166ab21be0d24f6aad)
(cherry picked from commit 71257eaefa7a0e03fe511993276fd250ab493d85)
Signed-off-by: Alex Meijer <ameijer@kubecost.com>

pkg/costmodel/allocation.go

@@ -38,6 +38,7 @@ const (
 	queryFmtPVActiveMins                = `count(kube_persistentvolume_capacity_bytes{%s}) by (persistentvolume, %s)[%s:%s]`
 	queryFmtPVBytes                     = `avg(avg_over_time(kube_persistentvolume_capacity_bytes{%s}[%s])) by (persistentvolume, %s)`
 	queryFmtPVCostPerGiBHour            = `avg(avg_over_time(pv_hourly_cost{%s}[%s])) by (volumename, %s)`
+	queryFmtPVMeta                      = `avg(avg_over_time(kubecost_pv_info{%s}[%s])) by (%s, persistentvolume, provider_id)`
 	queryFmtNetZoneGiB                  = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="true", %s}[%s])) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
 	queryFmtNetZoneCostPerGiB           = `avg(avg_over_time(kubecost_network_zone_egress_cost{%s}[%s])) by (%s)`
 	queryFmtNetRegionGiB                = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="false", %s}[%s])) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
@@ -458,6 +459,9 @@ func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Dur
 	queryPVCostPerGiBHour := fmt.Sprintf(queryFmtPVCostPerGiBHour, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChPVCostPerGiBHour := ctx.QueryAtTime(queryPVCostPerGiBHour, end)
 
+	queryPVMeta := fmt.Sprintf(queryFmtPVMeta, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	resChPVMeta := ctx.QueryAtTime(queryPVMeta, end)
+
 	queryNetTransferBytes := fmt.Sprintf(queryFmtNetTransferBytes, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChNetTransferBytes := ctx.QueryAtTime(queryNetTransferBytes, end)
 
@@ -549,6 +553,7 @@ func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Dur
 	resPVActiveMins, _ := resChPVActiveMins.Await()
 	resPVBytes, _ := resChPVBytes.Await()
 	resPVCostPerGiBHour, _ := resChPVCostPerGiBHour.Await()
+	resPVMeta, _ := resChPVMeta.Await()
 
 	resPVCInfo, _ := resChPVCInfo.Await()
 	resPVCBytesRequested, _ := resChPVCBytesRequested.Await()
@@ -654,7 +659,7 @@ func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Dur
 	// a PVC, we get time running there, so this is only inaccurate
 	// for short-lived, unmounted PVs.)
 	pvMap := map[pvKey]*pv{}
-	buildPVMap(resolution, pvMap, resPVCostPerGiBHour, resPVActiveMins, window)
+	buildPVMap(resolution, pvMap, resPVCostPerGiBHour, resPVActiveMins, resPVMeta, window)
 	applyPVBytes(pvMap, resPVBytes)
 
 	// Build out the map of all PVCs with time running, bytes requested,

pkg/costmodel/allocation_helpers.go

@@ -1791,7 +1791,7 @@ func (cm *CostModel) getNodePricing(nodeMap map[nodeKey]*nodePricing, nodeKey no
 
 /* PV/PVC Helpers */
 
-func buildPVMap(resolution time.Duration, pvMap map[pvKey]*pv, resPVCostPerGiBHour, resPVActiveMins []*prom.QueryResult, window kubecost.Window) {
+func buildPVMap(resolution time.Duration, pvMap map[pvKey]*pv, resPVCostPerGiBHour, resPVActiveMins, resPVMeta []*prom.QueryResult, window kubecost.Window) {
 	for _, result := range resPVActiveMins {
 		key, err := resultPVKey(result, env.GetPromClusterLabel(), "persistentvolume")
 		if err != nil {
@@ -1828,6 +1828,25 @@ func buildPVMap(resolution time.Duration, pvMap map[pvKey]*pv, resPVCostPerGiBHo
 		pvMap[key].CostPerGiBHour = result.Values[0].Value
 
 	}
+
+	for _, result := range resPVMeta {
+		key, err := resultPVKey(result, env.GetPromClusterLabel(), "persistentvolume")
+		if err != nil {
+			log.Warnf("error getting key for PV: %v", err)
+			continue
+		}
+
+		// only add metadata for disks that exist in the other metrics
+		if _, ok := pvMap[key]; ok {
+			provId, err := result.GetString("provider_id")
+			if err != nil {
+				log.Warnf("error getting provider id for PV %v: %v", key, err)
+				continue
+			}
+			pvMap[key].ProviderID = provId
+		}
+
+	}
 }
 
 func applyPVBytes(pvMap map[pvKey]*pv, resPVBytes []*prom.QueryResult) {
@@ -2072,8 +2091,9 @@ func applyPVCsToPods(window kubecost.Window, podMap map[podKey]*pod, podPVCMap m
 				// would be equal to the values of the original pv
 				count := float64(len(pod.Allocations))
 				alloc.PVs[pvKey] = &kubecost.PVAllocation{
-					ByteHours: byteHours * coef / count,
-					Cost:      cost * coef / count,
+					ByteHours:  byteHours * coef / count,
+					Cost:       cost * coef / count,
+					ProviderID: pvc.Volume.ProviderID,
 				}
 			}
 		}

pkg/costmodel/allocation_types.go

@@ -132,57 +132,7 @@ type pv struct {
 	Cluster        string    `json:"cluster"`
 	Name           string    `json:"name"`
 	StorageClass   string    `json:"storageClass"`
-}
-
-func (p *pv) clone() *pv {
-	if p == nil {
-		return nil
-	}
-	return &pv{
-		Start:          p.Start,
-		End:            p.End,
-		Bytes:          p.Bytes,
-		CostPerGiBHour: p.CostPerGiBHour,
-		Cluster:        p.Cluster,
-		Name:           p.Name,
-		StorageClass:   p.StorageClass,
-	}
-}
-
-func (p *pv) equal(that *pv) bool {
-	if p == nil {
-		return that == nil
-	}
-
-	if !p.Start.Equal(that.Start) {
-		return false
-	}
-
-	if !p.End.Equal(that.End) {
-		return false
-	}
-
-	if p.Bytes != that.Bytes {
-		return false
-	}
-
-	if p.CostPerGiBHour != that.CostPerGiBHour {
-		return false
-	}
-
-	if p.Cluster != that.Cluster {
-		return false
-	}
-
-	if p.Name != that.Name {
-		return false
-	}
-
-	if p.StorageClass != that.StorageClass {
-		return false
-	}
-
-	return true
+	ProviderID     string    `json:"providerID"`
 }
 
 // String returns a string representation of the pv
@@ -190,7 +140,7 @@ func (p *pv) String() string {
 	if p == nil {
 		return "<nil>"
 	}
-	return fmt.Sprintf("%s/%s{Bytes:%.2f, Cost/GiB*Hr:%.6f, StorageClass:%s}", p.Cluster, p.Name, p.Bytes, p.CostPerGiBHour, p.StorageClass)
+	return fmt.Sprintf("%s/%s{Bytes:%.2f, Cost/GiB*Hr:%.6f, StorageClass:%s, ProviderID: %s}", p.Cluster, p.Name, p.Bytes, p.CostPerGiBHour, p.StorageClass, p.ProviderID)
 }
 
 func (p *pv) minutes() float64 {
@@ -201,11 +151,6 @@ func (p *pv) minutes() float64 {
 	return p.End.Sub(p.Start).Minutes()
 }
 
-// key returns the pvKey for the calling pvc
-func (p *pv) key() pvKey {
-	return newPVKey(p.Cluster, p.Name)
-}
-
 // lbCost describes the start and end time of a Load Balancer along with cost
 type lbCost struct {
 	TotalCost float64

pkg/kubecost/allocation.go

@@ -213,8 +213,9 @@ func (pv PVAllocations) Clone() PVAllocations {
 	clonePV := make(map[PVKey]*PVAllocation, len(pv))
 	for k, v := range pv {
 		clonePV[k] = &PVAllocation{
-			ByteHours: v.ByteHours,
-			Cost:      v.Cost,
+			ByteHours:  v.ByteHours,
+			Cost:       v.Cost,
+			ProviderID: v.ProviderID,
 		}
 	}
 	return clonePV
@@ -237,6 +238,11 @@ func (pv PVAllocations) Add(that PVAllocations) PVAllocations {
 			apvAlloc.Cost += thatPVAlloc.Cost
 			apvAlloc.ByteHours += thatPVAlloc.ByteHours
 			apv[pvKey] = apvAlloc
+			if apvAlloc.ProviderID == thatPVAlloc.ProviderID {
+				apv[pvKey].ProviderID = apvAlloc.ProviderID
+			} else {
+				apv[pvKey].ProviderID = ""
+			}
 		}
 	}
 	return apv
@@ -296,8 +302,9 @@ func (pvk *PVKey) FromString(key string) error {
 // PVAllocation contains the byte hour usage
 // and cost of an Allocation for a single PV
 type PVAllocation struct {
-	ByteHours float64 `json:"byteHours"`
-	Cost      float64 `json:"cost"`
+	ByteHours  float64 `json:"byteHours"`
+	Cost       float64 `json:"cost"`
+	ProviderID string  `json:"providerID"` // @bingen:field[version=20]
 }
 
 // Equal returns true if the two PVAllocation instances contain approximately the same
@@ -2350,6 +2357,7 @@ func deriveProportionalAssetResourceCosts(options *AllocationAggregationOptions,
 				Cluster:            name.Cluster,
 				Name:               name.Name,
 				Type:               "PV",
+				ProviderID:         pvAlloc.ProviderID,
 				PVProportionalCost: pvAlloc.Cost,
 			}, options.IdleByNode)
 		}

pkg/kubecost/bingen.go

@@ -46,7 +46,7 @@ package kubecost
 // @bingen:end
 
 // Allocation Version Set: Includes Allocation pipeline specific resources
-// @bingen:set[name=Allocation,version=19]
+// @bingen:set[name=Allocation,version=20]
 // @bingen:generate:Allocation
 // @bingen:generate[stringtable]:AllocationSet
 // @bingen:generate:AllocationSetRange

pkg/kubecost/kubecost_codecs.go

@@ -13,12 +13,11 @@ package kubecost
 
 import (
 	"fmt"
+	util "github.com/opencost/opencost/pkg/util"
 	"reflect"
 	"strings"
 	"sync"
 	"time"
-
-	util "github.com/opencost/opencost/pkg/util"
 )
 
 const (
@@ -34,17 +33,17 @@ const (
 )
 
 const (
+	// DefaultCodecVersion is used for any resources listed in the Default version set
+	DefaultCodecVersion uint8 = 17
+
 	// AssetsCodecVersion is used for any resources listed in the Assets version set
 	AssetsCodecVersion uint8 = 21
 
 	// AllocationCodecVersion is used for any resources listed in the Allocation version set
-	AllocationCodecVersion uint8 = 19
+	AllocationCodecVersion uint8 = 20
 
 	// CloudCostCodecVersion is used for any resources listed in the CloudCost version set
 	CloudCostCodecVersion uint8 = 2
-
-	// DefaultCodecVersion is used for any resources listed in the Default version set
-	DefaultCodecVersion uint8 = 17
 )
 
 //--------------------------------------------------------------------------
@@ -6633,6 +6632,12 @@ func (target *PVAllocation) MarshalBinaryWithContext(ctx *EncodingContext) (err
 
 	buff.WriteFloat64(target.ByteHours) // write float64
 	buff.WriteFloat64(target.Cost)      // write float64
+	if ctx.IsStringTable() {
+		a := ctx.Table.AddOrGet(target.ProviderID)
+		buff.WriteInt(a) // write table index
+	} else {
+		buff.WriteString(target.ProviderID) // write string
+	}
 	return nil
 }
 
@@ -6696,6 +6701,22 @@ func (target *PVAllocation) UnmarshalBinaryWithContext(ctx *DecodingContext) (er
 	b := buff.ReadFloat64() // read float64
 	target.Cost = b
 
+	// field version check
+	if uint8(20) <= version {
+		var d string
+		if ctx.IsStringTable() {
+			e := buff.ReadInt() // read string index
+			d = ctx.Table[e]
+		} else {
+			d = buff.ReadString() // read string
+		}
+		c := d
+		target.ProviderID = c
+
+	} else {
+		target.ProviderID = "" // default
+	}
+
 	return nil
 }