Merge branch 'develop' into cache-optimization

Signed-off-by: r2k1 <yokree@gmail.com>

Dockerfile

@@ -50,5 +50,6 @@ ADD --chmod=644 ./configs/aws.json /models/aws.json
 ADD --chmod=644 ./configs/gcp.json /models/gcp.json
 ADD --chmod=644 ./configs/alibaba.json /models/alibaba.json
 ADD --chmod=644 ./configs/oracle.json /models/oracle.json
+ADD --chmod=644 ./configs/otc.json /models/otc.json
 USER 1001
 ENTRYPOINT ["/go/bin/app"]

Dockerfile.cross

@@ -23,6 +23,7 @@ ADD --chmod=644 ./configs/aws.json /models/aws.json
 ADD --chmod=644 ./configs/gcp.json /models/gcp.json
 ADD --chmod=644 ./configs/alibaba.json /models/alibaba.json
 ADD --chmod=644 ./configs/oracle.json /models/oracle.json
+ADD --chmod=644 ./configs/otc.json /models/otc.json
 
 COPY ${binarypath} /go/bin/app
 

Dockerfile.debug

@@ -22,6 +22,7 @@ ADD --chmod=644 ./configs/aws.json /models/aws.json
 ADD --chmod=644 ./configs/gcp.json /models/gcp.json
 ADD --chmod=644 ./configs/alibaba.json /models/alibaba.json
 ADD --chmod=644 ./configs/oracle.json /models/oracle.json
+ADD --chmod=644 ./configs/otc.json /models/otc.json
 
 COPY ${binary_path} main
 

MAINTAINERS.md

@@ -9,6 +9,7 @@ Official list of [OpenCost Maintainers](https://github.com/orgs/opencost/teams/o
 | Ajay Tripathy | @AjayTripathy | Kubecost | <Ajay@kubecost.com> |
 | Alex Meijer | @ameijer | Kubecost | <ameijer@kubecost.com> |
 | Artur Khantimirov | @r2k1 | Microsoft | <akhantimirov@microsoft.com> |
+| Cliff Colvin | @cliffcolvin | Kubecost | <ccolvin@kubecost.com> |
 | Matt Bolt | @​mbolt35 | Kubecost | <matt@kubecost.com> |
 | Niko Kovacevic | @nikovacevic | Kubecost | <niko@kubecost.com> |
 | Sean Holcomb | @Sean-Holcomb | Kubecost | <Sean@kubecost.com> |

README.md

@@ -1,5 +1,6 @@
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/6219/badge)](https://www.bestpractices.dev/projects/6219)
+[![Gurubase](https://img.shields.io/badge/Gurubase-Ask%20OpenCost%20Guru-006BFF)](https://gurubase.io/g/opencost)
 
 ![](./opencost-header.png)
 

configs/otc.json

@@ -0,0 +1,10 @@
+{
+    "provider": "OTC",
+    "description": "Default prices used to compute allocation between RAM and CPU. OTC pricing API data still used for total node cost.",
+    "CPU": "0.031611",
+    "RAM": "0.004237",
+    "storage": "0.0",
+    "zoneNetworkEgress": "0.0",
+    "regionNetworkEgress": "0.0",
+    "internetNetworkEgress": "0.0"
+}

core/go.mod

@@ -1,6 +1,6 @@
 module github.com/opencost/opencost/core
 
-go 1.22.0
+go 1.22.7
 
 require (
 	github.com/davecgh/go-spew v1.1.1
@@ -16,7 +16,7 @@ require (
 	golang.org/x/sync v0.6.0
 	golang.org/x/text v0.14.0
 	google.golang.org/grpc v1.62.0
-	google.golang.org/protobuf v1.32.0
+	google.golang.org/protobuf v1.33.0
 	k8s.io/api v0.25.3
 	k8s.io/apimachinery v0.25.3
 )
@@ -47,8 +47,8 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
-	golang.org/x/net v0.21.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

core/go.sum

@@ -386,8 +386,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -463,8 +463,8 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -641,8 +641,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
-google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=

core/pkg/opencost/allocation.go

@@ -100,9 +100,64 @@ type Allocation struct {
 	// UnmountedPVCost is used to track how much of the cost in PVs is for an
 	// unmounted PV. It is not additive of PVCost() and need not be sent in API
 	// responses.
-	UnmountedPVCost   float64 `json:"-"`                 //@bingen:field[ignore]
-	GPURequestAverage float64 `json:"gpuRequestAverage"` //@bingen:field[version=22]
-	GPUUsageAverage   float64 `json:"gpuUsageAverage"`   //@bingen:field[version=22]
+	UnmountedPVCost             float64        `json:"-"`             //@bingen:field[ignore]
+	deprecatedGPURequestAverage float64        `json:"-"`             //@bingen:field[version=22]
+	deprecatedGPUUsageAverage   float64        `json:"-"`             //@bingen:field[version=22]
+	GPUAllocation               *GPUAllocation `json:"GPUAllocation"` //@bingen:field[version=23]
+}
+
+type GPUAllocation struct {
+	GPUDevice string `json:"gpuDevice,omitempty"`
+	GPUModel  string `json:"gpuModel,omitempty"`
+	GPUUUID   string `json:"gpuUUID,omitempty"`
+
+	IsGPUShared       *bool    `json:"isGPUShared"`
+	GPUUsageAverage   *float64 `json:"gpuUsageAverage"`
+	GPURequestAverage *float64 `json:"gpuRequestAverage"`
+}
+
+func (orig *GPUAllocation) SanitizeNaN() {
+	if orig == nil {
+		return
+	}
+	if orig.GPURequestAverage == nil || math.IsNaN(*orig.GPURequestAverage) {
+		orig.GPURequestAverage = nil
+	}
+	if orig.GPUUsageAverage == nil || math.IsNaN(*orig.GPUUsageAverage) {
+		orig.GPUUsageAverage = nil
+	}
+}
+
+func (orig *GPUAllocation) Clone() *GPUAllocation {
+	if orig == nil {
+		return nil
+	}
+
+	return &GPUAllocation{
+		GPUDevice:         orig.GPUDevice,
+		GPUModel:          orig.GPUModel,
+		GPUUUID:           orig.GPUUUID,
+		IsGPUShared:       orig.IsGPUShared,
+		GPUUsageAverage:   orig.GPUUsageAverage,
+		GPURequestAverage: orig.GPURequestAverage,
+	}
+}
+
+func (orig *GPUAllocation) Equal(that *GPUAllocation) bool {
+	if orig == nil && that == nil {
+		return true
+	}
+	if orig == nil || that == nil {
+		return false
+	}
+
+	return orig.GPUDevice == that.GPUDevice &&
+		orig.GPUModel == that.GPUModel &&
+		orig.GPUUUID == that.GPUUUID &&
+		orig.IsGPUShared == that.IsGPUShared &&
+		orig.GPUUsageAverage == that.GPUUsageAverage &&
+		orig.GPURequestAverage == that.GPURequestAverage
+
 }
 
 type LbAllocations map[string]*LbAllocation
@@ -174,8 +229,9 @@ func (lba *LbAllocation) SanitizeNaN() {
 // then this type would be unnecessary and its fields would go into the regular Allocation
 // and not in the AggregatedAllocation.
 type RawAllocationOnlyData struct {
-	CPUCoreUsageMax  float64 `json:"cpuCoreUsageMax"`
-	RAMBytesUsageMax float64 `json:"ramByteUsageMax"`
+	CPUCoreUsageMax  float64  `json:"cpuCoreUsageMax"`
+	RAMBytesUsageMax float64  `json:"ramByteUsageMax"`
+	GPUUsageMax      *float64 `json:"gpuUsageMax"` //@bingen:field[version=23]
 }
 
 // Clone returns a deep copy of the given RawAllocationOnlyData
@@ -187,6 +243,7 @@ func (r *RawAllocationOnlyData) Clone() *RawAllocationOnlyData {
 	return &RawAllocationOnlyData{
 		CPUCoreUsageMax:  r.CPUCoreUsageMax,
 		RAMBytesUsageMax: r.RAMBytesUsageMax,
+		GPUUsageMax:      r.GPUUsageMax,
 	}
 }
 
@@ -198,8 +255,16 @@ func (r *RawAllocationOnlyData) Equal(that *RawAllocationOnlyData) bool {
 	if r == nil || that == nil {
 		return false
 	}
-	return util.IsApproximately(r.CPUCoreUsageMax, that.CPUCoreUsageMax) &&
+	cmpResult := util.IsApproximately(r.CPUCoreUsageMax, that.CPUCoreUsageMax) &&
 		util.IsApproximately(r.RAMBytesUsageMax, that.RAMBytesUsageMax)
+
+	if r.GPUUsageMax != nil && that.GPUUsageMax != nil {
+		cmpResult = cmpResult && util.IsApproximately(*r.GPUUsageMax, *that.GPUUsageMax)
+	} else if !(r.GPUUsageMax == nil && that.GPUUsageMax == nil) {
+		cmpResult = false
+	}
+
+	return cmpResult
 }
 
 func (r *RawAllocationOnlyData) SanitizeNaN() {
@@ -214,6 +279,10 @@ func (r *RawAllocationOnlyData) SanitizeNaN() {
 		log.DedupedWarningf(5, "RawAllocationOnlyData: Unexpected NaN found for RAMBytesUsageMax")
 		r.RAMBytesUsageMax = 0
 	}
+	if r.GPUUsageMax != nil && math.IsNaN(*r.GPUUsageMax) {
+		log.DedupedWarningf(5, "RawAllocationOnlyData: Unexpected NaN found for GPUUsageMax")
+		r.GPUUsageMax = nil
+	}
 }
 
 // PVAllocations is a map of Disk Asset Identifiers to the
@@ -675,8 +744,8 @@ func (a *Allocation) Clone() *Allocation {
 		CPUCostIdle:                    a.CPUCostIdle,
 		CPUCostAdjustment:              a.CPUCostAdjustment,
 		GPUHours:                       a.GPUHours,
-		GPURequestAverage:              a.GPURequestAverage,
-		GPUUsageAverage:                a.GPUUsageAverage,
+		deprecatedGPURequestAverage:    a.deprecatedGPURequestAverage,
+		deprecatedGPUUsageAverage:      a.deprecatedGPUUsageAverage,
 		GPUCost:                        a.GPUCost,
 		GPUCostIdle:                    a.GPUCostIdle,
 		GPUCostAdjustment:              a.GPUCostAdjustment,
@@ -704,6 +773,7 @@ func (a *Allocation) Clone() *Allocation {
 		SharedCostBreakdown:            a.SharedCostBreakdown.Clone(),
 		LoadBalancers:                  a.LoadBalancers.Clone(),
 		UnmountedPVCost:                a.UnmountedPVCost,
+		GPUAllocation:                  a.GPUAllocation.Clone(),
 	}
 }
 
@@ -816,6 +886,10 @@ func (a *Allocation) Equal(that *Allocation) bool {
 		return false
 	}
 
+	if !a.GPUAllocation.Equal(that.GPUAllocation) {
+		return false
+	}
+
 	return true
 }
 
@@ -963,17 +1037,24 @@ func (a *Allocation) RAMEfficiency() float64 {
 
 // GPUEfficiency is the ratio of usage to request. Note that, without the NVIDIA
 // DCGM exporter providing Prometheus with usage metrics, this will always be
-// zero, as GPUUsageAverage will be zero (the default value).
+// zero, as deprecatedGPUUsageAverage will be zero (the default value).
 func (a *Allocation) GPUEfficiency() float64 {
 	if a == nil {
 		return 0.0
 	}
+	if a.GPUAllocation == nil {
+		return 0.0
+	}
+
+	if a.GPUAllocation.GPURequestAverage == nil || a.GPUAllocation.GPUUsageAverage == nil {
+		return 0.0
+	}
 
-	if a.GPURequestAverage > 0 && a.GPUUsageAverage > 0 {
-		return a.GPUUsageAverage / a.GPURequestAverage
+	if *a.GPUAllocation.GPURequestAverage > 0 && *a.GPUAllocation.GPUUsageAverage > 0 {
+		return *a.GPUAllocation.GPUUsageAverage / *a.GPUAllocation.GPURequestAverage
 	}
 
-	if a.GPUUsageAverage == 0.0 || a.GPUTotalCost() == 0.0 {
+	if *a.GPUAllocation.GPURequestAverage == 0.0 || a.GPUTotalCost() == 0.0 {
 		return 0.0
 	}
 
@@ -1221,11 +1302,37 @@ func (a *Allocation) add(that *Allocation) {
 	ramUseByteMins := a.RAMBytesUsageAverage * a.Minutes()
 	ramUseByteMins += that.RAMBytesUsageAverage * that.Minutes()
 
-	gpuReqMins := a.GPURequestAverage * a.Minutes()
-	gpuReqMins += that.GPURequestAverage * that.Minutes()
+	var gpuReqMins *float64 = nil
+	if a.GPUAllocation != nil && a.GPUAllocation.GPURequestAverage != nil {
+		result := *a.GPUAllocation.GPURequestAverage * a.Minutes()
+		gpuReqMins = &result
+	}
+
+	if that.GPUAllocation != nil && that.GPUAllocation.GPURequestAverage != nil {
+		if gpuReqMins == nil {
+			result := *that.GPUAllocation.GPURequestAverage * that.Minutes()
+			gpuReqMins = &result
+		} else {
+			result := *gpuReqMins + *that.GPUAllocation.GPURequestAverage*that.Minutes()
+			gpuReqMins = &result
+		}
+	}
+
+	var gpuUseMins *float64 = nil
+	if a.GPUAllocation != nil && a.GPUAllocation.GPUUsageAverage != nil {
+		result := *a.GPUAllocation.GPUUsageAverage * a.Minutes()
+		gpuUseMins = &result
+	}
 
-	gpuUseMins := a.GPUUsageAverage * a.Minutes()
-	gpuUseMins += that.GPUUsageAverage * that.Minutes()
+	if that.GPUAllocation != nil && that.GPUAllocation.GPUUsageAverage != nil {
+		if gpuUseMins == nil {
+			result := *that.GPUAllocation.GPUUsageAverage * that.Minutes()
+			gpuUseMins = &result
+		} else {
+			result := *gpuUseMins + *that.GPUAllocation.GPUUsageAverage*that.Minutes()
+			gpuUseMins = &result
+		}
+	}
 
 	// Expand Start and End to be the "max" of among the given Allocations
 	if that.Start.Before(a.Start) {
@@ -1242,15 +1349,32 @@ func (a *Allocation) add(that *Allocation) {
 		a.CPUCoreUsageAverage = cpuUseCoreMins / a.Minutes()
 		a.RAMBytesRequestAverage = ramReqByteMins / a.Minutes()
 		a.RAMBytesUsageAverage = ramUseByteMins / a.Minutes()
-		a.GPURequestAverage = gpuReqMins / a.Minutes()
-		a.GPUUsageAverage = gpuUseMins / a.Minutes()
+
+		if a.GPUAllocation != nil {
+			if gpuReqMins != nil {
+				gpuReqMinsRes := *gpuReqMins / a.Minutes()
+				a.GPUAllocation.GPURequestAverage = &gpuReqMinsRes
+			} else {
+				a.GPUAllocation.GPURequestAverage = nil
+			}
+
+			if gpuUseMins != nil {
+				gpuUsageMinsRes := *gpuUseMins / a.Minutes()
+				a.GPUAllocation.GPUUsageAverage = &gpuUsageMinsRes
+			} else {
+				a.GPUAllocation.GPUUsageAverage = nil
+			}
+		}
 	} else {
 		a.CPUCoreRequestAverage = 0.0
 		a.CPUCoreUsageAverage = 0.0
 		a.RAMBytesRequestAverage = 0.0
 		a.RAMBytesUsageAverage = 0.0
-		a.GPURequestAverage = 0.0
-		a.GPUUsageAverage = 0.0
+
+		if a.GPUAllocation != nil {
+			a.GPUAllocation.GPURequestAverage = nil
+			a.GPUAllocation.GPUUsageAverage = nil
+		}
 	}
 
 	// Sum all cumulative resource fields
@@ -2587,14 +2711,7 @@ func (a *Allocation) SanitizeNaN() {
 		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for GPUHours name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
 		a.GPUHours = 0
 	}
-	if math.IsNaN(a.GPURequestAverage) {
-		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for GPURequestAverage name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
-		a.GPURequestAverage = 0
-	}
-	if math.IsNaN(a.GPUUsageAverage) {
-		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for GPUUsageAverage name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
-		a.GPUUsageAverage = 0
-	}
+
 	if math.IsNaN(a.GPUCost) {
 		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for GPUCost name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
 		a.GPUCost = 0
@@ -2682,6 +2799,7 @@ func (a *Allocation) SanitizeNaN() {
 
 	a.PVs.SanitizeNaN()
 	a.RawAllocationOnly.SanitizeNaN()
+	a.GPUAllocation.SanitizeNaN()
 	a.ProportionalAssetResourceCosts.SanitizeNaN()
 	a.SharedCostBreakdown.SanitizeNaN()
 	a.LoadBalancers.SanitizeNaN()
@@ -3570,3 +3688,15 @@ func (asr *AllocationSetRange) Clone() *AllocationSetRange {
 
 	return sasrClone
 }
+
+func migrateAllocation(as *Allocation, fromVersion uint8, toVersion uint8) {
+	if fromVersion == toVersion {
+		return
+	}
+
+	if fromVersion == 22 && toVersion >= 23 {
+		as.GPUAllocation = &GPUAllocation{}
+		as.GPUAllocation.GPUUsageAverage = &as.deprecatedGPUUsageAverage
+		as.GPUAllocation.GPURequestAverage = &as.deprecatedGPURequestAverage
+	}
+}

core/pkg/opencost/allocation_json.go

@@ -26,8 +26,8 @@ type AllocationJSON struct {
 	CPUCostIdle                    *float64                        `json:"cpuCostIdle"`
 	CPUEfficiency                  *float64                        `json:"cpuEfficiency"`
 	GPUCount                       *float64                        `json:"gpuCount"`
-	GPURequestAverage              *float64                        `json:"gpuRequestAverage"`
-	GPUUsageAverage                *float64                        `json:"gpuUsageAverage"`
+	GPURequestAverage              *float64                        `json:"-"`
+	GPUUsageAverage                *float64                        `json:"-"`
 	GPUHours                       *float64                        `json:"gpuHours"`
 	GPUCost                        *float64                        `json:"gpuCost"`
 	GPUCostAdjustment              *float64                        `json:"gpuCostAdjustment"`
@@ -63,6 +63,7 @@ type AllocationJSON struct {
 	ProportionalAssetResourceCosts *ProportionalAssetResourceCosts `json:"proportionalAssetResourceCosts,omitempty"`
 	LoadBalancers                  LbAllocations                   `json:"lbAllocations"`
 	SharedCostBreakdown            *SharedCostBreakdowns           `json:"sharedCostBreakdown,omitempty"`
+	GPUAllocation                  *GPUAllocation                  `json:"gpuAllocation"`
 }
 
 func (aj *AllocationJSON) BuildFromAllocation(a *Allocation) {
@@ -84,8 +85,6 @@ func (aj *AllocationJSON) BuildFromAllocation(a *Allocation) {
 	aj.CPUCostIdle = formatFloat64ForResponse(a.CPUCostIdle)
 	aj.CPUEfficiency = formatFloat64ForResponse(a.CPUEfficiency())
 	aj.GPUCount = formatFloat64ForResponse(a.GPUs())
-	aj.GPURequestAverage = formatFloat64ForResponse(a.GPURequestAverage)
-	aj.GPUUsageAverage = formatFloat64ForResponse(a.GPUUsageAverage)
 	aj.GPUHours = formatFloat64ForResponse(a.GPUHours)
 	aj.GPUCost = formatFloat64ForResponse(a.GPUCost)
 	aj.GPUCostAdjustment = formatFloat64ForResponse(a.GPUCostAdjustment)
@@ -121,6 +120,7 @@ func (aj *AllocationJSON) BuildFromAllocation(a *Allocation) {
 	aj.ProportionalAssetResourceCosts = &a.ProportionalAssetResourceCosts
 	aj.LoadBalancers = a.LoadBalancers
 	aj.SharedCostBreakdown = &a.SharedCostBreakdown
+	aj.GPUAllocation = a.GPUAllocation
 }
 
 // formatFloat64ForResponse - take an existing float64, round it to 6 decimal places and return is possible, or return nil if invalid

core/pkg/opencost/allocation_test.go

@@ -3784,12 +3784,38 @@ func TestRawAllocationOnlyData_SanitizeNaN(t *testing.T) {
 	raw.SanitizeNaN()
 	v := reflect.ValueOf(*raw)
 	checkAllFloat64sForNaN(t, v, "TestRawAllocationOnlyData_SanitizeNaN")
+
+	nan := math.NaN()
+	nilRawAllocation := &RawAllocationOnlyData{
+		CPUCoreUsageMax:  nan,
+		RAMBytesUsageMax: nan,
+		GPUUsageMax:      &nan,
+	}
+
+	nilRawAllocation.SanitizeNaN()
+
+	// SanitizeNaN allocates nil if NaN is passed
+	if nilRawAllocation.GPUUsageMax != nil {
+		t.Fatalf("want: nil, got: %v", nilRawAllocation.GPUUsageMax)
+	}
+
+	// SanitizeNaN allocates 0.0 if NaN is passed
+	if nilRawAllocation.CPUCoreUsageMax != 0.0 {
+		t.Fatalf("want: 0.0, got: %v", nilRawAllocation.CPUCoreUsageMax)
+	}
+
+	// SanitizeNaN allocates 0.0 if NaN is passed
+	if nilRawAllocation.RAMBytesUsageMax != 0.0 {
+		t.Fatalf("want: 0.0, got: %v", nilRawAllocation.RAMBytesUsageMax)
+	}
+
 }
 
 func getMockRawAllocationOnlyData(f float64) *RawAllocationOnlyData {
 	return &RawAllocationOnlyData{
 		CPUCoreUsageMax:  f,
 		RAMBytesUsageMax: f,
+		GPUUsageMax:      &f,
 	}
 }
 

core/pkg/opencost/asset_json.go

@@ -531,8 +531,12 @@ func (n *Node) InterfaceToNode(itf interface{}) error {
 
 	// parse labels map to AssetLabels
 	labels := make(map[string]string)
-	for k, v := range fmap["labels"].(map[string]interface{}) {
-		labels[k] = v.(string)
+	if labelsInterface, ok := fmap["labels"].(map[string]interface{}); ok {
+		for k, v := range labelsInterface {
+			if strValue, ok := v.(string); ok {
+				labels[k] = strValue
+			}
+		}
 	}
 
 	// parse start and end strings to time.Time

core/pkg/opencost/assetprops.go

@@ -187,6 +187,9 @@ const ScalewayProvider = "Scaleway"
 // OracleProvider describes the provider Oracle
 const OracleProvider = "Oracle"
 
+// OTCProvider describes the provider OTC
+const OTCProvider = "OTC"
+
 // NilProvider describes unknown provider
 const NilProvider = "-"
 

core/pkg/opencost/bingen.go

@@ -46,8 +46,8 @@ package opencost
 // @bingen:end
 
 // Allocation Version Set: Includes Allocation pipeline specific resources
-// @bingen:set[name=Allocation,version=22]
-// @bingen:generate:Allocation
+// @bingen:set[name=Allocation,version=23]
+// @bingen:generate[migrate]:Allocation
 // @bingen:generate[stringtable]:AllocationSet
 // @bingen:generate:AllocationSetRange
 // @bingen:generate:AllocationProperties
@@ -60,6 +60,7 @@ package opencost
 // @bingen:generate:PVAllocation
 // @bingen:generate:LbAllocations
 // @bingen:generate:LbAllocation
+// @bingen:generate:GPUAllocation
 // @bingen:end
 
 // @bingen:set[name=CloudCost,version=3]

core/pkg/opencost/opencost_codecs.go

@@ -13,11 +13,12 @@ package opencost
 
 import (
 	"fmt"
-	util "github.com/opencost/opencost/core/pkg/util"
 	"reflect"
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/opencost/opencost/core/pkg/util"
 )
 
 const (
@@ -40,7 +41,7 @@ const (
 	AssetsCodecVersion uint8 = 21
 
 	// AllocationCodecVersion is used for any resources listed in the Allocation version set
-	AllocationCodecVersion uint8 = 22
+	AllocationCodecVersion uint8 = 23
 
 	// CloudCostCodecVersion is used for any resources listed in the CloudCost version set
 	CloudCostCodecVersion uint8 = 3
@@ -72,6 +73,7 @@ var typeMap map[string]reflect.Type = map[string]reflect.Type{
 	"Coverage":              reflect.TypeOf((*Coverage)(nil)).Elem(),
 	"CoverageSet":           reflect.TypeOf((*CoverageSet)(nil)).Elem(),
 	"Disk":                  reflect.TypeOf((*Disk)(nil)).Elem(),
+	"GPUAllocation":         reflect.TypeOf((*GPUAllocation)(nil)).Elem(),
 	"LbAllocation":          reflect.TypeOf((*LbAllocation)(nil)).Elem(),
 	"LoadBalancer":          reflect.TypeOf((*LoadBalancer)(nil)).Elem(),
 	"Network":               reflect.TypeOf((*Network)(nil)).Elem(),
@@ -456,8 +458,22 @@ func (target *Allocation) MarshalBinaryWithContext(ctx *EncodingContext) (err er
 	}
 	// --- [end][write][alias](LbAllocations) ---
 
-	buff.WriteFloat64(target.GPURequestAverage) // write float64
-	buff.WriteFloat64(target.GPUUsageAverage)   // write float64
+	buff.WriteFloat64(target.deprecatedGPURequestAverage) // write float64
+	buff.WriteFloat64(target.deprecatedGPUUsageAverage)   // write float64
+	if target.GPUAllocation == nil {
+		buff.WriteUInt8(uint8(0)) // write nil byte
+	} else {
+		buff.WriteUInt8(uint8(1)) // write non-nil byte
+
+		// --- [begin][write][struct](GPUAllocation) ---
+		buff.WriteInt(0) // [compatibility, unused]
+		errI := target.GPUAllocation.MarshalBinaryWithContext(ctx)
+		if errI != nil {
+			return errI
+		}
+		// --- [end][write][struct](GPUAllocation) ---
+
+	}
 	return nil
 }
 
@@ -773,19 +789,45 @@ func (target *Allocation) UnmarshalBinaryWithContext(ctx *DecodingContext) (err
 	// field version check
 	if uint8(22) <= version {
 		fff := buff.ReadFloat64() // read float64
-		target.GPURequestAverage = fff
+		target.deprecatedGPURequestAverage = fff
 
 	} else {
-		target.GPURequestAverage = float64(0) // default
+		target.deprecatedGPURequestAverage = float64(0) // default
 	}
 
 	// field version check
 	if uint8(22) <= version {
 		ggg := buff.ReadFloat64() // read float64
-		target.GPUUsageAverage = ggg
+		target.deprecatedGPUUsageAverage = ggg
+
+	} else {
+		target.deprecatedGPUUsageAverage = float64(0) // default
+	}
 
+	// field version check
+	if uint8(23) <= version {
+		if buff.ReadUInt8() == uint8(0) {
+			target.GPUAllocation = nil
+		} else {
+			// --- [begin][read][struct](GPUAllocation) ---
+			hhh := &GPUAllocation{}
+			buff.ReadInt() // [compatibility, unused]
+			errI := hhh.UnmarshalBinaryWithContext(ctx)
+			if errI != nil {
+				return errI
+			}
+			target.GPUAllocation = hhh
+			// --- [end][read][struct](GPUAllocation) ---
+
+		}
 	} else {
-		target.GPUUsageAverage = float64(0) // default
+		target.GPUAllocation = nil
+
+	}
+
+	// execute migration func if version delta detected
+	if version != AllocationCodecVersion {
+		migrateAllocation(target, version, AllocationCodecVersion)
 	}
 
 	return nil
@@ -5486,6 +5528,196 @@ func (target *Disk) UnmarshalBinaryWithContext(ctx *DecodingContext) (err error)
 	return nil
 }
 
+//--------------------------------------------------------------------------
+//  GPUAllocation
+//--------------------------------------------------------------------------
+
+// MarshalBinary serializes the internal properties of this GPUAllocation instance
+// into a byte array
+func (target *GPUAllocation) MarshalBinary() (data []byte, err error) {
+	ctx := &EncodingContext{
+		Buffer: util.NewBuffer(),
+		Table:  nil,
+	}
+
+	e := target.MarshalBinaryWithContext(ctx)
+	if e != nil {
+		return nil, e
+	}
+
+	encBytes := ctx.Buffer.Bytes()
+	return encBytes, nil
+}
+
+// MarshalBinaryWithContext serializes the internal properties of this GPUAllocation instance
+// into a byte array leveraging a predefined context.
+func (target *GPUAllocation) MarshalBinaryWithContext(ctx *EncodingContext) (err error) {
+	// panics are recovered and propagated as errors
+	defer func() {
+		if r := recover(); r != nil {
+			if e, ok := r.(error); ok {
+				err = e
+			} else if s, ok := r.(string); ok {
+				err = fmt.Errorf("Unexpected panic: %s", s)
+			} else {
+				err = fmt.Errorf("Unexpected panic: %+v", r)
+			}
+		}
+	}()
+
+	buff := ctx.Buffer
+	buff.WriteUInt8(AllocationCodecVersion) // version
+
+	if ctx.IsStringTable() {
+		a := ctx.Table.AddOrGet(target.GPUDevice)
+		buff.WriteInt(a) // write table index
+	} else {
+		buff.WriteString(target.GPUDevice) // write string
+	}
+	if ctx.IsStringTable() {
+		b := ctx.Table.AddOrGet(target.GPUModel)
+		buff.WriteInt(b) // write table index
+	} else {
+		buff.WriteString(target.GPUModel) // write string
+	}
+	if ctx.IsStringTable() {
+		c := ctx.Table.AddOrGet(target.GPUUUID)
+		buff.WriteInt(c) // write table index
+	} else {
+		buff.WriteString(target.GPUUUID) // write string
+	}
+	if target.IsGPUShared == nil {
+		buff.WriteUInt8(uint8(0)) // write nil byte
+	} else {
+		buff.WriteUInt8(uint8(1)) // write non-nil byte
+
+		buff.WriteBool(*target.IsGPUShared) // write bool
+	}
+	if target.GPUUsageAverage == nil {
+		buff.WriteUInt8(uint8(0)) // write nil byte
+	} else {
+		buff.WriteUInt8(uint8(1)) // write non-nil byte
+
+		buff.WriteFloat64(*target.GPUUsageAverage) // write float64
+	}
+	if target.GPURequestAverage == nil {
+		buff.WriteUInt8(uint8(0)) // write nil byte
+	} else {
+		buff.WriteUInt8(uint8(1)) // write non-nil byte
+
+		buff.WriteFloat64(*target.GPURequestAverage) // write float64
+	}
+	return nil
+}
+
+// UnmarshalBinary uses the data passed byte array to set all the internal properties of
+// the GPUAllocation type
+func (target *GPUAllocation) UnmarshalBinary(data []byte) error {
+	var table []string
+	buff := util.NewBufferFromBytes(data)
+
+	// string table header validation
+	if isBinaryTag(data, BinaryTagStringTable) {
+		buff.ReadBytes(len(BinaryTagStringTable)) // strip tag length
+		tl := buff.ReadInt()                      // table length
+		if tl > 0 {
+			table = make([]string, tl, tl)
+			for i := 0; i < tl; i++ {
+				table[i] = buff.ReadString()
+			}
+		}
+	}
+
+	ctx := &DecodingContext{
+		Buffer: buff,
+		Table:  table,
+	}
+
+	err := target.UnmarshalBinaryWithContext(ctx)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// UnmarshalBinaryWithContext uses the context containing a string table and binary buffer to set all the internal properties of
+// the GPUAllocation type
+func (target *GPUAllocation) UnmarshalBinaryWithContext(ctx *DecodingContext) (err error) {
+	// panics are recovered and propagated as errors
+	defer func() {
+		if r := recover(); r != nil {
+			if e, ok := r.(error); ok {
+				err = e
+			} else if s, ok := r.(string); ok {
+				err = fmt.Errorf("Unexpected panic: %s", s)
+			} else {
+				err = fmt.Errorf("Unexpected panic: %+v", r)
+			}
+		}
+	}()
+
+	buff := ctx.Buffer
+	version := buff.ReadUInt8()
+
+	if version > AllocationCodecVersion {
+		return fmt.Errorf("Invalid Version Unmarshaling GPUAllocation. Expected %d or less, got %d", AllocationCodecVersion, version)
+	}
+
+	var b string
+	if ctx.IsStringTable() {
+		c := buff.ReadInt() // read string index
+		b = ctx.Table[c]
+	} else {
+		b = buff.ReadString() // read string
+	}
+	a := b
+	target.GPUDevice = a
+
+	var e string
+	if ctx.IsStringTable() {
+		f := buff.ReadInt() // read string index
+		e = ctx.Table[f]
+	} else {
+		e = buff.ReadString() // read string
+	}
+	d := e
+	target.GPUModel = d
+
+	var h string
+	if ctx.IsStringTable() {
+		k := buff.ReadInt() // read string index
+		h = ctx.Table[k]
+	} else {
+		h = buff.ReadString() // read string
+	}
+	g := h
+	target.GPUUUID = g
+
+	if buff.ReadUInt8() == uint8(0) {
+		target.IsGPUShared = nil
+	} else {
+		l := buff.ReadBool() // read bool
+		target.IsGPUShared = &l
+
+	}
+	if buff.ReadUInt8() == uint8(0) {
+		target.GPUUsageAverage = nil
+	} else {
+		m := buff.ReadFloat64() // read float64
+		target.GPUUsageAverage = &m
+
+	}
+	if buff.ReadUInt8() == uint8(0) {
+		target.GPURequestAverage = nil
+	} else {
+		n := buff.ReadFloat64() // read float64
+		target.GPURequestAverage = &n
+
+	}
+	return nil
+}
+
 //--------------------------------------------------------------------------
 //  LbAllocation
 //--------------------------------------------------------------------------
@@ -7012,6 +7244,13 @@ func (target *RawAllocationOnlyData) MarshalBinaryWithContext(ctx *EncodingConte
 
 	buff.WriteFloat64(target.CPUCoreUsageMax)  // write float64
 	buff.WriteFloat64(target.RAMBytesUsageMax) // write float64
+	if target.GPUUsageMax == nil {
+		buff.WriteUInt8(uint8(0)) // write nil byte
+	} else {
+		buff.WriteUInt8(uint8(1)) // write non-nil byte
+
+		buff.WriteFloat64(*target.GPUUsageMax) // write float64
+	}
 	return nil
 }
 
@@ -7075,6 +7314,20 @@ func (target *RawAllocationOnlyData) UnmarshalBinaryWithContext(ctx *DecodingCon
 	b := buff.ReadFloat64() // read float64
 	target.RAMBytesUsageMax = b
 
+	// field version check
+	if uint8(23) <= version {
+		if buff.ReadUInt8() == uint8(0) {
+			target.GPUUsageMax = nil
+		} else {
+			c := buff.ReadFloat64() // read float64
+			target.GPUUsageMax = &c
+
+		}
+	} else {
+		target.GPUUsageMax = nil
+
+	}
+
 	return nil
 }
 

core/pkg/opencost/summaryallocation.go

@@ -30,8 +30,8 @@ type SummaryAllocation struct {
 	CPUCoreUsageAverage    float64               `json:"cpuCoreUsageAverage"`
 	CPUCost                float64               `json:"cpuCost"`
 	CPUCostIdle            float64               `json:"cpuCostIdle"`
-	GPURequestAverage      float64               `json:"gpuRequestAverage"`
-	GPUUsageAverage        float64               `json:"gpuUsageAverage"`
+	GPURequestAverage      *float64              `json:"gpuRequestAverage"`
+	GPUUsageAverage        *float64              `json:"gpuUsageAverage"`
 	GPUCost                float64               `json:"gpuCost"`
 	GPUCostIdle            float64               `json:"gpuCostIdle"`
 	NetworkCost            float64               `json:"networkCost"`
@@ -57,6 +57,12 @@ func NewSummaryAllocation(alloc *Allocation, reconcile, reconcileNetwork bool) *
 		return nil
 	}
 
+	var gpuRequestAvg, gpuUsageAvg *float64
+	if alloc.GPUAllocation != nil {
+		gpuRequestAvg = alloc.GPUAllocation.GPURequestAverage
+		gpuUsageAvg = alloc.GPUAllocation.GPUUsageAverage
+	}
+
 	sa := &SummaryAllocation{
 		Name:                   alloc.Name,
 		Properties:             alloc.Properties,
@@ -65,8 +71,8 @@ func NewSummaryAllocation(alloc *Allocation, reconcile, reconcileNetwork bool) *
 		CPUCoreRequestAverage:  alloc.CPUCoreRequestAverage,
 		CPUCoreUsageAverage:    alloc.CPUCoreUsageAverage,
 		CPUCost:                alloc.CPUCost + alloc.CPUCostAdjustment,
-		GPURequestAverage:      alloc.GPURequestAverage,
-		GPUUsageAverage:        alloc.GPUUsageAverage,
+		GPURequestAverage:      gpuRequestAvg,
+		GPUUsageAverage:        gpuUsageAvg,
 		GPUCost:                alloc.GPUCost + alloc.GPUCostAdjustment,
 		NetworkCost:            alloc.NetworkCost + alloc.NetworkCostAdjustment,
 		LoadBalancerCost:       alloc.LoadBalancerCost + alloc.LoadBalancerCostAdjustment,
@@ -128,11 +134,37 @@ func (sa *SummaryAllocation) Add(that *SummaryAllocation) error {
 	ramUseByteMins := sa.RAMBytesUsageAverage * sa.Minutes()
 	ramUseByteMins += that.RAMBytesUsageAverage * that.Minutes()
 
-	gpuReqMins := sa.GPURequestAverage * sa.Minutes()
-	gpuReqMins += that.GPURequestAverage * that.Minutes()
+	var gpuReqMins *float64 = nil
+	if sa.GPURequestAverage != nil {
+		result := *sa.GPURequestAverage * sa.Minutes()
+		gpuReqMins = &result
+	}
 
-	gpuUseMins := sa.GPUUsageAverage * sa.Minutes()
-	gpuUseMins += that.GPUUsageAverage * that.Minutes()
+	if sa.GPURequestAverage != nil && that.GPURequestAverage != nil {
+		if gpuReqMins == nil {
+			result := *that.GPURequestAverage * that.Minutes()
+			gpuReqMins = &result
+		} else {
+			result := *gpuReqMins + *that.GPURequestAverage*that.Minutes()
+			gpuReqMins = &result
+		}
+	}
+
+	var gpuUseMins *float64 = nil
+	if sa.GPUUsageAverage != nil {
+		result := *sa.GPUUsageAverage * sa.Minutes()
+		gpuUseMins = &result
+	}
+
+	if that.GPUUsageAverage != nil {
+		if gpuUseMins == nil {
+			result := *that.GPUUsageAverage * that.Minutes()
+			gpuUseMins = &result
+		} else {
+			result := *gpuUseMins + *that.GPUUsageAverage*that.Minutes()
+			gpuUseMins = &result
+		}
+	}
 
 	// Expand Start and End to be the "max" of among the given Allocations
 	if that.Start.Before(sa.Start) {
@@ -148,15 +180,27 @@ func (sa *SummaryAllocation) Add(that *SummaryAllocation) error {
 		sa.CPUCoreUsageAverage = cpuUseCoreMins / sa.Minutes()
 		sa.RAMBytesRequestAverage = ramReqByteMins / sa.Minutes()
 		sa.RAMBytesUsageAverage = ramUseByteMins / sa.Minutes()
-		sa.GPURequestAverage = gpuReqMins / sa.Minutes()
-		sa.GPUUsageAverage = gpuUseMins / sa.Minutes()
+
+		var gpuReqAvgVal, gpuUsageAvgVal *float64
+		if gpuReqMins != nil {
+			result := *gpuReqMins / sa.Minutes()
+			gpuReqAvgVal = &result
+		}
+
+		if gpuUseMins != nil {
+			result := *gpuUseMins / sa.Minutes()
+			gpuUsageAvgVal = &result
+		}
+
+		sa.GPURequestAverage = gpuReqAvgVal
+		sa.GPUUsageAverage = gpuUsageAvgVal
 	} else {
 		sa.CPUCoreRequestAverage = 0.0
 		sa.CPUCoreUsageAverage = 0.0
 		sa.RAMBytesRequestAverage = 0.0
 		sa.RAMBytesUsageAverage = 0.0
-		sa.GPURequestAverage = 0.0
-		sa.GPUUsageAverage = 0.0
+		sa.GPURequestAverage = nil
+		sa.GPUUsageAverage = nil
 	}
 
 	// Sum all cumulative cost fields

core/pkg/opencost/summaryallocation_json.go

@@ -58,8 +58,8 @@ func (sa *SummaryAllocation) ToResponse() *SummaryAllocationResponse {
 		CPUCoreUsageAverage:    formatutil.Float64ToResponse(sa.CPUCoreUsageAverage),
 		CPUCost:                formatutil.Float64ToResponse(sa.CPUCost),
 		CPUCostIdle:            formatutil.Float64ToResponse(sa.CPUCostIdle),
-		GPURequestAverage:      formatutil.Float64ToResponse(sa.GPURequestAverage),
-		GPUUsageAverage:        formatutil.Float64ToResponse(sa.GPUUsageAverage),
+		GPURequestAverage:      sa.GPURequestAverage, // already in *float64
+		GPUUsageAverage:        sa.GPUUsageAverage,   // already in *float64
 		GPUCost:                formatutil.Float64ToResponse(sa.GPUCost),
 		GPUCostIdle:            formatutil.Float64ToResponse(sa.GPUCostIdle),
 		NetworkCost:            formatutil.Float64ToResponse(sa.NetworkCost),

core/pkg/opencost/window.go

@@ -956,7 +956,7 @@ func (w Window) getWeeklyWindow() Window {
 // getWeeklyWindows breaks up a window into weeks, with weeks starting on Sunday
 func (w Window) getWeeklyWindows() []Window {
 	wins := []Window{}
-	roundedWindow := w.getDailyWindow()
+	roundedWindow := w.getWeeklyWindow()
 
 	roundedStart := *roundedWindow.Start()
 	roundedEnd := *roundedWindow.End()

core/pkg/opencost/window_test.go

@@ -1192,3 +1192,118 @@ func TestBoundaryErrorIs(t *testing.T) {
 		t.Errorf("Multi wrap failure: %s != %s", baseError, multiWrapError)
 	}
 }
+func TestWindowGetWeeklyWindows(t *testing.T) {
+	testCases := []struct {
+		name     string
+		start    time.Time
+		end      time.Time
+		expected []Window
+	}{
+		{
+			name:  "Single week",
+			start: time.Date(2024, 9, 1, 0, 0, 0, 0, time.UTC),
+			end:   time.Date(2024, 9, 8, 0, 0, 0, 0, time.UTC),
+			expected: []Window{
+				NewClosedWindow(
+					time.Date(2024, 9, 1, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 9, 8, 0, 0, 0, 0, time.UTC),
+				),
+			},
+		},
+		{
+			name:  "Multiple weeks",
+			start: time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+			end:   time.Date(2024, 10, 20, 0, 0, 0, 0, time.UTC),
+			expected: []Window{
+				NewClosedWindow(
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 20, 0, 0, 0, 0, time.UTC),
+				),
+			},
+		},
+		{
+			name:  "Partial starting week",
+			start: time.Date(2024, 10, 1, 0, 0, 0, 0, time.UTC),
+			end:   time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+			expected: []Window{
+				NewClosedWindow(
+					time.Date(2024, 9, 29, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+				),
+			},
+		},
+		{
+			name:  "Partial ending week",
+			start: time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+			end:   time.Date(2024, 10, 7, 0, 0, 0, 0, time.UTC),
+			expected: []Window{
+				NewClosedWindow(
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+				),
+			},
+		},
+		{
+			name:  "multiple weeks, partial start and end",
+			start: time.Date(2024, 10, 1, 0, 0, 0, 0, time.UTC),
+			end:   time.Date(2024, 11, 21, 0, 0, 0, 0, time.UTC),
+			expected: []Window{
+				NewClosedWindow(
+					time.Date(2024, 9, 29, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 20, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 20, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 27, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 27, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 11, 3, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 11, 3, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 11, 10, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 11, 10, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 11, 17, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 11, 17, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 11, 24, 0, 0, 0, 0, time.UTC),
+				),
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			w := NewClosedWindow(tc.start, tc.end)
+			actual := w.getWeeklyWindows()
+			if len(actual) != len(tc.expected) {
+				t.Fatalf("expected %d windows, got %d", len(tc.expected), len(actual))
+			}
+			for i, expectedWindow := range tc.expected {
+				if !actual[i].Equal(expectedWindow) {
+					t.Errorf("expected window %s, got %s", expectedWindow, actual[i])
+				}
+			}
+		})
+	}
+}

docs/swagger.json

@@ -16,7 +16,7 @@
   }
   ],
   "paths": {
-    "/allocation/compute": {
+    "/allocation": {
       "get": {
         "summary": "query for costs and resources allocated to Kubernetes workloads",
         "description": "The standard OpenCost API query for costs and resources allocated to Kubernetes workloads. You may specify the `window` date range, the Kubernetes primitive to `aggregate` on, the `step` for the duration of returned sets, and the `resolution` for the duration to use for Prometheus queries.",

go.mod

@@ -115,7 +115,7 @@ require (
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/gofrs/uuid v4.2.0+incompatible // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.1 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
@@ -195,6 +195,4 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )
 
-go 1.22.0
-
-toolchain go1.22.4
+go 1.22.7

go.sum

@@ -230,8 +230,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69
 github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
-github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
-github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo=
+github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
 github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=

pkg/cloud/aws/provider.go

@@ -154,6 +154,7 @@ var awsRegions = []string{
 	"af-south-1",
 	"us-gov-east-1",
 	"us-gov-west-1",
+	"me-central-1",
 }
 
 // AWS represents an Amazon Provider
@@ -247,6 +248,7 @@ type AWSProduct struct {
 type AWSProductAttributes struct {
 	Location        string `json:"location"`
 	RegionCode      string `json:"regionCode"`
+	Operation       string `json:"operation"`
 	InstanceType    string `json:"instanceType"`
 	Memory          string `json:"memory"`
 	Storage         string `json:"storage"`
@@ -299,14 +301,15 @@ type AWSCurrencyCode struct {
 
 // AWSProductTerms represents the full terms of the product
 type AWSProductTerms struct {
-	Sku      string        `json:"sku"`
-	OnDemand *AWSOfferTerm `json:"OnDemand"`
-	Reserved *AWSOfferTerm `json:"Reserved"`
-	Memory   string        `json:"memory"`
-	Storage  string        `json:"storage"`
-	VCpu     string        `json:"vcpu"`
-	GPU      string        `json:"gpu"` // GPU represents the number of GPU on the instance
-	PV       *models.PV    `json:"pv"`
+	Sku          string               `json:"sku"`
+	OnDemand     *AWSOfferTerm        `json:"OnDemand"`
+	Reserved     *AWSOfferTerm        `json:"Reserved"`
+	Memory       string               `json:"memory"`
+	Storage      string               `json:"storage"`
+	VCpu         string               `json:"vcpu"`
+	GPU          string               `json:"gpu"` // GPU represents the number of GPU on the instance
+	PV           *models.PV           `json:"pv"`
+	LoadBalancer *models.LoadBalancer `json:"load_balancer"`
 }
 
 // ClusterIdEnvVar is the environment variable in which one can manually set the ClusterId
@@ -1036,6 +1039,19 @@ func (aws *AWS) populatePricing(resp *http.Response, inputkeys map[string]bool)
 					skusToKeys[product.Sku] = key
 					aws.ValidPricingKeys[key] = true
 					aws.ValidPricingKeys[spotKey] = true
+				} else if strings.Contains(product.Attributes.UsageType, "LoadBalancerUsage") && product.Attributes.Operation == "LoadBalancing:Network" {
+					// since the costmodel is only using services of type LoadBalancer
+					// (and not ingresses controlled by AWS load balancer controller)
+					// we can safely filter for Network load balancers only
+					productTerms := &AWSProductTerms{
+						Sku:          product.Sku,
+						LoadBalancer: &models.LoadBalancer{},
+					}
+					// there is no spot pricing for load balancers
+					key := product.Attributes.RegionCode + ",LoadBalancerUsage"
+					aws.Pricing[key] = productTerms
+					skusToKeys[product.Sku] = key
+					aws.ValidPricingKeys[key] = true
 				}
 			}
 		}
@@ -1077,7 +1093,9 @@ func (aws *AWS) populatePricing(resp *http.Response, inputkeys map[string]bool)
 					spotKey := key + ",preemptible"
 					if ok {
 						aws.Pricing[key].OnDemand = offerTerm
-						aws.Pricing[spotKey].OnDemand = offerTerm
+						if _, ok := aws.Pricing[spotKey]; ok {
+							aws.Pricing[spotKey].OnDemand = offerTerm
+						}
 						var cost string
 						if _, isMatch := OnDemandRateCodes[offerTerm.OfferTermCode]; isMatch {
 							priceDimensionKey := strings.Join([]string{sku.(string), offerTerm.OfferTermCode, HourlyRateCode}, ".")
@@ -1132,6 +1150,13 @@ func (aws *AWS) populatePricing(resp *http.Response, inputkeys map[string]bool)
 							hourlyPrice := costFloat / 730
 
 							aws.Pricing[key].PV.Cost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64)
+						} else if strings.Contains(key, "LoadBalancerUsage") {
+							costFloat, err := strconv.ParseFloat(cost, 64)
+							if err != nil {
+								return err
+							}
+
+							aws.Pricing[key].LoadBalancer.Cost = costFloat
 						}
 					}
 
@@ -1202,21 +1227,20 @@ func (aws *AWS) NetworkPricing() (*models.Network, error) {
 }
 
 func (aws *AWS) LoadBalancerPricing() (*models.LoadBalancer, error) {
-	fffrc := 0.025
-	afrc := 0.010
-	lbidc := 0.008
+	// TODO: determine key based on function arguments
+	// this is something that should be changed in the Provider interface
 
-	numForwardingRules := 1.0
-	dataIngressGB := 0.0
+	key := aws.ClusterRegion + ",LoadBalancerUsage"
 
-	var totalCost float64
-	if numForwardingRules < 5 {
-		totalCost = fffrc*numForwardingRules + lbidc*dataIngressGB
-	} else {
-		totalCost = fffrc*5 + afrc*(numForwardingRules-5) + lbidc*dataIngressGB
+	// set default price
+	hourlyCost := 0.025
+	// use price index when available
+	if terms, ok := aws.Pricing[key]; ok {
+		hourlyCost = terms.LoadBalancer.Cost
 	}
+
 	return &models.LoadBalancer{
-		Cost: totalCost,
+		Cost: hourlyCost,
 	}, nil
 }
 
@@ -1437,13 +1461,12 @@ func (awsProvider *AWS) ClusterInfo() (map[string]string, error) {
 			clusterName = awsClusterID
 			log.Warnf("Warning - %s will be deprecated in a future release. Use %s instead", ocenv.AWSClusterIDEnvVar, ocenv.ClusterIDEnvVar)
 		} else if clusterName = ocenv.GetClusterID(); clusterName != "" {
-			log.Infof("Setting cluster name to %s from %s ", clusterName, ocenv.ClusterIDEnvVar)
+			log.DedupedInfof(5, "Setting cluster name to %s from %s ", clusterName, ocenv.ClusterIDEnvVar)
 		} else {
 			clusterName = defaultClusterName
-			log.Warnf("Unable to detect cluster name - using default of %s", defaultClusterName)
-			log.Warnf("Please set cluster name through configmap or via %s env var", ocenv.ClusterIDEnvVar)
+			log.DedupedWarningf(5, "Unable to detect cluster name - using default of %s", defaultClusterName)
+			log.DedupedWarningf(5, "Please set cluster name through configmap or via %s env var", ocenv.ClusterIDEnvVar)
 		}
-
 	}
 
 	// this value requires configuration but is unavailable else where

pkg/cloud/aws/provider_test.go

@@ -1,7 +1,6 @@
 package aws
 
 import (
-	"bytes"
 	"encoding/json"
 	"io"
 	"net/http"
@@ -176,197 +175,19 @@ func Test_PricingData_Regression(t *testing.T) {
 func Test_populate_pricing(t *testing.T) {
 	awsTest := AWS{
 		ValidPricingKeys: map[string]bool{},
+		ClusterRegion:    "us-east-2",
 	}
 	inputkeys := map[string]bool{
 		"us-east-2,m5.large,linux": true,
 	}
-	// Case 0
-	awsUSEastString := `
-	{
-		"formatVersion" : "v1.0",
-		"disclaimer" : "This pricing list is for informational purposes only. All prices are subject to the additional terms included in the pricing pages on http://aws.amazon.com. All Free Tier prices are also subject to the terms included at https://aws.amazon.com/free/",
-		"offerCode" : "AmazonEC2",
-		"version" : "20230322145651",
-		"publicationDate" : "2023-03-22T14:56:51Z",
-		"products" : {
-			"8D49XP354UEYTHGM" : {
-				"sku" : "8D49XP354UEYTHGM",
-				"productFamily" : "Compute Instance",
-				"attributes" : {
-				  "servicecode" : "AmazonEC2",
-				  "location" : "US East (Ohio)",
-				  "locationType" : "AWS Region",
-				  "instanceType" : "m5.large",
-				  "currentGeneration" : "Yes",
-				  "instanceFamily" : "General purpose",
-				  "vcpu" : "2",
-				  "physicalProcessor" : "Intel Xeon Platinum 8175",
-				  "clockSpeed" : "3.1 GHz",
-				  "memory" : "8 GiB",
-				  "storage" : "EBS only",
-				  "networkPerformance" : "Up to 10 Gigabit",
-				  "processorArchitecture" : "64-bit",
-				  "tenancy" : "Shared",
-				  "operatingSystem" : "Linux",
-				  "licenseModel" : "No License required",
-				  "usagetype" : "USE2-BoxUsage:m5.large",
-				  "operation" : "RunInstances",
-				  "availabilityzone" : "NA",
-				  "capacitystatus" : "Used",
-				  "classicnetworkingsupport" : "false",
-				  "dedicatedEbsThroughput" : "Up to 2120 Mbps",
-				  "ecu" : "10",
-				  "enhancedNetworkingSupported" : "Yes",
-				  "gpuMemory" : "NA",
-				  "intelAvxAvailable" : "Yes",
-				  "intelAvx2Available" : "Yes",
-				  "intelTurboAvailable" : "Yes",
-				  "marketoption" : "OnDemand",
-				  "normalizationSizeFactor" : "4",
-				  "preInstalledSw" : "NA",
-				  "processorFeatures" : "Intel AVX; Intel AVX2; Intel AVX512; Intel Turbo",
-				  "regionCode" : "us-east-2",
-				  "servicename" : "Amazon Elastic Compute Cloud",
-				  "vpcnetworkingsupport" : "true"
-				}
-			},
-			"9ZEEN7WWWQKAG292" : {
-				"sku" : "9ZEEN7WWWQKAG292",
-				"productFamily" : "Compute Instance",
-				"attributes" : {
-				  "servicecode" : "AmazonEC2",
-				  "location" : "US East (Ohio)",
-				  "locationType" : "AWS Region",
-				  "instanceType" : "p3.8xlarge",
-				  "currentGeneration" : "Yes",
-				  "instanceFamily" : "GPU instance",
-				  "vcpu" : "32",
-				  "physicalProcessor" : "Intel Xeon E5-2686 v4 (Broadwell)",
-				  "clockSpeed" : "2.3 GHz",
-				  "memory" : "244 GiB",
-				  "storage" : "EBS only",
-				  "networkPerformance" : "10 Gigabit",
-				  "processorArchitecture" : "64-bit",
-				  "tenancy" : "Shared",
-				  "operatingSystem" : "Windows",
-				  "licenseModel" : "Bring your own license",
-				  "usagetype" : "USE2-BoxUsage:p3.8xlarge",
-				  "operation" : "RunInstances:0800",
-				  "availabilityzone" : "NA",
-				  "capacitystatus" : "Used",
-				  "classicnetworkingsupport" : "false",
-				  "dedicatedEbsThroughput" : "7000 Mbps",
-				  "ecu" : "97",
-				  "enhancedNetworkingSupported" : "Yes",
-				  "gpu" : "4",
-				  "gpuMemory" : "NA",
-				  "intelAvxAvailable" : "Yes",
-				  "intelAvx2Available" : "Yes",
-				  "intelTurboAvailable" : "Yes",
-				  "marketoption" : "OnDemand",
-				  "normalizationSizeFactor" : "64",
-				  "preInstalledSw" : "NA",
-				  "processorFeatures" : "Intel AVX; Intel AVX2; Intel Turbo",
-				  "regionCode" : "us-east-2",
-				  "servicename" : "Amazon Elastic Compute Cloud",
-				  "vpcnetworkingsupport" : "true"
-				}
-			},
-			"M6UGCCQ3CDJQAA37" : {
-				"sku" : "M6UGCCQ3CDJQAA37",
-				"productFamily" : "Storage",
-				"attributes" : {
-				  "servicecode" : "AmazonEC2",
-				  "location" : "US East (Ohio)",
-				  "locationType" : "AWS Region",
-				  "storageMedia" : "SSD-backed",
-				  "volumeType" : "General Purpose",
-				  "maxVolumeSize" : "16 TiB",
-				  "maxIopsvolume" : "16000",
-				  "maxThroughputvolume" : "1000 MiB/s",
-				  "usagetype" : "USE2-EBS:VolumeUsage.gp3",
-				  "operation" : "",
-				  "regionCode" : "us-east-2",
-				  "servicename" : "Amazon Elastic Compute Cloud",
-				  "volumeApiName" : "gp3"
-				}
-			}
-		},
-		"terms" : {
-			"OnDemand" : {
-				"M6UGCCQ3CDJQAA37" : {
-					"M6UGCCQ3CDJQAA37.JRTCKXETXF" : {
-					  "offerTermCode" : "JRTCKXETXF",
-					  "sku" : "M6UGCCQ3CDJQAA37",
-					  "effectiveDate" : "2023-03-01T00:00:00Z",
-					  "priceDimensions" : {
-						"M6UGCCQ3CDJQAA37.JRTCKXETXF.6YS6EN2CT7" : {
-						  "rateCode" : "M6UGCCQ3CDJQAA37.JRTCKXETXF.6YS6EN2CT7",
-						  "description" : "$0.08 per GB-month of General Purpose (gp3) provisioned storage - US East (Ohio)",
-						  "beginRange" : "0",
-						  "endRange" : "Inf",
-						  "unit" : "GB-Mo",
-						  "pricePerUnit" : {
-							"USD" : "0.0800000000"
-						  },
-						  "appliesTo" : [ ]
-						}
-					  },
-					  "termAttributes" : { }
-					}
-				},
-				"9ZEEN7WWWQKAG292" : {
-					"9ZEEN7WWWQKAG292.JRTCKXETXF" : {
-					  "offerTermCode" : "JRTCKXETXF",
-					  "sku" : "9ZEEN7WWWQKAG292",
-					  "effectiveDate" : "2023-03-01T00:00:00Z",
-					  "priceDimensions" : {
-						"9ZEEN7WWWQKAG292.JRTCKXETXF.6YS6EN2CT7" : {
-						  "rateCode" : "9ZEEN7WWWQKAG292.JRTCKXETXF.6YS6EN2CT7",
-						  "description" : "$12.24 per On Demand Windows BYOL p3.8xlarge Instance Hour",
-						  "beginRange" : "0",
-						  "endRange" : "Inf",
-						  "unit" : "Hrs",
-						  "pricePerUnit" : {
-							"USD" : "12.2400000000"
-						  },
-						  "appliesTo" : [ ]
-						}
-					  },
-					  "termAttributes" : { }
-					}
-				},
-				"8D49XP354UEYTHGM" : {
-					"8D49XP354UEYTHGM.MZU6U2429S" : {
-					  "offerTermCode" : "MZU6U2429S",
-					  "sku" : "8D49XP354UEYTHGM",
-					  "effectiveDate" : "2019-01-01T00:00:00Z",
-					  "priceDimensions" : {
-						"8D49XP354UEYTHGM.MZU6U2429S.2TG2D8R56U" : {
-						  "rateCode" : "8D49XP354UEYTHGM.MZU6U2429S.2TG2D8R56U",
-						  "description" : "Upfront Fee",
-						  "unit" : "Quantity",
-						  "pricePerUnit" : {
-							"USD" : "1161"
-						  },
-						  "appliesTo" : [ ]
-						},
-					  },
-					  "termAttributes" : {
-						"LeaseContractLength" : "3yr",
-						"OfferingClass" : "convertible",
-						"PurchaseOption" : "All Upfront"
-					  }
-					}
-				}
-			}
-		},
-		"attributesList" : { }
+
+	fixture, err := os.Open("testdata/pricing-us-east-2.json")
+	if err != nil {
+		t.Fatalf("failed to load pricing fixture: %s", err)
 	}
-	`
 
 	testResponse := http.Response{
-		Body: io.NopCloser(bytes.NewBufferString(awsUSEastString)),
+		Body: io.NopCloser(fixture),
 		Request: &http.Request{
 			URL: &url.URL{
 				Scheme: "https",
@@ -413,9 +234,17 @@ func Test_populate_pricing(t *testing.T) {
 		VCpu:    "2",
 		GPU:     "",
 		OnDemand: &AWSOfferTerm{
-			Sku:             "",
-			OfferTermCode:   "",
-			PriceDimensions: nil,
+			Sku:           "8D49XP354UEYTHGM",
+			OfferTermCode: "MZU6U2429S",
+			PriceDimensions: map[string]*AWSRateCode{
+				"8D49XP354UEYTHGM.MZU6U2429S.2TG2D8R56U": {
+					Unit: "Quantity",
+					PricePerUnit: AWSCurrencyCode{
+						USD: "1161",
+						CNY: "",
+					},
+				},
+			},
 		},
 	}
 
@@ -426,9 +255,37 @@ func Test_populate_pricing(t *testing.T) {
 		VCpu:    "2",
 		GPU:     "",
 		OnDemand: &AWSOfferTerm{
-			Sku:             "",
-			OfferTermCode:   "",
-			PriceDimensions: nil,
+			Sku:           "8D49XP354UEYTHGM",
+			OfferTermCode: "MZU6U2429S",
+			PriceDimensions: map[string]*AWSRateCode{
+				"8D49XP354UEYTHGM.MZU6U2429S.2TG2D8R56U": {
+					Unit: "Quantity",
+					PricePerUnit: AWSCurrencyCode{
+						USD: "1161",
+						CNY: "",
+					},
+				},
+			},
+		},
+	}
+
+	expectedProdTermsLoadbalancer := &AWSProductTerms{
+		Sku: "Y9RYMSE644KDSV4S",
+		OnDemand: &AWSOfferTerm{
+			Sku:           "Y9RYMSE644KDSV4S",
+			OfferTermCode: "JRTCKXETXF",
+			PriceDimensions: map[string]*AWSRateCode{
+				"Y9RYMSE644KDSV4S.JRTCKXETXF.6YS6EN2CT7": {
+					Unit: "Hrs",
+					PricePerUnit: AWSCurrencyCode{
+						USD: "0.0225000000",
+						CNY: "",
+					},
+				},
+			},
+		},
+		LoadBalancer: &models.LoadBalancer{
+			Cost: 0.0225,
 		},
 	}
 
@@ -437,161 +294,30 @@ func Test_populate_pricing(t *testing.T) {
 		"us-east-2,EBS:VolumeUsage.gp3,preemptible": expectedProdTermsDisk,
 		"us-east-2,m5.large,linux":                  expectedProdTermsInstanceOndemand,
 		"us-east-2,m5.large,linux,preemptible":      expectedProdTermsInstanceSpot,
+		"us-east-2,LoadBalancerUsage":               expectedProdTermsLoadbalancer,
 	}
 
 	if !reflect.DeepEqual(expectedPricing, awsTest.Pricing) {
-		t.Fatalf("expected parsed pricing did not match actual parsed result (us-east-1)")
+		t.Fatalf("expected parsed pricing did not match actual parsed result (us-east-2)")
+	}
+
+	lbPricing, _ := awsTest.LoadBalancerPricing()
+	if lbPricing.Cost != 0.0225 {
+		t.Fatalf("expected loadbalancer pricing of 0.0225 but got %f (us-east-2)", lbPricing.Cost)
 	}
 
 	// Case 1 - Only accept `"marketoption":"OnDemand"`
 	inputkeysCase1 := map[string]bool{
 		"us-east-1,p4d.24xlarge,linux": true,
 	}
-	pricingCase1 := `
-	{
-		"formatVersion" : "v1.0",
-		"disclaimer" : "This pricing list is for informational purposes only. All prices are subject to the additional terms included in the pricing pages on http://aws.amazon.com. All Free Tier prices are also subject to the terms included at https://aws.amazon.com/free/",
-		"offerCode" : "AmazonEC2",
-		"version" : "20240528203522",
-		"publicationDate" : "2024-05-28T20:35:22Z",
-		"products" : {
-			"H7NGEAC6UEHNTKSJ" : {
-				"sku" : "H7NGEAC6UEHNTKSJ",
-				"productFamily" : "Compute Instance",
-				"attributes" : {
-					"servicecode" : "AmazonEC2",
-					"location" : "US East (N. Virginia)",
-					"locationType" : "AWS Region",
-					"instanceType" : "p4d.24xlarge",
-					"currentGeneration" : "Yes",
-					"instanceFamily" : "GPU instance",
-					"vcpu" : "96",
-					"physicalProcessor" : "Intel Xeon Platinum 8275L",
-					"clockSpeed" : "3 GHz",
-					"memory" : "1152 GiB",
-					"storage" : "8 x 1000 SSD",
-					"networkPerformance" : "400 Gigabit",
-					"processorArchitecture" : "64-bit",
-					"tenancy" : "Shared",
-					"operatingSystem" : "Linux",
-					"licenseModel" : "No License required",
-					"usagetype" : "BoxUsage:p4d.24xlarge",
-					"operation" : "RunInstances",
-					"availabilityzone" : "NA",
-					"capacitystatus" : "Used",
-					"classicnetworkingsupport" : "false",
-					"dedicatedEbsThroughput" : "19000 Mbps",
-					"ecu" : "345",
-					"enhancedNetworkingSupported" : "No",
-					"gpu" : "8",
-					"gpuMemory" : "NA",
-					"intelAvxAvailable" : "Yes",
-					"intelAvx2Available" : "Yes",
-					"intelTurboAvailable" : "Yes",
-					"marketoption" : "OnDemand",
-					"normalizationSizeFactor" : "192",
-					"preInstalledSw" : "NA",
-					"processorFeatures" : "Intel AVX; Intel AVX2; Intel AVX512; Intel Turbo",
-					"regionCode" : "us-east-1",
-					"servicename" : "Amazon Elastic Compute Cloud",
-					"vpcnetworkingsupport" : "true"
-				}
-			},
-			"YSXJGN78QTXNVGDQ" : {
-				"sku" : "YSXJGN78QTXNVGDQ",
-				"productFamily" : "Compute Instance",
-				"attributes" : {
-					"servicecode" : "AmazonEC2",
-					"location" : "US East (N. Virginia)",
-					"locationType" : "AWS Region",
-					"instanceType" : "p4d.24xlarge",
-					"currentGeneration" : "Yes",
-					"instanceFamily" : "GPU instance",
-					"vcpu" : "96",
-					"physicalProcessor" : "Intel Xeon Platinum 8275L",
-					"clockSpeed" : "3 GHz",
-					"memory" : "1152 GiB",
-					"storage" : "8 x 1000 SSD",
-					"networkPerformance" : "400 Gigabit",
-					"processorArchitecture" : "64-bit",
-					"tenancy" : "Shared",
-					"operatingSystem" : "Linux",
-					"licenseModel" : "No License required",
-					"usagetype" : "BoxUsage:p4d.24xlarge",
-					"operation" : "RunInstances:CB",
-					"availabilityzone" : "NA",
-					"capacitystatus" : "Used",
-					"classicnetworkingsupport" : "false",
-					"dedicatedEbsThroughput" : "19000 Mbps",
-					"ecu" : "345",
-					"enhancedNetworkingSupported" : "No",
-					"gpu" : "8",
-					"gpuMemory" : "NA",
-					"intelAvxAvailable" : "Yes",
-					"intelAvx2Available" : "Yes",
-					"intelTurboAvailable" : "Yes",
-					"marketoption" : "CapacityBlock",
-					"normalizationSizeFactor" : "192",
-					"preInstalledSw" : "NA",
-					"processorFeatures" : "Intel AVX; Intel AVX2; Intel AVX512; Intel Turbo",
-					"regionCode" : "us-east-1",
-					"servicename" : "Amazon Elastic Compute Cloud",
-					"vpcnetworkingsupport" : "true"
-				}
-			}
-		},
-		"terms" : {
-			"OnDemand" : {
-				"H7NGEAC6UEHNTKSJ" : {
-					"H7NGEAC6UEHNTKSJ.JRTCKXETXF" : {
-						"offerTermCode" : "JRTCKXETXF",
-						"sku" : "H7NGEAC6UEHNTKSJ",
-						"effectiveDate" : "2024-05-01T00:00:00Z",
-						"priceDimensions" : {
-							"H7NGEAC6UEHNTKSJ.JRTCKXETXF.6YS6EN2CT7" : {
-								"rateCode" : "H7NGEAC6UEHNTKSJ.JRTCKXETXF.6YS6EN2CT7",
-								"description" : "$32.7726 per On Demand Linux p4d.24xlarge Instance Hour",
-								"beginRange" : "0",
-								"endRange" : "Inf",
-								"unit" : "Hrs",
-								"pricePerUnit" : {
-									"USD" : "32.7726000000"
-								},
-								"appliesTo" : [ ]
-							}
-						},
-						"termAttributes" : { }
-					}
-				},
-				"YSXJGN78QTXNVGDQ" : {
-					"YSXJGN78QTXNVGDQ.JRTCKXETXF" : {
-						"offerTermCode" : "JRTCKXETXF",
-						"sku" : "YSXJGN78QTXNVGDQ",
-						"effectiveDate" : "2024-05-01T00:00:00Z",
-						"priceDimensions" : {
-							"YSXJGN78QTXNVGDQ.JRTCKXETXF.6YS6EN2CT7" : {
-							"rateCode" : "YSXJGN78QTXNVGDQ.JRTCKXETXF.6YS6EN2CT7",
-							"description" : "$0.00 per Capacity Block Linux p4d.24xlarge Instance Hour",
-							"beginRange" : "0",
-							"endRange" : "Inf",
-							"unit" : "Hrs",
-							"pricePerUnit" : {
-								"USD" : "0.0000000000"
-							},
-							"appliesTo" : [ ]
-						}
-					},
-					"termAttributes" : { }
-					}
-				},
-			}
-		},
-		"attributesList" : { }
+
+	fixture, err = os.Open("testdata/pricing-us-east-1.json")
+	if err != nil {
+		t.Fatalf("failed to load pricing fixture: %s", err)
 	}
-	`
 
 	testResponseCase1 := http.Response{
-		Body: io.NopCloser(bytes.NewBufferString(pricingCase1)),
+		Body: io.NopCloser(fixture),
 		Request: &http.Request{
 			URL: &url.URL{
 				Scheme: "https",
@@ -636,68 +362,17 @@ func Test_populate_pricing(t *testing.T) {
 	}
 
 	// Case 2
-	awsCnString := `
-	{
-		"formatVersion" : "v1.0",
-		"disclaimer" : "This pricing list is for informational purposes only. All prices are subject to the additional terms included in the pricing pages on http://www.amazonaws.cn.",
-		"offerCode" : "AmazonEC2",
-		"version" : "20230314154740",
-		"publicationDate" : "2023-03-14T15:47:40Z",
-		"products" : {
-			"R83VXG9NAPDASEGN" : {
-				"sku" : "R83VXG9NAPDASEGN",
-				"productFamily" : "Storage",
-				"attributes" : {
-				  "servicecode" : "AmazonEC2",
-				  "location" : "China (Ningxia)",
-				  "locationType" : "AWS Region",
-				  "storageMedia" : "SSD-backed",
-				  "volumeType" : "General Purpose",
-				  "maxVolumeSize" : "16 TiB",
-				  "maxIopsvolume" : "16000",
-				  "maxThroughputvolume" : "1000 MiB/s",
-				  "usagetype" : "CNW1-EBS:VolumeUsage.gp3",
-				  "operation" : "",
-				  "regionCode" : "cn-northwest-1",
-				  "servicename" : "Amazon Elastic Compute Cloud",
-				  "volumeApiName" : "gp3"
-				}
-			}
-		},
-		"terms" : {
-			"OnDemand" : {
-			  "R83VXG9NAPDASEGN" : {
-				"R83VXG9NAPDASEGN.5Y9WH78GDR" : {
-				  "offerTermCode" : "5Y9WH78GDR",
-				  "sku" : "R83VXG9NAPDASEGN",
-				  "effectiveDate" : "2023-03-01T00:00:00Z",
-				  "priceDimensions" : {
-					"R83VXG9NAPDASEGN.5Y9WH78GDR.Q7UJUT2CE6" : {
-					  "rateCode" : "R83VXG9NAPDASEGN.5Y9WH78GDR.Q7UJUT2CE6",
-					  "description" : "0.5312 CNY per GB-month of General Purpose (gp3) provisioned storage - China (Ningxia)",
-					  "beginRange" : "0",
-					  "endRange" : "Inf",
-					  "unit" : "GB-Mo",
-					  "pricePerUnit" : {
-						"CNY" : "0.5312000000"
-					  },
-					  "appliesTo" : [ ]
-					}
-				  },
-				  "termAttributes" : { }
-				}
-			  }
-			}
-	    },
-	  "attributesList" : { }
-	}
-	`
 	awsTest = AWS{
 		ValidPricingKeys: map[string]bool{},
 	}
 
+	fixture, err = os.Open("testdata/pricing-cn-northwest-1.json")
+	if err != nil {
+		t.Fatalf("failed to load pricing fixture: %s", err)
+	}
+
 	testResponse = http.Response{
-		Body: io.NopCloser(bytes.NewBufferString(awsCnString)),
+		Body: io.NopCloser(fixture),
 		Request: &http.Request{
 			URL: &url.URL{
 				Scheme: "https",

pkg/cloud/aws/testdata/pricing-cn-northwest-1.json

@@ -0,0 +1,54 @@
+{
+  "formatVersion": "v1.0",
+  "disclaimer": "This pricing list is for informational purposes only. All prices are subject to the additional terms included in the pricing pages on http://www.amazonaws.cn.",
+  "offerCode": "AmazonEC2",
+  "version": "20230314154740",
+  "publicationDate": "2023-03-14T15:47:40Z",
+  "products": {
+    "R83VXG9NAPDASEGN": {
+      "sku": "R83VXG9NAPDASEGN",
+      "productFamily": "Storage",
+      "attributes": {
+        "servicecode": "AmazonEC2",
+        "location": "China (Ningxia)",
+        "locationType": "AWS Region",
+        "storageMedia": "SSD-backed",
+        "volumeType": "General Purpose",
+        "maxVolumeSize": "16 TiB",
+        "maxIopsvolume": "16000",
+        "maxThroughputvolume": "1000 MiB/s",
+        "usagetype": "CNW1-EBS:VolumeUsage.gp3",
+        "operation": "",
+        "regionCode": "cn-northwest-1",
+        "servicename": "Amazon Elastic Compute Cloud",
+        "volumeApiName": "gp3"
+      }
+    }
+  },
+  "terms": {
+    "OnDemand": {
+      "R83VXG9NAPDASEGN": {
+        "R83VXG9NAPDASEGN.5Y9WH78GDR": {
+          "offerTermCode": "5Y9WH78GDR",
+          "sku": "R83VXG9NAPDASEGN",
+          "effectiveDate": "2023-03-01T00:00:00Z",
+          "priceDimensions": {
+            "R83VXG9NAPDASEGN.5Y9WH78GDR.Q7UJUT2CE6": {
+              "rateCode": "R83VXG9NAPDASEGN.5Y9WH78GDR.Q7UJUT2CE6",
+              "description": "0.5312 CNY per GB-month of General Purpose (gp3) provisioned storage - China (Ningxia)",
+              "beginRange": "0",
+              "endRange": "Inf",
+              "unit": "GB-Mo",
+              "pricePerUnit": {
+                "CNY": "0.5312000000"
+              },
+              "appliesTo": []
+            }
+          },
+          "termAttributes": {}
+        }
+      }
+    }
+  },
+  "attributesList": {}
+}

pkg/cloud/aws/testdata/pricing-us-east-1.json

@@ -0,0 +1,140 @@
+{
+  "formatVersion": "v1.0",
+  "disclaimer": "This pricing list is for informational purposes only. All prices are subject to the additional terms included in the pricing pages on http://aws.amazon.com. All Free Tier prices are also subject to the terms included at https://aws.amazon.com/free/",
+  "offerCode": "AmazonEC2",
+  "version": "20240528203522",
+  "publicationDate": "2024-05-28T20:35:22Z",
+  "products": {
+    "H7NGEAC6UEHNTKSJ": {
+      "sku": "H7NGEAC6UEHNTKSJ",
+      "productFamily": "Compute Instance",
+      "attributes": {
+        "servicecode": "AmazonEC2",
+        "location": "US East (N. Virginia)",
+        "locationType": "AWS Region",
+        "instanceType": "p4d.24xlarge",
+        "currentGeneration": "Yes",
+        "instanceFamily": "GPU instance",
+        "vcpu": "96",
+        "physicalProcessor": "Intel Xeon Platinum 8275L",
+        "clockSpeed": "3 GHz",
+        "memory": "1152 GiB",
+        "storage": "8 x 1000 SSD",
+        "networkPerformance": "400 Gigabit",
+        "processorArchitecture": "64-bit",
+        "tenancy": "Shared",
+        "operatingSystem": "Linux",
+        "licenseModel": "No License required",
+        "usagetype": "BoxUsage:p4d.24xlarge",
+        "operation": "RunInstances",
+        "availabilityzone": "NA",
+        "capacitystatus": "Used",
+        "classicnetworkingsupport": "false",
+        "dedicatedEbsThroughput": "19000 Mbps",
+        "ecu": "345",
+        "enhancedNetworkingSupported": "No",
+        "gpu": "8",
+        "gpuMemory": "NA",
+        "intelAvxAvailable": "Yes",
+        "intelAvx2Available": "Yes",
+        "intelTurboAvailable": "Yes",
+        "marketoption": "OnDemand",
+        "normalizationSizeFactor": "192",
+        "preInstalledSw": "NA",
+        "processorFeatures": "Intel AVX; Intel AVX2; Intel AVX512; Intel Turbo",
+        "regionCode": "us-east-1",
+        "servicename": "Amazon Elastic Compute Cloud",
+        "vpcnetworkingsupport": "true"
+      }
+    },
+    "YSXJGN78QTXNVGDQ": {
+      "sku": "YSXJGN78QTXNVGDQ",
+      "productFamily": "Compute Instance",
+      "attributes": {
+        "servicecode": "AmazonEC2",
+        "location": "US East (N. Virginia)",
+        "locationType": "AWS Region",
+        "instanceType": "p4d.24xlarge",
+        "currentGeneration": "Yes",
+        "instanceFamily": "GPU instance",
+        "vcpu": "96",
+        "physicalProcessor": "Intel Xeon Platinum 8275L",
+        "clockSpeed": "3 GHz",
+        "memory": "1152 GiB",
+        "storage": "8 x 1000 SSD",
+        "networkPerformance": "400 Gigabit",
+        "processorArchitecture": "64-bit",
+        "tenancy": "Shared",
+        "operatingSystem": "Linux",
+        "licenseModel": "No License required",
+        "usagetype": "BoxUsage:p4d.24xlarge",
+        "operation": "RunInstances:CB",
+        "availabilityzone": "NA",
+        "capacitystatus": "Used",
+        "classicnetworkingsupport": "false",
+        "dedicatedEbsThroughput": "19000 Mbps",
+        "ecu": "345",
+        "enhancedNetworkingSupported": "No",
+        "gpu": "8",
+        "gpuMemory": "NA",
+        "intelAvxAvailable": "Yes",
+        "intelAvx2Available": "Yes",
+        "intelTurboAvailable": "Yes",
+        "marketoption": "CapacityBlock",
+        "normalizationSizeFactor": "192",
+        "preInstalledSw": "NA",
+        "processorFeatures": "Intel AVX; Intel AVX2; Intel AVX512; Intel Turbo",
+        "regionCode": "us-east-1",
+        "servicename": "Amazon Elastic Compute Cloud",
+        "vpcnetworkingsupport": "true"
+      }
+    }
+  },
+  "terms": {
+    "OnDemand": {
+      "H7NGEAC6UEHNTKSJ": {
+        "H7NGEAC6UEHNTKSJ.JRTCKXETXF": {
+          "offerTermCode": "JRTCKXETXF",
+          "sku": "H7NGEAC6UEHNTKSJ",
+          "effectiveDate": "2024-05-01T00:00:00Z",
+          "priceDimensions": {
+            "H7NGEAC6UEHNTKSJ.JRTCKXETXF.6YS6EN2CT7": {
+              "rateCode": "H7NGEAC6UEHNTKSJ.JRTCKXETXF.6YS6EN2CT7",
+              "description": "$32.7726 per On Demand Linux p4d.24xlarge Instance Hour",
+              "beginRange": "0",
+              "endRange": "Inf",
+              "unit": "Hrs",
+              "pricePerUnit": {
+                "USD": "32.7726000000"
+              },
+              "appliesTo": []
+            }
+          },
+          "termAttributes": {}
+        }
+      },
+      "YSXJGN78QTXNVGDQ": {
+        "YSXJGN78QTXNVGDQ.JRTCKXETXF": {
+          "offerTermCode": "JRTCKXETXF",
+          "sku": "YSXJGN78QTXNVGDQ",
+          "effectiveDate": "2024-05-01T00:00:00Z",
+          "priceDimensions": {
+            "YSXJGN78QTXNVGDQ.JRTCKXETXF.6YS6EN2CT7": {
+              "rateCode": "YSXJGN78QTXNVGDQ.JRTCKXETXF.6YS6EN2CT7",
+              "description": "$0.00 per Capacity Block Linux p4d.24xlarge Instance Hour",
+              "beginRange": "0",
+              "endRange": "Inf",
+              "unit": "Hrs",
+              "pricePerUnit": {
+                "USD": "0.0000000000"
+              },
+              "appliesTo": []
+            }
+          },
+          "termAttributes": {}
+        }
+      }
+    }
+  },
+  "attributesList": {}
+}

pkg/cloud/aws/testdata/pricing-us-east-2.json

@@ -0,0 +1,217 @@
+{
+  "formatVersion": "v1.0",
+  "disclaimer": "This pricing list is for informational purposes only. All prices are subject to the additional terms included in the pricing pages on http://aws.amazon.com. All Free Tier prices are also subject to the terms included at https://aws.amazon.com/free/",
+  "offerCode": "AmazonEC2",
+  "version": "20230322145651",
+  "publicationDate": "2023-03-22T14:56:51Z",
+  "products": {
+    "8D49XP354UEYTHGM": {
+      "sku": "8D49XP354UEYTHGM",
+      "productFamily": "Compute Instance",
+      "attributes": {
+        "servicecode": "AmazonEC2",
+        "location": "US East (Ohio)",
+        "locationType": "AWS Region",
+        "instanceType": "m5.large",
+        "currentGeneration": "Yes",
+        "instanceFamily": "General purpose",
+        "vcpu": "2",
+        "physicalProcessor": "Intel Xeon Platinum 8175",
+        "clockSpeed": "3.1 GHz",
+        "memory": "8 GiB",
+        "storage": "EBS only",
+        "networkPerformance": "Up to 10 Gigabit",
+        "processorArchitecture": "64-bit",
+        "tenancy": "Shared",
+        "operatingSystem": "Linux",
+        "licenseModel": "No License required",
+        "usagetype": "USE2-BoxUsage:m5.large",
+        "operation": "RunInstances",
+        "availabilityzone": "NA",
+        "capacitystatus": "Used",
+        "classicnetworkingsupport": "false",
+        "dedicatedEbsThroughput": "Up to 2120 Mbps",
+        "ecu": "10",
+        "enhancedNetworkingSupported": "Yes",
+        "gpuMemory": "NA",
+        "intelAvxAvailable": "Yes",
+        "intelAvx2Available": "Yes",
+        "intelTurboAvailable": "Yes",
+        "marketoption": "OnDemand",
+        "normalizationSizeFactor": "4",
+        "preInstalledSw": "NA",
+        "processorFeatures": "Intel AVX; Intel AVX2; Intel AVX512; Intel Turbo",
+        "regionCode": "us-east-2",
+        "servicename": "Amazon Elastic Compute Cloud",
+        "vpcnetworkingsupport": "true"
+      }
+    },
+    "9ZEEN7WWWQKAG292": {
+      "sku": "9ZEEN7WWWQKAG292",
+      "productFamily": "Compute Instance",
+      "attributes": {
+        "servicecode": "AmazonEC2",
+        "location": "US East (Ohio)",
+        "locationType": "AWS Region",
+        "instanceType": "p3.8xlarge",
+        "currentGeneration": "Yes",
+        "instanceFamily": "GPU instance",
+        "vcpu": "32",
+        "physicalProcessor": "Intel Xeon E5-2686 v4 (Broadwell)",
+        "clockSpeed": "2.3 GHz",
+        "memory": "244 GiB",
+        "storage": "EBS only",
+        "networkPerformance": "10 Gigabit",
+        "processorArchitecture": "64-bit",
+        "tenancy": "Shared",
+        "operatingSystem": "Windows",
+        "licenseModel": "Bring your own license",
+        "usagetype": "USE2-BoxUsage:p3.8xlarge",
+        "operation": "RunInstances:0800",
+        "availabilityzone": "NA",
+        "capacitystatus": "Used",
+        "classicnetworkingsupport": "false",
+        "dedicatedEbsThroughput": "7000 Mbps",
+        "ecu": "97",
+        "enhancedNetworkingSupported": "Yes",
+        "gpu": "4",
+        "gpuMemory": "NA",
+        "intelAvxAvailable": "Yes",
+        "intelAvx2Available": "Yes",
+        "intelTurboAvailable": "Yes",
+        "marketoption": "OnDemand",
+        "normalizationSizeFactor": "64",
+        "preInstalledSw": "NA",
+        "processorFeatures": "Intel AVX; Intel AVX2; Intel Turbo",
+        "regionCode": "us-east-2",
+        "servicename": "Amazon Elastic Compute Cloud",
+        "vpcnetworkingsupport": "true"
+      }
+    },
+    "M6UGCCQ3CDJQAA37": {
+      "sku": "M6UGCCQ3CDJQAA37",
+      "productFamily": "Storage",
+      "attributes": {
+        "servicecode": "AmazonEC2",
+        "location": "US East (Ohio)",
+        "locationType": "AWS Region",
+        "storageMedia": "SSD-backed",
+        "volumeType": "General Purpose",
+        "maxVolumeSize": "16 TiB",
+        "maxIopsvolume": "16000",
+        "maxThroughputvolume": "1000 MiB/s",
+        "usagetype": "USE2-EBS:VolumeUsage.gp3",
+        "operation": "",
+        "regionCode": "us-east-2",
+        "servicename": "Amazon Elastic Compute Cloud",
+        "volumeApiName": "gp3"
+      }
+    },
+    "Y9RYMSE644KDSV4S": {
+      "sku": "Y9RYMSE644KDSV4S",
+      "productFamily": "Load Balancer-Network",
+      "attributes": {
+        "servicecode": "AmazonEC2",
+        "location": "US East (Ohio)",
+        "locationType": "AWS Region",
+        "group": "ELB:Balancer",
+        "groupDescription": "LoadBalancer hourly usage by Network Load Balancer",
+        "usagetype": "USE2-LoadBalancerUsage",
+        "operation": "LoadBalancing:Network",
+        "regionCode": "us-east-2",
+        "servicename": "Amazon Elastic Compute Cloud"
+      }
+    }
+  },
+  "terms": {
+    "OnDemand": {
+      "M6UGCCQ3CDJQAA37": {
+        "M6UGCCQ3CDJQAA37.JRTCKXETXF": {
+          "offerTermCode": "JRTCKXETXF",
+          "sku": "M6UGCCQ3CDJQAA37",
+          "effectiveDate": "2023-03-01T00:00:00Z",
+          "priceDimensions": {
+            "M6UGCCQ3CDJQAA37.JRTCKXETXF.6YS6EN2CT7": {
+              "rateCode": "M6UGCCQ3CDJQAA37.JRTCKXETXF.6YS6EN2CT7",
+              "description": "$0.08 per GB-month of General Purpose (gp3) provisioned storage - US East (Ohio)",
+              "beginRange": "0",
+              "endRange": "Inf",
+              "unit": "GB-Mo",
+              "pricePerUnit": {
+                "USD": "0.0800000000"
+              },
+              "appliesTo": []
+            }
+          },
+          "termAttributes": {}
+        }
+      },
+      "9ZEEN7WWWQKAG292": {
+        "9ZEEN7WWWQKAG292.JRTCKXETXF": {
+          "offerTermCode": "JRTCKXETXF",
+          "sku": "9ZEEN7WWWQKAG292",
+          "effectiveDate": "2023-03-01T00:00:00Z",
+          "priceDimensions": {
+            "9ZEEN7WWWQKAG292.JRTCKXETXF.6YS6EN2CT7": {
+              "rateCode": "9ZEEN7WWWQKAG292.JRTCKXETXF.6YS6EN2CT7",
+              "description": "$12.24 per On Demand Windows BYOL p3.8xlarge Instance Hour",
+              "beginRange": "0",
+              "endRange": "Inf",
+              "unit": "Hrs",
+              "pricePerUnit": {
+                "USD": "12.2400000000"
+              },
+              "appliesTo": []
+            }
+          },
+          "termAttributes": {}
+        }
+      },
+      "8D49XP354UEYTHGM": {
+        "8D49XP354UEYTHGM.MZU6U2429S": {
+          "offerTermCode": "MZU6U2429S",
+          "sku": "8D49XP354UEYTHGM",
+          "effectiveDate": "2019-01-01T00:00:00Z",
+          "priceDimensions": {
+            "8D49XP354UEYTHGM.MZU6U2429S.2TG2D8R56U": {
+              "rateCode": "8D49XP354UEYTHGM.MZU6U2429S.2TG2D8R56U",
+              "description": "Upfront Fee",
+              "unit": "Quantity",
+              "pricePerUnit": {
+                "USD": "1161"
+              },
+              "appliesTo": []
+            }
+          },
+          "termAttributes": {
+            "LeaseContractLength": "3yr",
+            "OfferingClass": "convertible",
+            "PurchaseOption": "All Upfront"
+          }
+        }
+      },
+      "Y9RYMSE644KDSV4S": {
+        "Y9RYMSE644KDSV4S.JRTCKXETXF": {
+          "offerTermCode": "JRTCKXETXF",
+          "sku": "Y9RYMSE644KDSV4S",
+          "effectiveDate": "2024-05-01T00:00:00Z",
+          "priceDimensions": {
+            "Y9RYMSE644KDSV4S.JRTCKXETXF.6YS6EN2CT7": {
+              "rateCode": "Y9RYMSE644KDSV4S.JRTCKXETXF.6YS6EN2CT7",
+              "description": "$0.0225 per Network LoadBalancer-hour (or partial hour)",
+              "beginRange": "0",
+              "endRange": "Inf",
+              "unit": "Hrs",
+              "pricePerUnit": {
+                "USD": "0.0225000000"
+              },
+              "appliesTo": []
+            }
+          },
+          "termAttributes": {}
+        }
+      }
+    },
+    "attributesList": {}
+  }
+}

pkg/cloud/azure/provider.go

@@ -286,10 +286,12 @@ func getRetailPrice(region string, skuName string, currencyCode string, spot boo
 	}
 
 	retailPrice := ""
+	spotPrice := ""
 	for _, item := range pricingPayload.Items {
 		if item.Type == "Consumption" && !strings.Contains(item.ProductName, "Windows") {
-			// if spot is true SkuName should contain "spot, if it is false it should not
-			if spot == strings.Contains(strings.ToLower(item.SkuName), " spot") {
+			if !strings.Contains(strings.ToLower(item.SkuName), " spot") {
+				spotPrice = fmt.Sprintf("%f", item.RetailPrice)
+			} else {
 				retailPrice = fmt.Sprintf("%f", item.RetailPrice)
 			}
 		}
@@ -297,6 +299,10 @@ func getRetailPrice(region string, skuName string, currencyCode string, spot boo
 
 	log.DedupedInfof(5, "done parsing retail price payload from \"%s\"\n", pricingURL)
 
+	if spot && spotPrice != "" {
+		return spotPrice, nil
+	}
+
 	if retailPrice == "" {
 		return retailPrice, fmt.Errorf("Couldn't find price for product \"%s\" in \"%s\" region", skuName, region)
 	}
@@ -847,7 +853,7 @@ func (az *Azure) DownloadPricingData() error {
 	// rate-card client is old, it can hang indefinitely in some cases
 	// this happens on the main thread, so it may block the whole app
 	// there is can be a better way to set timeout for the client
-	ctx, cancel := context.WithTimeout(context.TODO(), 60*time.Second)
+	ctx, cancel := context.WithTimeout(context.TODO(), 300*time.Second)
 	defer cancel()
 	result, err := rcClient.Get(ctx, rateCardFilter)
 	if err != nil {
@@ -934,6 +940,11 @@ func convertMeterToPricings(info commerce.MeterInfo, regions map[string]string,
 		return nil, nil
 	}
 
+	if strings.Contains(meterSubCategory, "Cloud Services") || strings.Contains(meterSubCategory, "CloudServices") {
+		// This meter doesn't correspond to any pricings.
+		return nil, nil
+	}
+
 	if strings.Contains(meterCategory, "Storage") {
 		if strings.Contains(meterSubCategory, "HDD") || strings.Contains(meterSubCategory, "SSD") || strings.Contains(meterSubCategory, "Premium Files") {
 			var storageClass string = ""
@@ -1085,68 +1096,71 @@ func (az *Azure) AllNodePricing() (interface{}, error) {
 func (az *Azure) NodePricing(key models.Key) (*models.Node, models.PricingMetadata, error) {
 	az.DownloadPricingDataLock.RLock()
 	defer az.DownloadPricingDataLock.RUnlock()
-	pricingDataExists := true
-	if az.Pricing == nil {
-		pricingDataExists = false
-		log.DedupedWarningf(1, "Unable to download Azure pricing data")
-	}
 
 	meta := models.PricingMetadata{}
 
+	if az.Pricing == nil {
+		return nil, meta, fmt.Errorf("Unable to download Azure pricing data")
+	}
+
 	azKey, ok := key.(*azureKey)
 	if !ok {
 		return nil, meta, fmt.Errorf("azure: NodePricing: key is of type %T", key)
 	}
 	config, _ := az.GetConfig()
 
-	// Spot Node
 	slv, ok := azKey.Labels[config.SpotLabel]
 	isSpot := ok && slv == config.SpotLabelValue && config.SpotLabel != "" && config.SpotLabelValue != ""
+
+	features := strings.Split(azKey.Features(), ",")
+	region := features[0]
+	instance := features[1]
+	var featureString string
 	if isSpot {
-		features := strings.Split(azKey.Features(), ",")
-		region := features[0]
-		instance := features[1]
-		spotFeatures := fmt.Sprintf("%s,%s,%s", region, instance, "spot")
-		if n, ok := az.Pricing[spotFeatures]; ok {
-			log.DedupedInfof(5, "Returning pricing for node %s: %+v from key %s", azKey, n, spotFeatures)
-			if azKey.isValidGPUNode() {
-				n.Node.GPU = "1" // TODO: support multiple GPUs
-			}
-			return n.Node, meta, nil
+		featureString = fmt.Sprintf("%s,%s,spot", region, instance)
+	} else {
+		featureString = azKey.Features()
+	}
+
+	if n, ok := az.Pricing[featureString]; ok {
+		log.Debugf("Returning pricing for node %s: %+v from key %s", azKey, n, azKey.Features())
+		if azKey.isValidGPUNode() {
+			n.Node.GPU = azKey.GetGPUCount()
 		}
-		log.Infof("[Info] found spot instance, trying to get retail price for %s: %s, ", spotFeatures, azKey)
-		spotCost, err := getRetailPrice(region, instance, config.CurrencyCode, true)
-		if err != nil {
-			log.DedupedWarningf(5, "failed to retrieve spot retail pricing")
-		} else {
-			gpu := ""
-			if azKey.isValidGPUNode() {
-				gpu = "1"
-			}
-			spotNode := &models.Node{
-				Cost:      spotCost,
+		return n.Node, meta, nil
+	}
+
+	cost, err := getRetailPrice(region, instance, config.CurrencyCode, isSpot)
+
+	if err != nil {
+		log.DedupedWarningf(5, "failed to retrieve retail pricing: %s", err)
+	} else {
+		gpu := ""
+		if azKey.isValidGPUNode() {
+			gpu = azKey.GetGPUCount()
+		}
+		var node *models.Node
+		if isSpot {
+			node = &models.Node{
+				Cost:      cost,
 				UsageType: "spot",
 				GPU:       gpu,
 			}
-			az.addPricing(spotFeatures, &AzurePricing{
-				Node: spotNode,
-			})
-			return spotNode, meta, nil
-		}
-	}
-
-	// Use the downloaded pricing data if possible. Otherwise, use default
-	// configured pricing data.
-	if pricingDataExists {
-		if n, ok := az.Pricing[azKey.Features()]; ok {
-			log.Debugf("Returning pricing for node %s: %+v from key %s", azKey, n, azKey.Features())
-			if azKey.isValidGPUNode() {
-				n.Node.GPU = azKey.GetGPUCount()
+		} else {
+			node = &models.Node{
+				Cost: cost,
+				GPU:  gpu,
 			}
-			return n.Node, meta, nil
 		}
-		log.DedupedWarningf(5, "No pricing data found for node %s from key %s", azKey, azKey.Features())
+
+		az.addPricing(featureString, &AzurePricing{
+			Node: node,
+		})
+		return node, meta, nil
 	}
+
+	log.DedupedWarningf(5, "No pricing data found for node %s from key %s", azKey, azKey.Features())
+
 	c, err := az.GetConfig()
 	if err != nil {
 		return nil, meta, fmt.Errorf("No default pricing data available")

pkg/cloud/oracle/partnumbers/shape_part_numbers.json

@@ -41,6 +41,12 @@
     "GPU": "B98415",
     "Disk": ""
   },
+  "BM.GPU.L40S.4": {
+    "OCPU": "",
+    "Memory": "",
+    "GPU": "B109479",
+    "Disk": ""
+  },
   "BM.GPU2.2": {
     "OCPU": "",
     "Memory": "",
@@ -185,6 +191,84 @@
     "GPU": "",
     "Disk": ""
   },
+  "PostgreSQL.VM.Standard.Flex.E4": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.16.256GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.2.32GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.32.512GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.4.64GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.64.1024GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.8.128GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex.16.256GB": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex.2.32GB": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex.32.512GB": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex.4.64GB": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex.8.128GB": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
   "VM.DenseIO.E4.Flex": {
     "OCPU": "B93121",
     "Memory": "B93122",
@@ -227,6 +311,12 @@
     "GPU": "",
     "Disk": ""
   },
+  "VM.DenseIO2.8": {
+    "OCPU": "B88516",
+    "Memory": "",
+    "GPU": "",
+    "Disk": ""
+  },
   "VM.GPU.A10.1": {
     "OCPU": "",
     "Memory": "",
@@ -275,6 +365,12 @@
     "GPU": "",
     "Disk": ""
   },
+  "VM.Standard.A2.Flex": {
+    "OCPU": "B109529",
+    "Memory": "B109530",
+    "GPU": "",
+    "Disk": ""
+  },
   "VM.Standard.E3.Flex": {
     "OCPU": "B92306",
     "Memory": "B92307",
@@ -335,4 +431,4 @@
     "GPU": "",
     "Disk": ""
   }
-}
+}

pkg/cloud/oracle/provider.go

@@ -255,7 +255,7 @@ func (o *Oracle) Regions() []string {
 		log.Debugf("Overriding Oracle regions with configured region list: %+v", regionOverrides)
 		return regionOverrides
 	}
-	return oracleRegions
+	return oracleRegions()
 }
 
 func (o *Oracle) PricingSourceSummary() interface{} {

pkg/cloud/oracle/provider_test.go

@@ -74,6 +74,11 @@ func TestGetPVKey(t *testing.T) {
 	assert.Equal(t, providerID, pvkey.ID())
 }
 
+func TestRegions(t *testing.T) {
+	regions := (&Oracle{}).Regions()
+	assert.Len(t, regions, 39)
+}
+
 func testNode(gpus int) *clustercache.Node {
 	capacity := map[v1.ResourceName]resource.Quantity{}
 	if gpus > 0 {

pkg/cloud/oracle/region.go

@@ -2,40 +2,46 @@ package oracle
 
 // Regions retrieved from https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm.
 // May also be listed using the OCI CLI, "oci iam region list"
-var oracleRegions = []string{
-	"eu-amsterdam-1",
-	"eu-stockholm-1",
-	"me-abudhabi-1",
-	"ap-mumbai-1",
-	"eu-paris-1",
-	"uk-cardiff-1",
-	"me-dubai-1",
-	"eu-frankfurt-1",
-	"sa-saopaulo-1",
-	"ap-hyderabad-1",
-	"us-ashburn-1",
-	"ap-seoul-1",
-	"me-jeddah-1",
-	"af-johannesburg-1",
-	"ap-osaka-1",
-	"uk-london-1",
-	"eu-milan-1",
-	"eu-madrid-1",
-	"ap-melbourne-1",
-	"eu-marseille-1",
-	"mx-monterrey-1",
-	"il-jerusalem-1",
-	"ap-tokyo-1",
-	"us-chicago-1",
-	"us-phoenix-1",
-	"mx-queretaro-1",
-	"sa-santiago-1",
-	"ap-singapore-1",
-	"us-sanjose-1",
-	"ap-sydney-1",
-	"sa-vinhedo-1",
-	"ap-chuncheon-1",
-	"ca-montreal-1",
-	"ca-toronto-1",
-	"eu-zurich-1",
+func oracleRegions() []string {
+	return []string{
+		"af-johannesburg-1",
+		"ap-chuncheon-1",
+		"ap-hyderabad-1",
+		"ap-melbourne-1",
+		"ap-mumbai-1",
+		"ap-osaka-1",
+		"ap-seoul-1",
+		"ap-singapore-1",
+		"ap-singapore-2",
+		"ap-sydney-1",
+		"ap-tokyo-1",
+		"ca-montreal-1",
+		"ca-toronto-1",
+		"eu-amsterdam-1",
+		"eu-frankfurt-1",
+		"eu-madrid-1",
+		"eu-marseille-1",
+		"eu-milan-1",
+		"eu-paris-1",
+		"eu-stockholm-1",
+		"eu-zurich-1",
+		"il-jerusalem-1",
+		"me-abudhabi-1",
+		"me-dubai-1",
+		"me-jeddah-1",
+		"me-riyadh-1",
+		"mx-monterrey-1",
+		"mx-queretaro-1",
+		"sa-bogota-1",
+		"sa-santiago-1",
+		"sa-saopaulo-1",
+		"sa-valparaiso-1",
+		"sa-vinhedo-1",
+		"uk-cardiff-1",
+		"uk-london-1",
+		"us-ashburn-1",
+		"us-chicago-1",
+		"us-phoenix-1",
+		"us-sanjose-1",
+	}
 }

pkg/cloud/oracle/region_test.go

@@ -0,0 +1,17 @@
+package oracle
+
+import (
+	"testing"
+
+	"github.com/oracle/oci-go-sdk/v65/common"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRegionValidation(t *testing.T) {
+	for _, r := range oracleRegions() {
+		// Use the OCI SDK to validate static regions.
+		region := common.StringToRegion(r)
+		_, err := region.RealmID()
+		assert.NoError(t, err)
+	}
+}

pkg/cloud/otc/provider.go

@@ -0,0 +1,586 @@
+package otc
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/opencost/opencost/core/pkg/log"
+	"github.com/opencost/opencost/core/pkg/opencost"
+	"github.com/opencost/opencost/core/pkg/util"
+	"github.com/opencost/opencost/pkg/cloud/models"
+	"github.com/opencost/opencost/pkg/clustercache"
+	"github.com/opencost/opencost/pkg/env"
+	v1 "k8s.io/api/core/v1"
+)
+
+// OTC node pricing attributes
+type OTCNodeAttributes struct {
+	Type  string // like s2.large.1
+	OS    string // like windows
+	Price string // (in EUR) like 0.023
+	RAM   string // (in GB) like 2
+	VCPU  string // like 8
+}
+
+type OTCPVAttributes struct {
+	Type  string // like vss.ssd
+	Price string // (in EUR/GB/h) like 0.01
+}
+
+// OTC pricing is either for a node, a persistent volume (or a database, network, cluster, ...)
+type OTCPricing struct {
+	NodeAttributes *OTCNodeAttributes
+	PVAttributes   *OTCPVAttributes
+}
+
+// the main provider struct
+type OTC struct {
+	Clientset               clustercache.ClusterCache
+	Pricing                 map[string]*OTCPricing
+	Config                  models.ProviderConfig
+	ClusterRegion           string
+	projectID               string
+	clusterManagementPrice  float64
+	BaseCPUPrice            string
+	BaseRAMPrice            string
+	BaseGPUPrice            string
+	ValidPricingKeys        map[string]bool
+	DownloadPricingDataLock sync.RWMutex
+}
+
+// Kubernetes to OTC OS conversion
+/* Note:
+Kubernetes cannot fill the "kubernetes.io/os" label with the variety that OTC provides
+because it is based on the runtime.GOOS variable (https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-os)
+which can only contain some os. The pricing between everything but windows differs
+by about 5ct - 30ct per hour, so most os get treated like Open Linux.
+*/
+var kubernetesOSTypes = map[string]string{
+	"linux":        "Open Linux",
+	"windows":      "Windows",
+	"Open Linux":   "linux",
+	"Oracle Linux": "linux",
+	"SUSE Linux":   "linux",
+	"SUSE for SAP": "linux",
+	"RedHat Linux": "linux",
+	"Windows":      "windows",
+}
+
+// Currently assumes that no GPU is present
+// but aws does that too, so its fine.
+type otcKey struct {
+	ProviderID string
+	Labels     map[string]string
+}
+
+func (k *otcKey) GPUCount() int {
+	return 0
+}
+
+func (k *otcKey) GPUType() string {
+	return ""
+}
+
+func (k *otcKey) ID() string {
+	return k.ProviderID
+}
+
+type otcPVKey struct {
+	RegionID               string
+	Type                   string
+	Size                   string // in GB
+	Labels                 map[string]string
+	ProviderId             string
+	StorageClassParameters map[string]string
+}
+
+func (k *otcPVKey) Features() string {
+	fmt.Printf("features for pv %s", k.ID())
+	return k.RegionID + "," + k.Type
+}
+
+func (k *otcKey) Features() string {
+	instanceType, _ := util.GetInstanceType(k.Labels)
+	operatingSystem, _ := util.GetOperatingSystem(k.Labels)
+	ClusterRegion, _ := util.GetRegion(k.Labels)
+
+	key := ClusterRegion + "," + instanceType + "," + operatingSystem
+	return key
+}
+
+// Extract/generate a key that holds the data required to calculate
+// the cost of the given node (like s2.large.4).
+func (otc *OTC) GetKey(labels map[string]string, n *v1.Node) models.Key {
+	return &otcKey{
+		Labels:     labels,
+		ProviderID: labels["providerID"],
+	}
+}
+
+// Returns the storage class for a persistent volume key.
+func (k *otcPVKey) GetStorageClass() string {
+	return k.Type
+}
+
+// Returns the provider id for a persistent volume key.
+func (k *otcPVKey) ID() string {
+	return k.ProviderId
+}
+
+func (otc *OTC) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) models.PVKey {
+	providerID := ""
+	return &otcPVKey{
+		Labels:                 pv.Labels,
+		Type:                   pv.Spec.StorageClassName,
+		StorageClassParameters: parameters,
+		RegionID:               defaultRegion,
+		ProviderId:             providerID,
+	}
+}
+
+// Takes a resopnse from the otc api and the respective service name as an input
+// and extracts the resulting data into a product slice.
+func (otc *OTC) loadStructFromResponse(resp http.Response, serviceName string) ([]Product, error) {
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	// Unmarshal the first bit of the response.
+	wrapper := make(map[string]map[string]interface{})
+	err = json.Unmarshal(body, &wrapper)
+	if err != nil {
+		return nil, err
+	}
+
+	// Unmarshal the second, more specific, bit of the response.
+	data := make(map[string][]Product)
+	tmp, err := json.Marshal(wrapper["response"]["result"])
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(tmp, &data)
+	if err != nil {
+		return nil, err
+	}
+
+	return data[serviceName], nil
+}
+
+// The product (price) data that is fetched from OTC
+//
+// If OsUnit, VCpu and Ram aren't given, the product
+// is a persistent volume, else it's a node.
+type Product struct {
+	OpiFlavour  string `json:"opiFlavour"`
+	OsUnit      string `json:"osUnit,omitempty"`
+	PriceAmount string `json:"priceAmount"`
+	VCpu        string `json:"vCpu,omitempty"`
+	Ram         string `json:"ram,omitempty"`
+}
+
+/*
+Download the pricing data from the OTC API
+
+When a node has a specified price of e.g. 0.014 and
+the kubernetes node has a RAM attribute of 8232873984 Bytes.
+
+The price in Prometheus will be composed of:
+  - the cpu/h price multiplied with the amount of VCPUs:
+    0.006904 * 1 => 0.006904
+  - the RAM/h price multiplied with the amount of ram in GiB:
+    0.000925 * (8232873984/1024/1024/1024) => 0.0070924
+
+And the resulting node_total_hourly_price{} metric in Prometheus
+will approach the total node cost retrieved from OTC:
+
+	==> 0.006904 + 0.0070924 = 0.013996399999999999
+	    ~ 0.014
+*/
+func (otc *OTC) DownloadPricingData() error {
+	otc.DownloadPricingDataLock.Lock()
+	defer otc.DownloadPricingDataLock.Unlock()
+
+	// Fetch pricing data from the otc.json config in case downloading the pricing maps fails.
+	c, err := otc.Config.GetCustomPricingData()
+	if err != nil {
+		log.Errorf("Error downloading default pricing data: %s", err.Error())
+	}
+
+	otc.BaseCPUPrice = c.CPU
+	otc.BaseRAMPrice = c.RAM
+	otc.BaseGPUPrice = c.GPU
+	otc.clusterManagementPrice = 0.10 // TODO: What is the cluster management price?
+	otc.projectID = c.ProjectID
+
+	// Slice with all nodes currently present in the cluster.
+	nodeList := otc.Clientset.GetAllNodes()
+
+	// Slice with all storage classes.
+	storageClasses := otc.Clientset.GetAllStorageClasses()
+	for _, tmp := range storageClasses {
+		fmt.Println("storage class found:")
+		fmt.Println(tmp.Parameters)
+		fmt.Println(tmp.Labels)
+		fmt.Println(tmp.TypeMeta)
+		fmt.Println(tmp.Size())
+	}
+
+	// Slice with all persistent volumes present in the cluster
+	pvList := otc.Clientset.GetAllPersistentVolumes()
+
+	// Create a slice of all existing keys in the current cluster.
+	// (keys like "eu-de,s3.medium.1,linux" or "eu-de,s3.xlarge.2,windows")
+	inputkeys := make(map[string]bool)
+	tmp := []string{}
+	for _, node := range nodeList {
+		labels := node.GetObjectMeta().GetLabels()
+		key := otc.GetKey(labels, node)
+		inputkeys[key.Features()] = true
+		tmp = append(tmp, key.Features())
+	}
+	for _, pv := range pvList {
+		fmt.Println("storage class name \"" + pv.Spec.StorageClassName + "\" found")
+		key := otc.GetPVKey(pv, map[string]string{}, "eu-de")
+		inputkeys[key.Features()] = true
+		tmp = append(tmp, key.Features())
+	}
+
+	otc.Pricing = make(map[string]*OTCPricing)
+	otc.ValidPricingKeys = make(map[string]bool)
+
+	// Get pricing data from API.
+	nodePricingURL := "https://calculator.otc-service.com/de/open-telekom-price-api/?serviceName=ecs" /* + "&limitMax=200"*/ + "&columns%5B1%5D=opiFlavour" + "&columns%5B2%5D=osUnit" + "&columns%5B3%5D=vCpu" + "&columns%5B4%5D=ram" + "&columns%5B5%5D=priceAmount"
+	pvPricingURL := "https://calculator.otc-service.com/de/open-telekom-price-api/?serviceName%5B0%5D=evs&columns%5B1%5D=opiFlavour&columns%5B2%5D=priceAmount&limitFrom=0&region%5B3%5D=eu-de"
+
+	log.Info("Started downloading OTC pricing data...")
+	resp, err := http.Get(nodePricingURL)
+	if err != nil {
+		return err
+	}
+	pvResp, err := http.Get(pvPricingURL)
+	if err != nil {
+		return err
+	}
+	log.Info("Succesfully downloaded OTC pricing data")
+
+	var products []Product
+
+	nodeProducts, err := otc.loadStructFromResponse(*resp, "ecs")
+	if err != nil {
+		return err
+	}
+	products = append(products, nodeProducts...)
+	pvProducts, err := otc.loadStructFromResponse(*pvResp, "evs")
+	if err != nil {
+		return err
+	}
+	products = append(products, pvProducts...)
+
+	// convert the otc-reponse product-structs to opencost-compatible node structs
+	const ClusterRegion = "eu-de"
+	for _, product := range products {
+		var productPricing *OTCPricing
+		var key string
+		// if os is empty the product must be a persistent volume
+		if product.OsUnit == "" {
+			productPricing = &OTCPricing{
+				PVAttributes: &OTCPVAttributes{
+					Type:  product.OpiFlavour,
+					Price: strings.Split(strings.ReplaceAll(product.PriceAmount, ",", "."), " ")[0],
+				},
+			}
+			key = ClusterRegion + "," + productPricing.PVAttributes.Type
+		} else {
+			// else it must be a node
+			adjustedOS := kubernetesOSTypes[product.OsUnit]
+			productPricing = &OTCPricing{
+				NodeAttributes: &OTCNodeAttributes{
+					Type:  product.OpiFlavour,
+					OS:    adjustedOS,
+					Price: strings.Split(strings.ReplaceAll(product.PriceAmount, ",", "."), " ")[0],
+					RAM:   strings.Split(product.Ram, " ")[0],
+					VCPU:  product.VCpu,
+				},
+			}
+			key = ClusterRegion + "," + productPricing.NodeAttributes.Type + "," + productPricing.NodeAttributes.OS
+		}
+
+		// create a key similiar to the ones created with otcKey.Features()
+		// so that the pricing data can be fetched using an otcKey
+		log.Info("product \"" + key + "\" found")
+
+		otc.Pricing[key] = productPricing
+		otc.ValidPricingKeys[key] = true
+	}
+
+	return nil
+}
+
+func (otc *OTC) NetworkPricing() (*models.Network, error) {
+	cpricing, err := otc.Config.GetCustomPricingData()
+	if err != nil {
+		return nil, err
+	}
+	znec, err := strconv.ParseFloat(cpricing.ZoneNetworkEgress, 64)
+	if err != nil {
+		return nil, err
+	}
+	rnec, err := strconv.ParseFloat(cpricing.RegionNetworkEgress, 64)
+	if err != nil {
+		return nil, err
+	}
+	inec, err := strconv.ParseFloat(cpricing.InternetNetworkEgress, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	return &models.Network{
+		ZoneNetworkEgressCost:     znec,
+		RegionNetworkEgressCost:   rnec,
+		InternetNetworkEgressCost: inec,
+	}, nil
+}
+
+// NodePricing(Key) (*Node, PricingMetadata, error)
+// Read the keys features and determine the price of the Node described by
+// the key to construct a Pricing Node object to return and work with.
+func (otc *OTC) NodePricing(k models.Key) (*models.Node, models.PricingMetadata, error) {
+	otc.DownloadPricingDataLock.RLock()
+	defer otc.DownloadPricingDataLock.RUnlock()
+
+	key := k.Features()
+	meta := models.PricingMetadata{}
+
+	log.Info("looking for pricing data of node with key features " + key)
+	pricing, ok := otc.Pricing[key]
+	if ok {
+		// The pricing key was found in the pricing list of the otc provider.
+		// Now create a pricing node from that data and return it.
+		log.Info("pricing data found")
+		return otc.createNode(pricing, k)
+	} else if _, ok := otc.ValidPricingKeys[key]; ok {
+		// The pricing key is actually valid, but somehow it could not be found.
+		// Try re-downloading the pricing data to check for changes.
+		log.Info("key is valid, but no associated pricing data could be found; trying to re-download pricing data")
+		otc.DownloadPricingDataLock.RUnlock()
+		err := otc.DownloadPricingData()
+		otc.DownloadPricingDataLock.RLock()
+		if err != nil {
+			return &models.Node{
+				Cost:             otc.BaseCPUPrice,
+				BaseCPUPrice:     otc.BaseCPUPrice,
+				BaseRAMPrice:     otc.BaseRAMPrice,
+				BaseGPUPrice:     otc.BaseGPUPrice,
+				UsesBaseCPUPrice: true,
+			}, meta, err
+		}
+		pricing, ok = otc.Pricing[key]
+		if !ok {
+			// The given key does not exist in OTC or locally, return a default pricing node.
+			return &models.Node{
+				Cost:             otc.BaseCPUPrice,
+				BaseCPUPrice:     otc.BaseCPUPrice,
+				BaseRAMPrice:     otc.BaseRAMPrice,
+				BaseGPUPrice:     otc.BaseGPUPrice,
+				UsesBaseCPUPrice: true,
+			}, meta, fmt.Errorf("unable to find any Pricing data for \"%s\"", key)
+		}
+		// The local pricing date was just outdated.
+		log.Info("pricing data found after re-download")
+		return otc.createNode(pricing, k)
+	} else {
+		// The given key is not valid, fall back to base pricing (handled by the costmodel)?
+		log.Info("given key \"" + key + "\" is invalid; falling back to default pricing")
+		return nil, meta, fmt.Errorf("invalid Pricing Key \"%s\"", key)
+	}
+}
+
+// create a Pricing Node from the internal pricing struct and a key describing the kubernetes node
+func (otc *OTC) createNode(pricing *OTCPricing, key models.Key) (*models.Node, models.PricingMetadata, error) {
+	// aws does some fancy stuff here, but it probably isn't that necessary
+
+	// so just return the pricing node constructed directly from the internal struct
+	meta := models.PricingMetadata{}
+	return &models.Node{
+		Cost:         pricing.NodeAttributes.Price,
+		VCPU:         pricing.NodeAttributes.VCPU,
+		RAM:          pricing.NodeAttributes.RAM,
+		BaseCPUPrice: otc.BaseCPUPrice,
+		BaseRAMPrice: otc.BaseRAMPrice,
+		BaseGPUPrice: otc.BaseGPUPrice,
+	}, meta, nil
+}
+
+// give the order to read the custom provider config file
+func (otc *OTC) GetConfig() (*models.CustomPricing, error) {
+	c, err := otc.Config.GetCustomPricingData()
+	if err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+// load balancer cost
+// taken straight up from aws
+func (otc *OTC) LoadBalancerPricing() (*models.LoadBalancer, error) {
+	return &models.LoadBalancer{
+		Cost: 0.05,
+	}, nil
+}
+
+// returns general info about the cluster
+// This method HAS to be overwritten as long as the CustomProvider
+// Field of the OTC struct is not set when initializing the provider
+// in "provider.go" (see all the other providers).
+func (otc *OTC) ClusterInfo() (map[string]string, error) {
+	c, err := otc.GetConfig()
+	if err != nil {
+		return nil, err
+	}
+	m := make(map[string]string)
+	m["name"] = "OTC Cluster #1"
+	if clusterName := otc.getClusterName(c); clusterName != "" {
+		m["name"] = clusterName
+	}
+	m["provider"] = opencost.OTCProvider
+	m["account"] = c.ProjectID
+	m["region"] = otc.ClusterRegion
+	m["remoteReadEnabled"] = strconv.FormatBool(env.IsRemoteEnabled())
+	m["id"] = env.GetClusterID()
+	return m, nil
+}
+
+func (otc *OTC) getClusterName(cfg *models.CustomPricing) string {
+	if cfg.ClusterName != "" {
+		return cfg.ClusterName
+	}
+	for _, node := range otc.Clientset.GetAllNodes() {
+		if clusterName, ok := node.Labels["name"]; ok {
+			return clusterName
+		}
+	}
+	return ""
+}
+
+// search for pricing data matching the given persistent volume key
+// in the provider's pricing list and return it
+func (otc *OTC) PVPricing(pvk models.PVKey) (*models.PV, error) {
+	pricing, ok := otc.Pricing[pvk.Features()]
+	if !ok {
+		log.Info("Persistent Volume pricing not found for features \"" + pvk.Features() + "\"")
+		log.Info("continuing with pricing for \"eu-de,vss.ssd\"")
+		pricing, ok = otc.Pricing["eu-de,vss.ssd"]
+		if !ok {
+			log.Errorf("something went wrong, the DownloadPricing method probably didn't execute correctly")
+			return &models.PV{}, nil
+		}
+	}
+
+	// otc pv pricing is in the format: price per GB per month
+	// this convertes that to: GB price per hour
+	hourly, err := strconv.ParseFloat(pricing.PVAttributes.Price, 32)
+	if err != nil {
+		return &models.PV{}, err
+	}
+	hourly = hourly / 730
+
+	return &models.PV{
+		Cost:  fmt.Sprintf("%v", hourly),
+		Class: pricing.PVAttributes.Type,
+	}, nil
+
+}
+
+// TODO: Implement method
+func (otc *OTC) GetAddresses() ([]byte, error) {
+	return []byte{}, nil
+}
+
+// TODO: Implement method
+func (otc *OTC) GetDisks() ([]byte, error) {
+	return []byte{}, nil
+}
+
+// TODO: Implement method
+func (otc *OTC) GetOrphanedResources() ([]models.OrphanedResource, error) {
+	return []models.OrphanedResource{}, nil
+}
+
+// TODO: Implement method
+func (otc *OTC) AllNodePricing() (interface{}, error) {
+	return nil, nil
+}
+
+// TODO: Implement method
+func (otc *OTC) UpdateConfig(r io.Reader, updateType string) (*models.CustomPricing, error) {
+	return &models.CustomPricing{}, nil
+}
+
+// TODO: Implement method
+func (otc *OTC) UpdateConfigFromConfigMap(configMap map[string]string) (*models.CustomPricing, error) {
+	return &models.CustomPricing{}, nil
+}
+
+// TODO: Implement method
+func (otc *OTC) GetManagementPlatform() (string, error) {
+	return "", nil
+}
+
+// TODO: Implement method
+func (otc *OTC) GetLocalStorageQuery(start, end time.Duration, isPVC, isDeleted bool) string {
+	return ""
+}
+
+// TODO: Implement method
+func (otc *OTC) ApplyReservedInstancePricing(nodes map[string]*models.Node) {
+}
+
+func (otc *OTC) ServiceAccountStatus() *models.ServiceAccountStatus {
+	return &models.ServiceAccountStatus{
+		Checks: []*models.ServiceAccountCheck{},
+	}
+}
+
+// TODO: Implement method
+func (otc *OTC) PricingSourceStatus() map[string]*models.PricingSource {
+	return map[string]*models.PricingSource{}
+}
+
+// TODO: Implement method
+func (otc *OTC) ClusterManagementPricing() (string, float64, error) {
+	return "", 0.0, nil
+}
+
+func (otc *OTC) CombinedDiscountForNode(nodeType string, reservedInstance bool, defaultDiscount, negotiatedDiscount float64) float64 {
+	return 1.0 - ((1.0 - defaultDiscount) * (1.0 - negotiatedDiscount))
+}
+
+// Regions retrieved from https://www.open-telekom-cloud.com/de/business-navigator/hochverfuegbare-rechenzentren
+var otcRegions = []string{
+	"eu-de",
+	"eu-nl",
+}
+
+func (otc *OTC) Regions() []string {
+	regionOverrides := env.GetRegionOverrideList()
+	if len(regionOverrides) > 0 {
+		log.Debugf("Overriding OTC regions with configured region list: %+v", regionOverrides)
+		return regionOverrides
+	}
+	return otcRegions
+}
+
+// PricingSourceSummary returns the pricing source summary for the provider.
+// The summary represents what was _parsed_ from the pricing source, not what
+// was returned from the relevant API.
+func (otc *OTC) PricingSourceSummary() interface{} {
+	// encode the pricing source summary as a JSON string
+	return otc.Pricing
+}

pkg/cloud/provider/provider.go

@@ -16,6 +16,7 @@ import (
 	"github.com/opencost/opencost/pkg/cloud/gcp"
 	"github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/cloud/oracle"
+	"github.com/opencost/opencost/pkg/cloud/otc"
 	"github.com/opencost/opencost/pkg/cloud/scaleway"
 
 	"github.com/opencost/opencost/core/pkg/opencost"
@@ -249,6 +250,13 @@ func NewProvider(cache clustercache.ClusterCache, apiKey string, config *config.
 			ClusterAccountID:     cp.accountID,
 			ServiceAccountChecks: models.NewServiceAccountChecks(),
 		}, nil
+	case opencost.OTCProvider:
+		log.Info("Found node label \"cce.cloud.com/cce-nodepool\", using OTC Provider")
+		return &otc.OTC{
+			Clientset:     cache,
+			Config:        NewProviderConfig(config, cp.configFileName),
+			ClusterRegion: cp.region,
+		}, nil
 	default:
 		log.Info("Unsupported provider, falling back to default")
 		return &CustomProvider{
@@ -312,6 +320,9 @@ func getClusterProperties(node *clustercache.Node) clusterProperties {
 	} else if strings.HasPrefix(providerID, "ocid") {
 		cp.provider = opencost.OracleProvider
 		cp.configFileName = "oracle.json"
+	} else if _, ok := node.Labels["cce.cloud.com/cce-nodepool"]; ok { // The node label "cce.cloud.com/cce-nodepool" exists
+		cp.provider = opencost.OTCProvider
+		cp.configFileName = "otc.json"
 	}
 	// Override provider to CSV if CSVProvider is used and custom provider is not set
 	if env.IsUseCSVProvider() {

pkg/cloud/provider/providerconfig.go

@@ -15,6 +15,7 @@ import (
 	"github.com/opencost/opencost/pkg/cloud/gcp"
 	"github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/cloud/oracle"
+	"github.com/opencost/opencost/pkg/cloud/otc"
 	"github.com/opencost/opencost/pkg/cloud/utils"
 	"github.com/opencost/opencost/pkg/config"
 	"github.com/opencost/opencost/pkg/env"
@@ -320,6 +321,8 @@ func ExtractConfigFromProviders(prov models.Provider) models.ProviderConfig {
 		return p.Config
 	case *oracle.Oracle:
 		return p.Config
+	case *otc.OTC:
+		return p.Config
 	default:
 		log.Errorf("failed to extract config from provider")
 		return nil

pkg/clustercache/watchcontroller.go

@@ -88,7 +88,19 @@ func NewCachingWatcher(restClient rest.Interface, resource string, resourceType
 }
 
 func (c *CachingWatchController) GetAll() []interface{} {
-	return c.indexer.List()
+	list := c.indexer.List()
+
+	// since the indexer returns the as-is pointer to the resource,
+	// we deep copy the resources such that callers don't corrupt the
+	// index
+	cloneList := make([]interface{}, 0, len(list))
+	for _, v := range list {
+		if deepCopyable, ok := v.(rt.Object); ok {
+			cloneList = append(cloneList, deepCopyable.DeepCopyObject())
+		}
+	}
+
+	return cloneList
 }
 
 func (c *CachingWatchController) SetUpdateHandler(handler WatchHandler) WatchController {

pkg/costmodel/allocation.go

@@ -29,6 +29,7 @@ const (
 	queryFmtCPUUsageAvg                 = `avg(rate(container_cpu_usage_seconds_total{container!="", container_name!="POD", container!="POD", %s}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
 	queryFmtGPUsRequested               = `avg(avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
 	queryFmtGPUsUsageAvg                = `avg(avg_over_time(DCGM_FI_PROF_GR_ENGINE_ACTIVE{container!=""}[%s])) by (container, pod, namespace, %s)`
+	queryFmtGPUsUsageMax                = `max(max_over_time(DCGM_FI_PROF_GR_ENGINE_ACTIVE{container!=""}[%s])) by (container, pod, namespace, %s)`
 	queryFmtGPUsAllocated               = `avg(avg_over_time(container_gpu_allocation{container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
 	queryFmtNodeCostPerCPUHr            = `avg(avg_over_time(node_cpu_hourly_cost{%s}[%s])) by (node, %s, instance_type, provider_id)`
 	queryFmtNodeCostPerRAMGiBHr         = `avg(avg_over_time(node_ram_hourly_cost{%s}[%s])) by (node, %s, instance_type, provider_id)`
@@ -66,6 +67,8 @@ const (
 	queryFmtLBActiveMins                = `count(kubecost_load_balancer_cost{%s}) by (namespace, service_name, %s)[%s:%s]`
 	queryFmtOldestSample                = `min_over_time(timestamp(group(node_cpu_hourly_cost{%s}))[%s:%s])`
 	queryFmtNewestSample                = `max_over_time(timestamp(group(node_cpu_hourly_cost{%s}))[%s:%s])`
+	queryFmtIsGPuShared                 = `avg(avg_over_time(kube_pod_container_resource_requests{container!="", node != "", pod != "", container!= "", unit = "integer",  %s}[%s])) by (container, pod, namespace, node, resource)`
+	queryFmtGetGPuInfo                  = `avg(avg_over_time(DCGM_FI_DEV_DEC_UTIL{container!="",%s}[%s])) by (container, pod, namespace, device, modelName, UUID)`
 
 	// Because we use container_cpu_usage_seconds_total to calculate CPU usage
 	// at any given "instant" of time, we need to use an irate or rate. To then
@@ -276,6 +279,13 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 			if alloc.RawAllocationOnly.RAMBytesUsageMax > resultAlloc.RawAllocationOnly.RAMBytesUsageMax {
 				resultAlloc.RawAllocationOnly.RAMBytesUsageMax = alloc.RawAllocationOnly.RAMBytesUsageMax
 			}
+
+			if alloc.RawAllocationOnly.GPUUsageMax != nil {
+				if *alloc.RawAllocationOnly.GPUUsageMax > *resultAlloc.RawAllocationOnly.GPUUsageMax {
+					resultAlloc.RawAllocationOnly.GPUUsageMax = alloc.RawAllocationOnly.GPUUsageMax
+				}
+			}
+
 		}
 	}
 
@@ -426,12 +436,17 @@ func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Dur
 		}
 	}
 
+	// GPU Queries
+	//queryIsGpuShared := fmt.Sprintf(queryFmtIsGPuShared, durStr)
 	queryGPUsRequested := fmt.Sprintf(queryFmtGPUsRequested, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChGPUsRequested := ctx.QueryAtTime(queryGPUsRequested, end)
 
 	queryGPUsUsageAvg := fmt.Sprintf(queryFmtGPUsUsageAvg, durStr, env.GetPromClusterLabel())
 	resChGPUsUsageAvg := ctx.Query(queryGPUsUsageAvg)
 
+	queryGPUsUsageMax := fmt.Sprintf(queryFmtGPUsUsageMax, durStr, env.GetPromClusterLabel())
+	resChGPUsUsageMax := ctx.Query(queryGPUsUsageMax)
+
 	queryGPUsAllocated := fmt.Sprintf(queryFmtGPUsAllocated, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChGPUsAllocated := ctx.QueryAtTime(queryGPUsAllocated, end)
 
@@ -492,6 +507,13 @@ func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Dur
 	queryNetInternetCostPerGiB := fmt.Sprintf(queryFmtNetInternetCostPerGiB, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChNetInternetCostPerGiB := ctx.QueryAtTime(queryNetInternetCostPerGiB, end)
 
+	//GPU Queries
+	queryIsGpuShared := fmt.Sprintf(queryFmtIsGPuShared, env.GetPromClusterFilter(), durStr)
+	resChIsGpuShared := ctx.QueryAtTime(queryIsGpuShared, end)
+
+	queryGetGPUInfo := fmt.Sprintf(queryFmtGetGPuInfo, env.GetPromClusterFilter(), durStr)
+	resChGetGPUInfo := ctx.QueryAtTime(queryGetGPUInfo, end)
+
 	var resChNodeLabels prom.QueryResultsChan
 	if env.GetAllocationNodeLabelsEnabled() {
 		queryNodeLabels := fmt.Sprintf(queryFmtNodeLabels, env.GetPromClusterFilter(), durStr)
@@ -549,8 +571,12 @@ func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Dur
 	resRAMUsageMax, _ := resChRAMUsageMax.Await()
 	resGPUsRequested, _ := resChGPUsRequested.Await()
 	resGPUsUsageAvg, _ := resChGPUsUsageAvg.Await()
+	resGPUsUsageMax, _ := resChGPUsUsageMax.Await()
 	resGPUsAllocated, _ := resChGPUsAllocated.Await()
 
+	resIsGpuShared, _ := resChIsGpuShared.Await()
+	resGetGPUInfo, _ := resChGetGPUInfo.Await()
+
 	resNodeCostPerCPUHr, _ := resChNodeCostPerCPUHr.Await()
 	resNodeCostPerRAMGiBHr, _ := resChNodeCostPerRAMGiBHr.Await()
 	resNodeCostPerGPUHr, _ := resChNodeCostPerGPUHr.Await()
@@ -615,7 +641,10 @@ func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Dur
 	applyRAMBytesRequested(podMap, resRAMRequests, podUIDKeyMap)
 	applyRAMBytesUsedAvg(podMap, resRAMUsageAvg, podUIDKeyMap)
 	applyRAMBytesUsedMax(podMap, resRAMUsageMax, podUIDKeyMap)
-	applyGPUUsageAvg(podMap, resGPUsUsageAvg, podUIDKeyMap)
+	applyGPUUsage(podMap, resGPUsUsageAvg, podUIDKeyMap, GpuUsageAverageMode)
+	applyGPUUsage(podMap, resGPUsUsageMax, podUIDKeyMap, GpuUsageMaxMode)
+	applyGPUUsage(podMap, resIsGpuShared, podUIDKeyMap, GpuIsSharedMode)
+	applyGPUUsage(podMap, resGetGPUInfo, podUIDKeyMap, GpuInfoMode)
 	applyGPUsAllocated(podMap, resGPUsRequested, resGPUsAllocated, podUIDKeyMap)
 	applyNetworkTotals(podMap, resNetTransferBytes, resNetReceiveBytes, podUIDKeyMap)
 	applyNetworkAllocation(podMap, resNetZoneGiB, resNetZoneCostPerGiB, podUIDKeyMap, networkCrossZoneCost)

pkg/costmodel/allocation_helpers.go

@@ -29,6 +29,13 @@ const TiB = 1024.0 * GiB
 const PiB = 1024.0 * TiB
 const PV_USAGE_SANITY_LIMIT_BYTES = 10.0 * PiB
 
+const (
+	GpuUsageAverageMode = "AVERAGE"
+	GpuUsageMaxMode     = "MAX"
+	GpuIsSharedMode     = "SHARED"
+	GpuInfoMode         = "GPU_INFO"
+)
+
 /* Pod Helpers */
 
 func (cm *CostModel) buildPodMap(window opencost.Window, resolution, maxBatchSize time.Duration, podMap map[podKey]*pod, clusterStart, clusterEnd map[string]time.Time, ingestPodUID bool, podUIDKeyMap map[podKey][]podKey) error {
@@ -614,12 +621,13 @@ func applyRAMBytesUsedMax(podMap map[podKey]*pod, resRAMBytesUsedMax []*prom.Que
 	}
 }
 
-func applyGPUUsageAvg(podMap map[podKey]*pod, resGPUUsageAvg []*prom.QueryResult, podUIDKeyMap map[podKey][]podKey) {
+// same func is used for both GPUUsageAvg and GPUUsageMax
+func applyGPUUsage(podMap map[podKey]*pod, resGPUUsageAvgOrMax []*prom.QueryResult, podUIDKeyMap map[podKey][]podKey, mode string) {
 	// Example PromQueryResult: {container="dcgmproftester12", namespace="gpu", pod="dcgmproftester3-deployment-fc89c8dd6-ph7z5"} 0.997307
-	for _, res := range resGPUUsageAvg {
+	for _, res := range resGPUUsageAvgOrMax {
 		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
 		if err != nil {
-			log.DedupedWarningf(10, "CostModel.ComputeAllocation: GPU usage avg result missing field: %s", err)
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: GPU usage avg/max result missing field: %s", err)
 			continue
 		}
 
@@ -642,7 +650,7 @@ func applyGPUUsageAvg(podMap map[podKey]*pod, resGPUUsageAvg []*prom.QueryResult
 		for _, thisPod := range pods {
 			container, err := res.GetString("container")
 			if err != nil {
-				log.DedupedWarningf(10, "CostModel.ComputeAllocation: GPU usage avg query result missing 'container': %s", key)
+				log.DedupedWarningf(10, "CostModel.ComputeAllocation: GPU usage avg/max query result missing 'container': %s", key)
 				continue
 			}
 			if _, ok := thisPod.Allocations[container]; !ok {
@@ -650,7 +658,64 @@ func applyGPUUsageAvg(podMap map[podKey]*pod, resGPUUsageAvg []*prom.QueryResult
 			}
 
 			// DCGM_FI_PROF_GR_ENGINE_ACTIVE metric is a float between 0-1.
-			thisPod.Allocations[container].GPUUsageAverage = res.Values[0].Value
+			switch mode {
+			case GpuUsageAverageMode:
+
+				if thisPod.Allocations[container].GPUAllocation == nil {
+					thisPod.Allocations[container].GPUAllocation = &opencost.GPUAllocation{GPUUsageAverage: &res.Values[0].Value}
+				} else {
+					thisPod.Allocations[container].GPUAllocation.GPUUsageAverage = &res.Values[0].Value
+				}
+			case GpuUsageMaxMode:
+				if thisPod.Allocations[container].RawAllocationOnly == nil {
+					thisPod.Allocations[container].RawAllocationOnly = &opencost.RawAllocationOnlyData{
+						GPUUsageMax: &res.Values[0].Value,
+					}
+				} else {
+					thisPod.Allocations[container].RawAllocationOnly.GPUUsageMax = &res.Values[0].Value
+				}
+			case GpuIsSharedMode:
+				// if a container is using a GPU and it is shared, isGPUShared will be true
+				// if a container is using GPU and it is NOT shared, isGPUShared will be false
+				// if a container is NOT using a GPU, isGPUShared will be null
+				if res.Metric["resource"] == "nvidia_com_gpu_shared" {
+					trueVal := true
+					if res.Values[0].Value == 1 {
+						if thisPod.Allocations[container].GPUAllocation == nil {
+
+							thisPod.Allocations[container].GPUAllocation = &opencost.GPUAllocation{IsGPUShared: &trueVal}
+						} else {
+							thisPod.Allocations[container].GPUAllocation.IsGPUShared = &trueVal
+						}
+					}
+				} else if res.Metric["resource"] == "nvidia_com_gpu" {
+					falseVal := false
+					if res.Values[0].Value == 1 {
+						if thisPod.Allocations[container].GPUAllocation == nil {
+							thisPod.Allocations[container].GPUAllocation = &opencost.GPUAllocation{IsGPUShared: &falseVal}
+						} else {
+							thisPod.Allocations[container].GPUAllocation.IsGPUShared = &falseVal
+						}
+					}
+				} else {
+					continue
+				}
+			case GpuInfoMode:
+				if thisPod.Allocations[container].GPUAllocation == nil {
+					thisPod.Allocations[container].GPUAllocation = &opencost.GPUAllocation{
+						GPUDevice: getSanitizedDeviceName(fmt.Sprintf("%s", res.Metric["device_name"])),
+						GPUModel:  fmt.Sprintf("%s", res.Metric["modelName"]),
+						GPUUUID:   fmt.Sprintf("%s", res.Metric["UUID"]),
+					}
+				} else {
+					thisPod.Allocations[container].GPUAllocation.GPUDevice = getSanitizedDeviceName(fmt.Sprintf("%s", res.Metric["device"]))
+					thisPod.Allocations[container].GPUAllocation.GPUModel = fmt.Sprintf("%s", res.Metric["modelName"])
+					thisPod.Allocations[container].GPUAllocation.GPUUUID = fmt.Sprintf("%s", res.Metric["UUID"])
+				}
+
+			default:
+				log.DedupedInfof(10, "CostModel.ComputeAllocation: Unknown mode: %s", mode)
+			}
 		}
 	}
 }
@@ -702,7 +767,14 @@ func applyGPUsAllocated(podMap map[podKey]*pod, resGPUsRequested []*prom.QueryRe
 			// Therefore max(usage,request) will always equal request. In the
 			// future this may need to be refactored when building support for
 			// GPU Time Slicing.
-			thisPod.Allocations[container].GPURequestAverage = res.Values[0].Value
+
+			if thisPod.Allocations[container].GPUAllocation == nil {
+				thisPod.Allocations[container].GPUAllocation = &opencost.GPUAllocation{
+					GPURequestAverage: &res.Values[0].Value,
+				}
+			} else {
+				thisPod.Allocations[container].GPUAllocation.GPURequestAverage = &res.Values[0].Value
+			}
 		}
 	}
 }
@@ -2324,3 +2396,11 @@ func calculateStartAndEnd(result *prom.QueryResult, resolution time.Duration, wi
 
 	return s, e
 }
+
+func getSanitizedDeviceName(deviceName string) string {
+	if strings.Contains(deviceName, "nvidia") {
+		return "nvidia"
+	}
+
+	return deviceName
+}

pkg/customcost/ingestor.go

@@ -337,14 +337,14 @@ func (ing *CustomCostIngestor) run() {
 			// Wait for next tick
 		}
 
-		// Start from the last covered time, minus the RunWindow
-		start := ing.lastRun
-		start = start.Add(-ing.resolution)
-
 		queryWin := ing.config.DailyQueryWindow
 		if ing.resolution == time.Hour {
 			queryWin = ing.config.HourlyQueryWindow
 		}
+		// Start from the last covered time, minus the query window
+		// this allows re-querying of data as the plugin providers' data may stabilize over time
+		start := ing.lastRun
+		start = start.Add(-1 * queryWin)
 
 		// Round start time back to the nearest Resolution point in the past from the
 		// last update to the QueryWindow

pkg/customcost/queryservice_helper.go

@@ -40,17 +40,61 @@ func ParseCustomCostTotalRequest(qp mapper.PrimitiveMap) (*CostTotalRequest, err
 		}
 	}
 
+	costTypeStr := qp.Get("costType", string(CostTypeBlended))
+	parsedCostType, err := ParseCostType(costTypeStr)
+	if err != nil {
+		return nil, fmt.Errorf("parsing 'costType' parameter: %s", err)
+	}
+
+	sortByStr := qp.Get("sortBy", string(SortPropertyCost))
+	parsedSortBy, err := ParseSortBy(sortByStr)
+	if err != nil {
+		return nil, fmt.Errorf("parsing 'sortBy' parameter: %s", err)
+	}
+
+	sortDirStr := qp.Get("sortDirection", string(SortDirectionDesc))
+	parsedSortDir, err := ParseSortDirection(sortDirStr)
+	if err != nil {
+		return nil, fmt.Errorf("parsing 'sortDirection' parameter: %s", err)
+	}
+
 	opts := &CostTotalRequest{
-		Start:       *window.Start(),
-		End:         *window.End(),
-		AggregateBy: aggregateBy,
-		Accumulate:  accumulate,
-		Filter:      filter,
+		Start:         *window.Start(),
+		End:           *window.End(),
+		AggregateBy:   aggregateBy,
+		Accumulate:    accumulate,
+		Filter:        filter,
+		CostType:      parsedCostType,
+		SortBy:        parsedSortBy,
+		SortDirection: parsedSortDir,
 	}
 
 	return opts, nil
 }
 
+func ParseSortDirection(sortDirStr string) (SortDirection, error) {
+	switch sortDirStr {
+	case string(SortDirectionAsc):
+		return SortDirectionAsc, nil
+	case string(SortDirectionDesc):
+		return SortDirectionDesc, nil
+	default:
+		return "", fmt.Errorf("unrecognized sortDirection field: %s", sortDirStr)
+	}
+}
+
+func ParseSortBy(sortByStr string) (SortProperty, error) {
+	switch sortByStr {
+	case string(SortPropertyCost):
+		return SortPropertyCost, nil
+	case string(SortPropertyAggregate):
+		return SortPropertyAggregate, nil
+	case string(SortPropertyCostType):
+		return SortPropertyCostType, nil
+	default:
+		return "", fmt.Errorf("unrecognized sortBy field: %s", sortByStr)
+	}
+}
 func ParseCustomCostTimeseriesRequest(qp mapper.PrimitiveMap) (*CostTimeseriesRequest, error) {
 	windowStr := qp.Get("window", "")
 	if windowStr == "" {
@@ -82,13 +126,33 @@ func ParseCustomCostTimeseriesRequest(qp mapper.PrimitiveMap) (*CostTimeseriesRe
 			return nil, fmt.Errorf("parsing 'filter' parameter: %s", err)
 		}
 	}
+	costTypeStr := qp.Get("costType", string(CostTypeBlended))
+	parsedCostType, err := ParseCostType(costTypeStr)
+	if err != nil {
+		return nil, fmt.Errorf("parsing 'costType' parameter: %s", err)
+	}
+
+	sortByStr := qp.Get("sortBy", string(SortPropertyCost))
+	parsedSortBy, err := ParseSortBy(sortByStr)
+	if err != nil {
+		return nil, fmt.Errorf("parsing 'sortBy' parameter: %s", err)
+	}
+
+	sortDirStr := qp.Get("sortDirection", string(SortDirectionDesc))
+	parsedSortDir, err := ParseSortDirection(sortDirStr)
+	if err != nil {
+		return nil, fmt.Errorf("parsing 'sortDirection' parameter: %s", err)
+	}
 
 	opts := &CostTimeseriesRequest{
-		Start:       *window.Start(),
-		End:         *window.End(),
-		AggregateBy: aggregateBy,
-		Accumulate:  accumulate,
-		Filter:      filter,
+		Start:         *window.Start(),
+		End:           *window.End(),
+		AggregateBy:   aggregateBy,
+		Accumulate:    accumulate,
+		Filter:        filter,
+		CostType:      parsedCostType,
+		SortBy:        parsedSortBy,
+		SortDirection: parsedSortDir,
 	}
 
 	return opts, nil

pkg/customcost/repositoryquerier.go

@@ -63,7 +63,7 @@ func (rq *RepositoryQuerier) QueryTotal(ctx context.Context, request CostTotalRe
 				continue
 			}
 
-			customCosts := ParseCustomCostResponse(ccResponse)
+			customCosts := ParseCustomCostResponse(ccResponse, request.CostType)
 			for _, customCost := range customCosts {
 				if matcher.Matches(customCost) {
 					ccs.Add(customCost)
@@ -79,7 +79,8 @@ func (rq *RepositoryQuerier) QueryTotal(ctx context.Context, request CostTotalRe
 		return nil, err
 	}
 
-	return NewCostResponse(ccs), nil
+	ccs.Sort(request.SortBy, request.SortDirection)
+	return NewCostResponse(ccs, request.CostType), nil
 }
 
 func (rq *RepositoryQuerier) QueryTimeseries(ctx context.Context, request CostTimeseriesRequest) (*CostTimeseriesResponse, error) {
@@ -105,11 +106,14 @@ func (rq *RepositoryQuerier) QueryTimeseries(ctx context.Context, request CostTi
 		go func(i int, window opencost.Window, res []*CostResponse) {
 			defer wg.Done()
 			totals[i], errors[i] = rq.QueryTotal(ctx, CostTotalRequest{
-				Start:       *window.Start(),
-				End:         *window.End(),
-				AggregateBy: request.AggregateBy,
-				Filter:      request.Filter,
-				Accumulate:  accumulate,
+				Start:         *window.Start(),
+				End:           *window.End(),
+				AggregateBy:   request.AggregateBy,
+				Filter:        request.Filter,
+				Accumulate:    accumulate,
+				CostType:      request.CostType,
+				SortBy:        request.SortBy,
+				SortDirection: request.SortDirection,
 			})
 		}(i, w, totals)
 	}

pkg/customcost/types.go

@@ -2,54 +2,79 @@ package customcost
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 	"time"
 
 	"github.com/opencost/opencost/core/pkg/filter"
+	"github.com/opencost/opencost/core/pkg/log"
 	"github.com/opencost/opencost/core/pkg/model/pb"
 	"github.com/opencost/opencost/core/pkg/opencost"
 )
 
+type CostType string
+type SortProperty string
+type SortDirection string
+
+const (
+	CostTypeBlended CostType = "blended"
+	CostTypeList    CostType = "list"
+	CostTypeBilled  CostType = "billed"
+
+	SortPropertyCost      SortProperty = "cost"
+	SortPropertyAggregate SortProperty = "aggregate"
+	SortPropertyCostType  SortProperty = "costType"
+
+	SortDirectionAsc  SortDirection = "asc"
+	SortDirectionDesc SortDirection = "desc"
+)
+
 type CostTotalRequest struct {
-	Start       time.Time
-	End         time.Time
-	AggregateBy []CustomCostProperty
-	Accumulate  opencost.AccumulateOption
-	Filter      filter.Filter
+	Start         time.Time
+	End           time.Time
+	AggregateBy   []CustomCostProperty
+	Accumulate    opencost.AccumulateOption
+	Filter        filter.Filter
+	CostType      CostType
+	SortBy        SortProperty
+	SortDirection SortDirection
 }
 
 type CostTimeseriesRequest struct {
-	Start       time.Time
-	End         time.Time
-	AggregateBy []CustomCostProperty
-	Accumulate  opencost.AccumulateOption
-	Filter      filter.Filter
+	Start         time.Time
+	End           time.Time
+	AggregateBy   []CustomCostProperty
+	Accumulate    opencost.AccumulateOption
+	Filter        filter.Filter
+	CostType      CostType
+	SortBy        SortProperty
+	SortDirection SortDirection
 }
 
 type CostResponse struct {
-	Window          opencost.Window `json:"window"`
-	TotalBilledCost float32         `json:"totalBilledCost"`
-	TotalListCost   float32         `json:"totalListCost"`
-	CustomCosts     []*CustomCost   `json:"customCosts"`
+	Window        opencost.Window `json:"window"`
+	TotalCost     float32         `json:"totalCost"`
+	TotalCostType CostType        `json:"totalCostType"`
+	CustomCosts   []*CustomCost   `json:"customCosts"`
 }
 
 type CustomCost struct {
-	Id             string  `json:"id"`
-	Zone           string  `json:"zone"`
-	AccountName    string  `json:"account_name"`
-	ChargeCategory string  `json:"charge_category"`
-	Description    string  `json:"description"`
-	ResourceName   string  `json:"resource_name"`
-	ResourceType   string  `json:"resource_type"`
-	ProviderId     string  `json:"provider_id"`
-	BilledCost     float32 `json:"billedCost"`
-	ListCost       float32 `json:"listCost"`
-	ListUnitPrice  float32 `json:"list_unit_price"`
-	UsageQuantity  float32 `json:"usage_quantity"`
-	UsageUnit      string  `json:"usage_unit"`
-	Domain         string  `json:"domain"`
-	CostSource     string  `json:"cost_source"`
-	Aggregate      string  `json:"aggregate"`
+	Id             string   `json:"id"`
+	Zone           string   `json:"zone"`
+	AccountName    string   `json:"account_name"`
+	ChargeCategory string   `json:"charge_category"`
+	Description    string   `json:"description"`
+	ResourceName   string   `json:"resource_name"`
+	ResourceType   string   `json:"resource_type"`
+	ProviderId     string   `json:"provider_id"`
+	Cost           float32  `json:"cost"`
+	ListUnitPrice  float32  `json:"list_unit_price"`
+	UsageQuantity  float32  `json:"usage_quantity"`
+	UsageUnit      string   `json:"usage_unit"`
+	Domain         string   `json:"domain"`
+	CostSource     string   `json:"cost_source"`
+	Aggregate      string   `json:"aggregate"`
+	CostType       CostType `json:"cost_type"`
 }
 
 type CostTimeseriesResponse struct {
@@ -57,27 +82,45 @@ type CostTimeseriesResponse struct {
 	Timeseries []*CostResponse `json:"timeseries"`
 }
 
-func NewCostResponse(ccs *CustomCostSet) *CostResponse {
+func NewCostResponse(ccs *CustomCostSet, costType CostType) *CostResponse {
 	costResponse := &CostResponse{
-		Window:      ccs.Window,
-		CustomCosts: []*CustomCost{},
+		Window:        ccs.Window,
+		CustomCosts:   []*CustomCost{},
+		TotalCostType: costType,
 	}
 
 	for _, cc := range ccs.CustomCosts {
-		costResponse.TotalBilledCost += cc.BilledCost
-		costResponse.TotalListCost += cc.ListCost
+		costResponse.TotalCost += cc.Cost
 		costResponse.CustomCosts = append(costResponse.CustomCosts, cc)
 	}
 
 	return costResponse
 }
 
-func ParseCustomCostResponse(ccResponse *pb.CustomCostResponse) []*CustomCost {
+func ParseCostType(costTypeStr string) (CostType, error) {
+	switch costTypeStr {
+	case string(CostTypeBlended):
+		return CostTypeBlended, nil
+	case string(CostTypeList):
+		return CostTypeList, nil
+	case string(CostTypeBilled):
+		return CostTypeBilled, nil
+	default:
+		return "", fmt.Errorf("unsupported cost type: %s", costTypeStr)
+	}
+}
+
+func ParseCustomCostResponse(ccResponse *pb.CustomCostResponse, costType CostType) []*CustomCost {
 	costs := ccResponse.GetCosts()
 
-	customCosts := make([]*CustomCost, len(costs))
-	for i, cost := range costs {
-		customCosts[i] = &CustomCost{
+	customCosts := []*CustomCost{}
+	for _, cost := range costs {
+		selectedCost, selectedCostType := determineCost(cost, costType)
+		if selectedCost == 0 {
+			log.Debugf("cost %s had 0 cost for cost type %s, not including in response", cost.ProviderId, costType)
+			continue
+		}
+		customCosts = append(customCosts, &CustomCost{
 			Id:             cost.GetId(),
 			Zone:           cost.GetZone(),
 			AccountName:    cost.GetAccountName(),
@@ -86,24 +129,47 @@ func ParseCustomCostResponse(ccResponse *pb.CustomCostResponse) []*CustomCost {
 			ResourceName:   cost.GetResourceName(),
 			ResourceType:   cost.GetResourceType(),
 			ProviderId:     cost.GetProviderId(),
-			BilledCost:     cost.GetBilledCost(),
-			ListCost:       cost.GetListCost(),
+			Cost:           selectedCost,
 			ListUnitPrice:  cost.GetListUnitPrice(),
 			UsageQuantity:  cost.GetUsageQuantity(),
 			UsageUnit:      cost.GetUsageUnit(),
 			Domain:         ccResponse.GetDomain(),
 			CostSource:     ccResponse.GetCostSource(),
-		}
+			CostType:       selectedCostType,
+		})
 	}
 
 	return customCosts
 }
 
+func determineCost(cc *pb.CustomCost, costType CostType) (float32, CostType) {
+	switch costType {
+	// if the cost type is blended, first check if the billed cost is non-zero
+	// if it is, return the billed cost
+	// if it is zero, return the list cost
+	case CostTypeBlended:
+		if cc.BilledCost > 0 {
+			return cc.BilledCost, CostTypeBilled
+		}
+		return cc.ListCost, CostTypeList
+		// if the cost type is list, return the list cost
+	case CostTypeList:
+		return cc.ListCost, CostTypeList
+		// if the cost type is billed, return the billed cost
+	case CostTypeBilled:
+		return cc.BilledCost, CostTypeBilled
+	default:
+		return 0, ""
+	}
+}
 func (cc *CustomCost) Add(other *CustomCost) {
-	cc.BilledCost += other.BilledCost
-	cc.ListCost += other.ListCost
+	cc.Cost += other.Cost
 	cc.ListUnitPrice += other.ListUnitPrice
 
+	if cc.CostType != other.CostType {
+		cc.CostType = CostTypeBlended
+	}
+
 	if cc.Id != other.Id {
 		cc.Id = ""
 	}
@@ -240,3 +306,39 @@ func generateAggKey(cc *CustomCost, aggregateBy []CustomCostProperty) (string, e
 
 	return aggKey, nil
 }
+
+func (ccs *CustomCostSet) Sort(sortBy SortProperty, sortDirection SortDirection) {
+
+	switch sortBy {
+	case SortPropertyCost:
+		if sortDirection == SortDirectionAsc {
+			sort.Slice(ccs.CustomCosts, func(i, j int) bool {
+				return ccs.CustomCosts[i].Cost < ccs.CustomCosts[j].Cost
+			})
+		} else {
+			sort.Slice(ccs.CustomCosts, func(i, j int) bool {
+				return ccs.CustomCosts[i].Cost > ccs.CustomCosts[j].Cost
+			})
+		}
+	case SortPropertyAggregate:
+		if sortDirection == SortDirectionAsc {
+			sort.Slice(ccs.CustomCosts, func(i, j int) bool {
+				return ccs.CustomCosts[i].Aggregate < ccs.CustomCosts[j].Aggregate
+			})
+		} else {
+			sort.Slice(ccs.CustomCosts, func(i, j int) bool {
+				return ccs.CustomCosts[i].Aggregate > ccs.CustomCosts[j].Aggregate
+			})
+		}
+	case SortPropertyCostType:
+		if sortDirection == SortDirectionAsc {
+			sort.Slice(ccs.CustomCosts, func(i, j int) bool {
+				return ccs.CustomCosts[i].CostType < ccs.CustomCosts[j].CostType
+			})
+		} else {
+			sort.Slice(ccs.CustomCosts, func(i, j int) bool {
+				return ccs.CustomCosts[i].CostType > ccs.CustomCosts[j].CostType
+			})
+		}
+	}
+}

pkg/customcost/types_test.go

@@ -0,0 +1,299 @@
+package customcost
+
+import (
+	"testing"
+
+	"github.com/opencost/opencost/core/pkg/model/pb"
+)
+
+func TestSortByCostAsc(t *testing.T) {
+	ccs := &CustomCostSet{
+		CustomCosts: []*CustomCost{
+			{Cost: 300.0},
+			{Cost: 100.0},
+			{Cost: 200.0},
+		},
+	}
+
+	ccs.Sort(SortPropertyCost, SortDirectionAsc)
+
+	expected := []float32{100.0, 200.0, 300.0}
+	for i, cc := range ccs.CustomCosts {
+		if cc.Cost != expected[i] {
+			t.Errorf("expected cost %f, got %f", expected[i], cc.Cost)
+		}
+	}
+}
+
+func TestSortByCostDesc(t *testing.T) {
+	ccs := &CustomCostSet{
+		CustomCosts: []*CustomCost{
+			{Cost: 300.0},
+			{Cost: 100.0},
+			{Cost: 200.0},
+		},
+	}
+
+	ccs.Sort(SortPropertyCost, SortDirectionDesc)
+
+	expected := []float32{300.0, 200.0, 100.0}
+	for i, cc := range ccs.CustomCosts {
+		if cc.Cost != expected[i] {
+			t.Errorf("expected cost %f, got %f", expected[i], cc.Cost)
+		}
+	}
+}
+
+func TestSortByAggregateAsc(t *testing.T) {
+	ccs := &CustomCostSet{
+		CustomCosts: []*CustomCost{
+			{Aggregate: "c"},
+			{Aggregate: "a"},
+			{Aggregate: "b"},
+		},
+	}
+
+	ccs.Sort(SortPropertyAggregate, SortDirectionAsc)
+
+	expected := []string{"a", "b", "c"}
+	for i, cc := range ccs.CustomCosts {
+		if cc.Aggregate != expected[i] {
+			t.Errorf("expected aggregate %s, got %s", expected[i], cc.Aggregate)
+		}
+	}
+}
+
+func TestSortByAggregateDesc(t *testing.T) {
+	ccs := &CustomCostSet{
+		CustomCosts: []*CustomCost{
+			{Aggregate: "c"},
+			{Aggregate: "a"},
+			{Aggregate: "b"},
+		},
+	}
+
+	ccs.Sort(SortPropertyAggregate, SortDirectionDesc)
+
+	expected := []string{"c", "b", "a"}
+	for i, cc := range ccs.CustomCosts {
+		if cc.Aggregate != expected[i] {
+			t.Errorf("expected aggregate %s, got %s", expected[i], cc.Aggregate)
+		}
+	}
+}
+
+func TestSortByCostTypeAsc(t *testing.T) {
+	ccs := &CustomCostSet{
+		CustomCosts: []*CustomCost{
+			{CostType: CostTypeBilled},
+			{CostType: CostTypeList},
+			{CostType: CostTypeBlended},
+		},
+	}
+
+	ccs.Sort(SortPropertyCostType, SortDirectionAsc)
+
+	expected := []CostType{CostTypeBilled, CostTypeBlended, CostTypeList}
+	for i, cc := range ccs.CustomCosts {
+		if cc.CostType != expected[i] {
+			t.Errorf("expected cost type %s, got %s", expected[i], cc.CostType)
+		}
+	}
+}
+
+func TestSortByCostTypeDesc(t *testing.T) {
+	ccs := &CustomCostSet{
+		CustomCosts: []*CustomCost{
+			{CostType: CostTypeBilled},
+			{CostType: CostTypeList},
+			{CostType: CostTypeBlended},
+		},
+	}
+
+	ccs.Sort(SortPropertyCostType, SortDirectionDesc)
+
+	expected := []CostType{CostTypeList, CostTypeBlended, CostTypeBilled}
+	for i, cc := range ccs.CustomCosts {
+		if cc.CostType != expected[i] {
+			t.Errorf("expected cost type %s, got %s", expected[i], cc.CostType)
+		}
+	}
+}
+
+func TestParseCustomCostResponse(t *testing.T) {
+	tests := []struct {
+		name       string
+		ccResponse *pb.CustomCostResponse
+		costType   CostType
+		expected   []*CustomCost
+	}{
+		{
+			name: "BlendedCost",
+			ccResponse: &pb.CustomCostResponse{
+				Costs: []*pb.CustomCost{
+					{
+						Id:             "1",
+						Zone:           "us-east-1a",
+						AccountName:    "account1",
+						ChargeCategory: "category1",
+						Description:    "description1",
+						ResourceName:   "resource1",
+						ResourceType:   "type1",
+						ProviderId:     "provider1",
+						BilledCost:     100.0,
+						ListCost:       120.0,
+						ListUnitPrice:  1.2,
+						UsageQuantity:  100,
+						UsageUnit:      "unit1",
+					},
+				},
+				Domain:     "domain1",
+				CostSource: "source1",
+			},
+			costType: CostTypeBlended,
+			expected: []*CustomCost{
+				{
+					Id:             "1",
+					Zone:           "us-east-1a",
+					AccountName:    "account1",
+					ChargeCategory: "category1",
+					Description:    "description1",
+					ResourceName:   "resource1",
+					ResourceType:   "type1",
+					ProviderId:     "provider1",
+					Cost:           100.0,
+					ListUnitPrice:  1.2,
+					UsageQuantity:  100,
+					UsageUnit:      "unit1",
+					Domain:         "domain1",
+					CostSource:     "source1",
+					CostType:       CostTypeBilled,
+				},
+			},
+		},
+		{
+			name: "ListCost",
+			ccResponse: &pb.CustomCostResponse{
+				Costs: []*pb.CustomCost{
+					{
+						Id:             "2",
+						Zone:           "us-west-2b",
+						AccountName:    "account2",
+						ChargeCategory: "category2",
+						Description:    "description2",
+						ResourceName:   "resource2",
+						ResourceType:   "type2",
+						ProviderId:     "provider2",
+						BilledCost:     0.0,
+						ListCost:       150.0,
+						ListUnitPrice:  1.5,
+						UsageQuantity:  100,
+						UsageUnit:      "unit2",
+					},
+				},
+				Domain:     "domain2",
+				CostSource: "source2",
+			},
+			costType: CostTypeList,
+			expected: []*CustomCost{
+				{
+					Id:             "2",
+					Zone:           "us-west-2b",
+					AccountName:    "account2",
+					ChargeCategory: "category2",
+					Description:    "description2",
+					ResourceName:   "resource2",
+					ResourceType:   "type2",
+					ProviderId:     "provider2",
+					Cost:           150.0,
+					ListUnitPrice:  1.5,
+					UsageQuantity:  100,
+					UsageUnit:      "unit2",
+					Domain:         "domain2",
+					CostSource:     "source2",
+					CostType:       CostTypeList,
+				},
+			},
+		},
+		{
+			name: "ZeroCost",
+			ccResponse: &pb.CustomCostResponse{
+				Costs: []*pb.CustomCost{
+					{
+						Id:             "3",
+						Zone:           "us-central-1c",
+						AccountName:    "account3",
+						ChargeCategory: "category3",
+						Description:    "description3",
+						ResourceName:   "resource3",
+						ResourceType:   "type3",
+						ProviderId:     "provider3",
+						BilledCost:     0.0,
+						ListCost:       0.0,
+						ListUnitPrice:  0.0,
+						UsageQuantity:  0,
+						UsageUnit:      "unit3",
+					},
+				},
+				Domain:     "domain3",
+				CostSource: "source3",
+			},
+			costType: CostTypeBlended,
+			expected: []*CustomCost{},
+		},
+		{
+			name: "Non Matching cost",
+			ccResponse: &pb.CustomCostResponse{
+				Costs: []*pb.CustomCost{
+					{
+						Id:             "3",
+						Zone:           "us-central-1c",
+						AccountName:    "account3",
+						ChargeCategory: "category3",
+						Description:    "description3",
+						ResourceName:   "resource3",
+						ResourceType:   "type3",
+						ProviderId:     "provider3",
+						BilledCost:     0.0,
+						ListCost:       1.0,
+						ListUnitPrice:  0.0,
+						UsageQuantity:  0,
+						UsageUnit:      "unit3",
+					},
+				},
+				Domain:     "domain3",
+				CostSource: "source3",
+			},
+			costType: CostTypeBilled,
+			expected: []*CustomCost{},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := ParseCustomCostResponse(tt.ccResponse, tt.costType)
+			if len(result) != len(tt.expected) {
+				t.Errorf("expected %d custom costs, got %d", len(tt.expected), len(result))
+			}
+			for i, expectedCost := range tt.expected {
+				if result[i].Id != expectedCost.Id ||
+					result[i].Zone != expectedCost.Zone ||
+					result[i].AccountName != expectedCost.AccountName ||
+					result[i].ChargeCategory != expectedCost.ChargeCategory ||
+					result[i].Description != expectedCost.Description ||
+					result[i].ResourceName != expectedCost.ResourceName ||
+					result[i].ResourceType != expectedCost.ResourceType ||
+					result[i].ProviderId != expectedCost.ProviderId ||
+					result[i].Cost != expectedCost.Cost ||
+					result[i].ListUnitPrice != expectedCost.ListUnitPrice ||
+					result[i].UsageQuantity != expectedCost.UsageQuantity ||
+					result[i].UsageUnit != expectedCost.UsageUnit ||
+					result[i].Domain != expectedCost.Domain ||
+					result[i].CostSource != expectedCost.CostSource ||
+					result[i].CostType != expectedCost.CostType {
+					t.Errorf("expected %+v, got %+v", expectedCost, result[i])
+				}
+			}
+		})
+	}
+}

pkg/env/costmodelenv.go

@@ -72,6 +72,7 @@ const (
 	MultiClusterBearerToken       = "MC_BEARER_TOKEN"
 
 	InsecureSkipVerify = "INSECURE_SKIP_VERIFY"
+	KubeRbacProxyEnabled = "KUBE_RBAC_PROXY_ENABLED"
 
 	KubeConfigPathEnvVar = "KUBECONFIG_PATH"
 
@@ -382,6 +383,10 @@ func GetInsecureSkipVerify() bool {
 	return env.GetBool(InsecureSkipVerify, false)
 }
 
+func IsKubeRbacProxyEnabled() bool {
+	return env.GetBool(KubeRbacProxyEnabled, false)
+}
+
 // IsAggregateCostModelCacheDisabled returns the environment variable value for DisableAggregateCostModelCache which
 // will inform the aggregator on whether to load cached data. Defaults to false
 func IsAggregateCostModelCacheDisabled() bool {

pkg/prom/prom.go

@@ -3,6 +3,7 @@ package prom
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"net"
 	"net/http"
@@ -17,10 +18,13 @@ import (
 	"github.com/opencost/opencost/core/pkg/util/fileutil"
 	"github.com/opencost/opencost/core/pkg/util/httputil"
 	"github.com/opencost/opencost/core/pkg/version"
+	"github.com/opencost/opencost/pkg/env"
 
 	golog "log"
 
 	prometheus "github.com/prometheus/client_golang/api"
+	restclient "k8s.io/client-go/rest"
+	certutil "k8s.io/client-go/util/cert"
 )
 
 var UserAgent = fmt.Sprintf("Opencost/%s", version.Version)
@@ -374,6 +378,22 @@ type PrometheusClientConfig struct {
 
 // NewPrometheusClient creates a new rate limited client which limits by outbound concurrent requests.
 func NewPrometheusClient(address string, config *PrometheusClientConfig) (prometheus.Client, error) {
+
+	var tlsCaCert *x509.CertPool
+	// We will use the service account token and service-ca.crt to authenticate with the Prometheus server via kube-rbac-proxy.
+	// We need to ensure that the service account has the necessary permissions to access the Prometheus server by binding it to the appropriate role.
+	if env.IsKubeRbacProxyEnabled() {
+		restConfig, err := restclient.InClusterConfig()
+		if err != nil {
+			log.Errorf("KUBE_RBAC_PROXY_ENABLED was set to true but failed to get in-cluster config: %s", err)
+		}
+		config.Auth.BearerToken = restConfig.BearerToken
+		tlsCaCert, err = certutil.NewPool(`/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt`)
+		if err != nil {
+			log.Errorf("KUBE_RBAC_PROXY_ENABLED was set to true but failed to load service-ca.crt: %s", err)
+		}
+	}
+
 	// may be necessary for long prometheus queries
 	rt := httputil.NewUserAgentTransport(UserAgent, &http.Transport{
 		Proxy: http.ProxyFromEnvironment,
@@ -384,6 +404,7 @@ func NewPrometheusClient(address string, config *PrometheusClientConfig) (promet
 		TLSHandshakeTimeout: config.TLSHandshakeTimeout,
 		TLSClientConfig: &tls.Config{
 			InsecureSkipVerify: config.TLSInsecureSkipVerify,
+			RootCAs:            tlsCaCert,
 		},
 	})
 	pc := prometheus.Config{

spec/opencost-specv01.md

@@ -1,13 +1,13 @@
 # OpenCost Specification
 
 
-The OpenCost Spec is a vendor-neutral specification for measuring and allocating infrastructure and container costs in Kubernetes environments.
+The OpenCost Specification is a vendor-neutral specification for measuring and allocating infrastructure and container costs in Kubernetes environments.
 
 
 ## Introduction
 
 
-Kubernetes enables complex deployments of containerized workloads, which are often transient and consume variable amounts of cluster resources. While this enables teams to construct powerful solutions to a broad range of technical problems, it also creates complexities when measuring the resource utilization and costs of workloads and their associated infrastructure within the  dynamics of shared Kubernetes environments.
+Kubernetes enables complex deployments of containerized workloads, which are often transient and consume variable amounts of cluster resources. While this enables teams to construct powerful solutions to a broad range of technical problems, it also creates complexities when measuring the resource utilization and costs of workloads and their associated infrastructure within the dynamics of shared Kubernetes environments.
 
 
 As Kubernetes adoption increases within an organization, these complexities become a business-critical challenge to solve. In this document, we specify a vendor-agnostic methodology for accurately measuring and allocating the costs of a Kubernetes cluster to its hosted tenants. This community resource is maintained by Kubernetes practitioners and we welcome all contributions.
@@ -15,7 +15,7 @@ As Kubernetes adoption increases within an organization, these complexities beco
 
 ## Foundational definitions
 
-**Total Cluster Costs** represent all costs required to operate a Kubernetes cluster. **Cluster Assets Costs** are the portion of these costs that are related to directly observable entities within a cluster; these include expenses from nodes, persistent volumes, attached disks, load balancers, and network ingress/egress costs. From a financial accounting perspective, these are equivalent to the Cost of Goods Sold when measuring product costs. **Cluster Overhead Costs** measure the overhead required to operate all of the Assets of a cluster, e.g. Cluster Management Fees. These are the equivalent to Selling, General and Administrative Expenses (SG&A), or indirect costs, when viewed from a financial accounting perspective.
+**Total Cluster Costs** represent all costs required to operate a Kubernetes cluster. **Cluster Assets Costs** are the portion of these costs that are related to directly observable entities within a cluster; these include expenses from nodes, persistent volumes, attached disks, load balancers, and network ingress/egress costs. From a financial accounting perspective, these are equivalent to the Cost of Goods Sold when measuring product costs. **Cluster Overhead Costs** measure the overhead required to operate all of the Assets of a cluster, e.g. Cluster Management Fees. These are the equivalent of Selling, General, and Administrative Expenses (SG&A), or indirect costs when viewed from a financial accounting perspective.
 
 
 <table>
@@ -33,9 +33,7 @@ As Kubernetes adoption increases within an organization, these complexities beco
   </tr>
 </table>
 
-
-Cluster Asset Costs can be further segmented into **Resource Allocation Costs** and **Resource Usage Costs**. Resource Allocation Costs are expenses that accumulate based on the amount of time provisioned irrespective of usage (e.g. CPU hourly rate) whereas Resource Usage Costs accumulate on a per-unit basis (e.g. cost per byte egressed). Costs for an individual Asset are the summation of it’s Resource Allocation and Usage Costs, e.g. a Node’s cost is equal to CPU cost + GPU cost + RAM cost + Node Network Costs
-
+Cluster Asset Costs can be further segmented into **Resource Allocation Costs** and **Resource Usage Costs**. Resource Allocation Costs are expenses that accumulate based on the amount of time provisioned irrespective of usage (e.g. CPU hourly rate) whereas Resource Usage Costs accumulate on a per-unit basis (e.g. cost per byte egressed). Costs for an individual Asset are the summation of its Resource Allocation and Usage Costs, e.g. a Node’s cost is equal to CPU cost + GPU cost + RAM cost + Node Network Costs.
 
 <table>
   <tr>
@@ -65,19 +63,16 @@ Cluster Asset Costs can be further segmented into **Resource Allocation Costs**
   </tr>
 </table>
 
-
 The following chart shows these relationships:
 
 <img width="796" alt="image4" src="https://user-images.githubusercontent.com/453512/171577990-8f7c9a53-f5b1-4fbc-b2f6-75cd6ea67960.png"/>
 
-While billing models can differ by environment, below are common examples of segmentation by Allocation, Usage and Overhead Costs.
+While billing models can differ by environment, below are common examples of segmentation by Allocation, Usage, and Overhead Costs.
 
 <img width="292" alt="image1" src="https://user-images.githubusercontent.com/453512/171578190-d84dc3a7-1d20-4575-9bcc-2a5722de5eea.png"/>
 
-
 Once calculated, these Asset Costs can then be distributed to the tenants that consume them, where Workload Costs plus Idle Costs equals Asset Costs. **Workload costs** are expenses that can be directly attributed to a set of Kubernetes workloads, e.g. a container, pod, deployment, etc. **Cluster Idle Costs** are the portion of Resource Allocation Costs that are not allocated to any workload[^1].
 
-
 <table>
   <tr>
    <td><strong>Total Cluster Costs</strong>
@@ -104,22 +99,18 @@ The following chart shows these relationships:
 
 ## Cluster Asset Costs
 
-Cluster Assets are observable entities within a Kubernetes cluster that directly incur costs related to their resources. Asset Costs consist of Resource Allocation Costs and Resource Usage Costs. Every Asset conforming to this specification MUST include at least one cost component with Amount, Unit and Rate attributes as well as a TotalCost value.
+Cluster Assets are observable entities within a Kubernetes cluster that directly incur costs related to their resources. Asset Costs consist of Resource Allocation Costs and Resource Usage Costs. Every Asset conforming to this specification MUST include at least one cost component with Amount, Unit, and Rate attributes as well as a TotalCost value.
 
 Attributes for measured Resource Allocation Costs:
 
-
-
 * [float] Amount - the amount of resource reserved by the asset, e.g. 2 CPU cores
 * [float] Duration - time between the start and end of the allocation period measured in hours, e.g. 24 hours
 * [string] Unit - the amount’s unit of measurement, e.g. CPU cores
-* [float] HourlyRate - cost per one unit hour, e.g. $0.2 per CPU hourly rate
+* [float] HourlyRate - cost per one unit hour, e.g. $0.20 per CPU hourly rate
 * [float] Total Cost - defined as Amount * Duration * HourlyRate
 
 Attributes for measured Resource Usage Costs:
 
-
-
 * [float] Amount - the amount of resource used, e.g. 1GB of internet egress
 * [string] Unit - the amount’s unit of measurement, e.g. GB
 * [float] UnitRate - cost per unit, e.g $ per GB egressed
@@ -127,8 +118,6 @@ Attributes for measured Resource Usage Costs:
 
 Below are example inputs when measuring asset costs over a designated time window (e.g. 24 hours) with common billing models:
 
-
-
 * **Nodes**
     * CPU allocation costs
         * cores = avg_over_time(cpu) by (node) [cores]
@@ -163,7 +152,6 @@ Below are example inputs when measuring asset costs over a designated time windo
 
 Workloads are defined as entities to which Asset Costs are committed. Some resources solely have Usage Costs, but others have Allocation Costs independent of actual usage. Workload Costs should be understood as _max(request, usage)_ when Assets have Resource Allocation Costs, e.g. CPU or GPU. This formula effectively assigns costs that have been directly reserved or allocated by _kube-scheduler_. Workload Costs should be calculated at the lowest level possible, i.e. _container_ level[^2], and then they can be aggregated by any dimension.
 
-
 <table>
   <tr>
    <td>Resource Type
@@ -225,9 +213,7 @@ The following workload cost aggregations are supported in a complete implementat
 
 ## Shared Costs
 
-Shared Workload Costs, Cluster Idle Costs, and Overhead Costs are common examples of costs that organizations can optionally distribute amongst tenants. A common example would be system workload costs, e.g. kube-system pods, that benefit all tenants. Common methods for distributing these costs include the following:
-
-
+Shared Workload Costs, Cluster Idle Costs, and Overhead Costs are common examples of costs that organizations can optionally distribute amongst tenants. A common example would be system workload costs, e.g. _kube-system_ pods, that benefit all tenants. Common methods for distributing these costs include the following:
 
 1. Uniformly across other tenants
 2. Proportionate to a tenant's consumption of Cluster Asset costs
@@ -235,10 +221,9 @@ Shared Workload Costs, Cluster Idle Costs, and Overhead Costs are common example
 
 A full implementation of the spec should support various methods of distributing shared costs.
 
-
 ## Idle Costs
 
-Idle Costs can be calculated at both the Asset/Resource level as well as the Workload level. Asset Idle Costs represent the cost-weighted difference between Cluster Asset Costs and costs of resources being allocated or consumed. Idle Costs and then Idle Percentage can be calculated as follows:
+Idle Costs can be calculated at both the Asset/Resource level as well as the Workload level. Asset Idle Costs represent the cost-weighted difference between Cluster Asset Costs and the costs of resources being allocated or consumed. Idle Costs and then Idle Percentage can be calculated as follows:
 
 
 <table>
@@ -256,8 +241,6 @@ Idle Costs can be calculated at both the Asset/Resource level as well as the Wor
   </tr>
 </table>
 
-
-
 <table>
   <tr>
    <td><strong>Cluster </strong>
@@ -276,14 +259,12 @@ Idle Costs can be calculated at both the Asset/Resource level as well as the Wor
   </tr>
 </table>
 
-
-
 ##
 The following chart shows these relationships:
 ![image3](https://user-images.githubusercontent.com/453512/171579570-055bebe8-cc97-4129-9238-c4bcda8e123c.png)
 
 
-Asset Idle Cost can be calculated by individual assets, groups of assets, cluster(s), and by individual resources, e.g. CPU. Resources that are strictly billed on usage can be viewed to have 100% efficiency but should not be included when measuring idle percentage of a cluster.
+Asset Idle Cost can be calculated by individual assets, groups of assets, cluster(s), and by individual resources, e.g. CPU. Resources that are strictly billed on usage can be viewed to have 100% efficiency but should not be included when measuring the idle percentage of a cluster.
 
 Workload Idle Costs is a cost-weighted measurement of [requested](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container) resources that are unused. Workload Idle Costs can be calculated on any grouping of Kubernetes workloads, e.g. containers, pods, labels, annotations, namespaces, etc.
 
@@ -299,19 +280,19 @@ The state of a pod will affect the ability to assign costs and whether a resourc
 ## Glossary
 
 
-**Cluster Assets** – Observable entities within a Kubernetes cluster that directly incur costs related to their resources. Examples include nodes, persistent volumes, attached disks, load balancers.
+**Cluster Assets** – Observable entities within a Kubernetes cluster that directly incur costs related to their resources. Examples include nodes, persistent volumes, attached disks, and load balancers.
 
 
 **Container** - An instance of a container image. You may have multiple copies of the same image running at the same time. [More info](https://kubernetes.io/docs/concepts/containers/)
 
 
-**Image** - A template of a container which contains software (usually microservices) that needs to be run. [More info](https://kubernetes.io/docs/concepts/containers/images/)
+**Image** - A template of a container that contains software (usually microservices) that needs to be run. [More info](https://kubernetes.io/docs/concepts/containers/images/)
 
 
 **Server / Instance / Node / Node Pool** - A machine (possibly cloud or on-prem, physical or virtual) in this context used by Kubernetes [More info](https://kubernetes.io/docs/concepts/architecture/nodes/)
 
 
-**Pod** - A Kubernetes specific concept that consists of a group of containers. A pod is treated as a single block of resources that may be scheduled or scaled on a cluster. [More info](https://kubernetes.io/docs/concepts/workloads/pods/)
+**Pod** - A Kubernetes-specific concept that consists of a group of containers. A pod is treated as a single block of resources that may be scheduled or scaled on a cluster. [More info](https://kubernetes.io/docs/concepts/workloads/pods/)
 
 
 **Container Orchestration** - Manages the cluster of server instances and maintains the lifecycle of containers and pods. Scheduling is a function of the container orchestrator which schedules pods/containers to run on a server instance.
@@ -320,30 +301,30 @@ The state of a pod will affect the ability to assign costs and whether a resourc
 **Cluster** - A group of server instances
 
 
-**Namespace** - A Kubernetes concept which creates a ‘virtual’ cluster where pods/containers may be deployed and observed discreetly from other namespaces. [More info](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/)
+**Namespace** - A Kubernetes concept that creates a ‘virtual’ cluster where pods/containers may be deployed and observed discreetly from other namespaces. [More info](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/)
 
 
-**Pod Labels** - Key / Value pairs which may be used to identify objects that are meaningful to the user. There is no semantic meaning to the core of the system. Labels are typically used where a grouping of multiple namespaces need to be associated with a workload. [More info](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)
+**Pod Labels** - Key / Value pairs that may be used to identify objects that are meaningful to the user. There is no semantic meaning to the core of the system. Labels are typically used where a grouping of multiple namespaces need to be associated with a workload. [More info](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)
 
 
 ## Appendix A
 
-Various cloud providers supply an hourly resource cost directly in their user billing model.The OpenCost model recommends utilizing the fully Amortized Net Cost for each resource as an input when this is the case. When explicit RAM, CPU or GPU prices are not provided by a cloud provider, the OpenCost model needs to derive these values. The recommendation is to use a scalable ratio of CPU, GPU, RAM and other price inputs. These default values should be based on the marginal resource rates of the provider by family.
+Various cloud providers supply an hourly resource cost directly in their user billing model. The OpenCost model recommends utilizing the fully Amortized Net Cost for each resource as an input when this is the case. When explicit RAM, CPU or GPU prices are not provided by a cloud provider, the OpenCost model needs to derive these values. The recommendation is to use a scalable ratio of CPU, GPU, RAM, and other price inputs. These default values should be based on the marginal resource rates of the provider by family.
 
 One approach for calculating is to ensure the sum of each component is equal to the total price of the Asset (e.g. node) based on billing rates from your provider. When the sum of resources (e.g. RAM/CPU/GPU) cost is greater (or less) than the price of the node, then the ratio between the input prices is held constant but the total value is adjusted.
 
-As an example, you have provisioned a node with 1 GPU, 1 CPU and 1 GB of RAM that costs $35/mo. If your base GPU price is $30, base CPU price is $30, and RAM GB price is $10, based on the average marginal costs across instances in this family class, then these inputs will be normalized to $15 for GPU, $15 for CPU and $5 for RAM so that the sum equals the cost of the node. Note that the price of a GPU, as well as the price of a CPU remain 3x the price of a Gb of RAM.
+As an example, you have provisioned a node with 1 GPU, 1 CPU, and 1 GB of RAM that costs $35/mo. If your base GPU price is $30, base CPU price is $30, and RAM GB price is $10, based on the average marginal costs across instances in this family class, then these inputs will be normalized to $15 for GPU, $15 for CPU and $5 for RAM so that the sum equals the cost of the node. Note that the price of a GPU, as well as the price of a CPU remain 3x the price of a GB of RAM.
 
 
 ## Appendix B
 
-Sampling Kubernetes resources is recommended with the following metrics / datasources:
+Sampling Kubernetes resources is recommended with the following metrics/data sources:
 
 
 
-* container_cpu_usage_seconds_total – sample from cAdvisor
+* container_cpu/usage_seconds_total – sample from cAdvisor
 * container_memory_working_set_bytes –  sampled from cAdvisor
-* gpu_usage – sampled via chipset specific metrics
+* gpu_usage – sampled via chipset-specific metrics
 * cpu_requested – sampled from kube API
 * ram_requested – sampled from kube API
 * gpu_requested – sampled from kube API
@@ -357,7 +338,7 @@ Working examples of OpenCost data to come!
 ## Notes
 
 [^1]:
-     Resource **usage** costs cannot be part of idle cost because they are always used, the corresponding resource never "sits idle."
+     Resource **usage** costs cannot be part of the idle cost because they are always used, the corresponding resource never "sits idle."
 
 [^2]:
      This is because containers are the smallest identifiable unit of "thing that uses resources." For example, the lowest level of reliable CPU usage information is usually a container.