Merge branch 'develop' into custom-s3

.github/configs/stale.yaml

@@ -0,0 +1,5 @@
+## https://github.com/marketplace/actions/close-stale-issues#recommended-permissions
+# Give stalebot permission to update issues and pull requests
+permissions:
+  issues: write
+  pull-requests: write

.github/workflows/stale.yml

@@ -0,0 +1,19 @@
+name: 'Close stale issues and PRs'
+on:
+  schedule:
+    - cron: '30 1 * * *'
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v8
+        with:
+          stale-issue-message: 'This issue has been marked as stale because it has been open for 180 days with no activity. Please remove the stale label or comment or this issue will be closed in 5 days.'
+          close-issue-message: 'This issue was closed because it has been inactive for 185 days with no activity.'
+          stale-pr-message: 'This pull request has been marked as stale because it has been open for 60 days with no activity. Please remove the stale label or comment or this pull request will be closed in 5 days.'
+          close-pr-message: 'This pull request was closed because it has been inactive for 65 days with no activity.'
+          days-before-issue-stale: 180
+          days-before-issue-close: 5
+          days-before-pr-stale: 60
+          days-before-pr-close: 5

README.md

@@ -21,13 +21,13 @@ To see the full functionality of OpenCost you can view [OpenCost features](https
 
 You can deploy OpenCost on any Kubernetes 1.8+ cluster in a matter of minutes, if not seconds!
 
-Visit the full documentation for [recommended install options](https://www.opencost.io/docs/install).
+Visit the full documentation for [recommended install options](https://www.opencost.io/docs/installation/install).
 
 ## Usage
 
-- [Cost APIs](https://www.opencost.io/docs/api)
-- [CLI / kubectl cost](https://www.opencost.io/docs/kubectl-cost)
-- [Prometheus Metrics](https://www.opencost.io/docs/prometheus)
+- [Cost APIs](https://www.opencost.io/docs/integrations/api)
+- [CLI / kubectl cost](https://www.opencost.io/docs/integrations/kubectl-cost)
+- [Prometheus Metrics](https://www.opencost.io/docs/integrations/prometheus)
 - Reference [User Interface](https://github.com/opencost/opencost/tree/develop/ui)
 
 ## Contributing

configs/azure.json

@@ -2,17 +2,18 @@
     "provider": "Azure",
     "description": "Azure estimates based on April 2019 advertised prices",
     "CPU": "0.03900",
-    "spotCPU": "0.007764", 
-    "RAM": "0.001917", 
+    "spotCPU": "0.007764",
+    "RAM": "0.001917",
+    "GPU": "0.0428925",
     "spotRAM": "0.000382",
-    "storage": "0.00005479452" ,
+    "storage": "0.00005479452",
     "zoneNetworkEgress": "0.01",
     "regionNetworkEgress": "0.01",
     "internetNetworkEgress": "0.0725",
     "spotLabel": "kubernetes.azure.com/scalesetpriority",
     "spotLabelValue": "spot",
     "azureSubscriptionID": "",
-    "azureClientID": "" ,
-    "azureClientSecret": "" ,
+    "azureClientID": "",
+    "azureClientSecret": "",
     "azureTenantID": ""
 }

docs/swagger.json

@@ -69,7 +69,7 @@
           {
           "name": "aggregate",
           "in": "query",
-          "description": "Field by which to aggregate the results. Accepts: `cluster`, `node`, `namespace`, `controllerKind`, `controller`, `service`, `pod`, `container`, `label:<name>`, and `annotation:<name>`. Also accepts comma-separated lists for multi-aggregation, like `namespace,label:app`.",
+          "description": "Field by which to aggregate the results. Accepts: `all`, `cluster`, `node`, `namespace`, `controllerKind`, `controller`, `service`, `pod`, `container`, `label:<name>`, and `annotation:<name>`. Also accepts comma-separated lists for multi-aggregation, like `namespace,label:app`. Defaults to `cluster,node,namespace,pod,container`.",
           "required": false,
           "style": "form",
           "explode": true,
@@ -108,6 +108,10 @@
             "container": {
               "summary": "Aggregates by the containers present in the cluster",
               "value": "container"
+            },
+            "all": {
+              "summary": "Aggregates into a single allocation",
+              "value": "all"
             }
           }
         },

pkg/cloud/alibaba/boaquerier.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/opencost/opencost/pkg/cloud"
 	cloudconfig "github.com/opencost/opencost/pkg/cloud/config"
 
 	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
@@ -20,6 +21,11 @@ const (
 
 type BoaQuerier struct {
 	BOAConfiguration
+	ConnectionStatus cloud.ConnectionStatus
+}
+
+func (bq *BoaQuerier) GetStatus() cloud.ConnectionStatus {
+	return bq.ConnectionStatus
 }
 
 func (bq *BoaQuerier) Equals(config cloudconfig.Config) bool {

pkg/cloud/alibaba/provider.go

@@ -514,23 +514,25 @@ func (alibaba *Alibaba) AllNodePricing() (interface{}, error) {
 }
 
 // NodePricing gives pricing information of a specific node given by the key
-func (alibaba *Alibaba) NodePricing(key models.Key) (*models.Node, error) {
+func (alibaba *Alibaba) NodePricing(key models.Key) (*models.Node, models.PricingMetadata, error) {
 	alibaba.DownloadPricingDataLock.RLock()
 	defer alibaba.DownloadPricingDataLock.RUnlock()
 
 	// Get node features for the key
 	keyFeature := key.Features()
 
+	meta := models.PricingMetadata{}
+
 	pricing, ok := alibaba.Pricing[keyFeature]
 	if !ok {
 		log.Errorf("Node pricing information not found for node with feature: %s", keyFeature)
-		return nil, fmt.Errorf("Node pricing information not found for node with feature: %s letting it use default values", keyFeature)
+		return nil, meta, fmt.Errorf("Node pricing information not found for node with feature: %s letting it use default values", keyFeature)
 	}
 
 	log.Debugf("returning the node price for the node with feature: %s", keyFeature)
 	returnNode := pricing.Node
 
-	return returnNode, nil
+	return returnNode, meta, nil
 }
 
 // PVPricing gives a pricing information of a specific PV given by PVkey

pkg/cloud/aws/athenaquerier.go

@@ -24,6 +24,10 @@ type AthenaQuerier struct {
 	ConnectionStatus cloud.ConnectionStatus
 }
 
+func (aq *AthenaQuerier) GetStatus() cloud.ConnectionStatus {
+	return aq.ConnectionStatus
+}
+
 func (aq *AthenaQuerier) Equals(config cloudconfig.Config) bool {
 	thatConfig, ok := config.(*AthenaQuerier)
 	if !ok {
@@ -106,6 +110,9 @@ func (aq *AthenaQuerier) queryAthenaPaginated(ctx context.Context, query string,
 
 	// Create Athena Client
 	cli, err := aq.GetAthenaClient()
+	if err != nil {
+		return fmt.Errorf("QueryAthenaPaginated: GetAthenaClient error: %s", err.Error())
+	}
 
 	// Query Athena
 	startQueryExecutionOutput, err := cli.StartQueryExecution(ctx, startQueryExecutionInput)

pkg/cloud/aws/provider.go

@@ -5,6 +5,7 @@ import (
 	"compress/gzip"
 	"context"
 	"encoding/csv"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -15,6 +16,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/aws/smithy-go"
 	"github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/cloud/utils"
 	"github.com/opencost/opencost/pkg/kubecost"
@@ -1069,9 +1071,45 @@ func (aws *AWS) populatePricing(resp *http.Response, inputkeys map[string]bool)
 						aws.Pricing[spotKey].OnDemand = offerTerm
 						var cost string
 						if _, isMatch := OnDemandRateCodes[offerTerm.OfferTermCode]; isMatch {
-							cost = offerTerm.PriceDimensions[strings.Join([]string{sku.(string), offerTerm.OfferTermCode, HourlyRateCode}, ".")].PricePerUnit.USD
+							priceDimensionKey := strings.Join([]string{sku.(string), offerTerm.OfferTermCode, HourlyRateCode}, ".")
+							dimension, ok := offerTerm.PriceDimensions[priceDimensionKey]
+							if ok {
+								cost = dimension.PricePerUnit.USD
+							} else {
+								// this is an edge case seen in AWS CN pricing files, including here just in case
+								// if there is only one dimension, use it, even if the key is incorrect, otherwise assume defaults
+								if len(offerTerm.PriceDimensions) == 1 {
+									for key, backupDimension := range offerTerm.PriceDimensions {
+										cost = backupDimension.PricePerUnit.USD
+										log.DedupedWarningf(5, "using:%s for a price dimension instead of missing dimension: %s", offerTerm.PriceDimensions[key], priceDimensionKey)
+										break
+									}
+								} else if len(offerTerm.PriceDimensions) == 0 {
+									log.DedupedWarningf(5, "populatePricing: no pricing dimension available for: %s.", priceDimensionKey)
+								} else {
+									log.DedupedWarningf(5, "populatePricing: no assumable pricing dimension available for: %s.", priceDimensionKey)
+								}
+							}
 						} else if _, isMatch := OnDemandRateCodesCn[offerTerm.OfferTermCode]; isMatch {
-							cost = offerTerm.PriceDimensions[strings.Join([]string{sku.(string), offerTerm.OfferTermCode, HourlyRateCodeCn}, ".")].PricePerUnit.CNY
+							priceDimensionKey := strings.Join([]string{sku.(string), offerTerm.OfferTermCode, HourlyRateCodeCn}, ".")
+							dimension, ok := offerTerm.PriceDimensions[priceDimensionKey]
+							if ok {
+								cost = dimension.PricePerUnit.CNY
+							} else {
+								// fall through logic for handling inconsistencies in AWS CN pricing files
+								// if there is only one dimension, use it, even if the key is incorrect, otherwise assume defaults
+								if len(offerTerm.PriceDimensions) == 1 {
+									for key, backupDimension := range offerTerm.PriceDimensions {
+										cost = backupDimension.PricePerUnit.CNY
+										log.DedupedWarningf(5, "using:%s for a price dimension instead of missing dimension: %s", offerTerm.PriceDimensions[key], priceDimensionKey)
+										break
+									}
+								} else if len(offerTerm.PriceDimensions) == 0 {
+									log.DedupedWarningf(5, "populatePricing: no pricing dimension available for: %s.", priceDimensionKey)
+								} else {
+									log.DedupedWarningf(5, "populatePricing: no assumable pricing dimension available for: %s.", priceDimensionKey)
+								}
+							}
 						}
 						if strings.Contains(key, "EBS:VolumeP-IOPS.piops") {
 							// If the specific UsageType is the per IO cost used on io1 volumes
@@ -1204,9 +1242,11 @@ func (aws *AWS) savingsPlanPricing(instanceID string) (*SavingsPlanData, bool) {
 	return data, ok
 }
 
-func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k models.Key) (*models.Node, error) {
+func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k models.Key) (*models.Node, models.PricingMetadata, error) {
 	key := k.Features()
 
+	meta := models.PricingMetadata{}
+
 	if spotInfo, ok := aws.spotPricing(k.ID()); ok {
 		var spotcost string
 		log.DedupedInfof(5, "Looking up spot data from feed for node %s", k.ID())
@@ -1226,7 +1266,7 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k models.Ke
 			BaseRAMPrice: aws.BaseRAMPrice,
 			BaseGPUPrice: aws.BaseGPUPrice,
 			UsageType:    PreemptibleType,
-		}, nil
+		}, meta, nil
 	} else if aws.isPreemptible(key) { // Preemptible but we don't have any data in the pricing report.
 		log.DedupedWarningf(5, "Node %s marked preemptible but we have no data in spot feed", k.ID())
 		return &models.Node{
@@ -1239,7 +1279,7 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k models.Ke
 			BaseRAMPrice: aws.BaseRAMPrice,
 			BaseGPUPrice: aws.BaseGPUPrice,
 			UsageType:    PreemptibleType,
-		}, nil
+		}, meta, nil
 	} else if sp, ok := aws.savingsPlanPricing(k.ID()); ok {
 		strCost := fmt.Sprintf("%f", sp.EffectiveCost)
 		return &models.Node{
@@ -1252,7 +1292,7 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k models.Ke
 			BaseRAMPrice: aws.BaseRAMPrice,
 			BaseGPUPrice: aws.BaseGPUPrice,
 			UsageType:    usageType,
-		}, nil
+		}, meta, nil
 
 	} else if ri, ok := aws.reservedInstancePricing(k.ID()); ok {
 		strCost := fmt.Sprintf("%f", ri.EffectiveCost)
@@ -1266,7 +1306,7 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k models.Ke
 			BaseRAMPrice: aws.BaseRAMPrice,
 			BaseGPUPrice: aws.BaseGPUPrice,
 			UsageType:    usageType,
-		}, nil
+		}, meta, nil
 
 	}
 	var cost string
@@ -1279,7 +1319,7 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k models.Ke
 		if ok {
 			cost = c.PricePerUnit.CNY
 		} else {
-			return nil, fmt.Errorf("Could not fetch data for \"%s\"", k.ID())
+			return nil, meta, fmt.Errorf("Could not fetch data for \"%s\"", k.ID())
 		}
 	}
 
@@ -1293,11 +1333,11 @@ func (aws *AWS) createNode(terms *AWSProductTerms, usageType string, k models.Ke
 		BaseRAMPrice: aws.BaseRAMPrice,
 		BaseGPUPrice: aws.BaseGPUPrice,
 		UsageType:    usageType,
-	}, nil
+	}, meta, nil
 }
 
 // NodePricing takes in a key from GetKey and returns a Node object for use in building the cost model.
-func (aws *AWS) NodePricing(k models.Key) (*models.Node, error) {
+func (aws *AWS) NodePricing(k models.Key) (*models.Node, models.PricingMetadata, error) {
 	aws.DownloadPricingDataLock.RLock()
 	defer aws.DownloadPricingDataLock.RUnlock()
 
@@ -1307,6 +1347,8 @@ func (aws *AWS) NodePricing(k models.Key) (*models.Node, error) {
 		usageType = PreemptibleType
 	}
 
+	meta := models.PricingMetadata{}
+
 	terms, ok := aws.Pricing[key]
 	if ok {
 		return aws.createNode(terms, usageType, k)
@@ -1322,7 +1364,7 @@ func (aws *AWS) NodePricing(k models.Key) (*models.Node, error) {
 				BaseGPUPrice:     aws.BaseGPUPrice,
 				UsageType:        usageType,
 				UsesBaseCPUPrice: true,
-			}, err
+			}, meta, err
 		}
 		terms, termsOk := aws.Pricing[key]
 		if !termsOk {
@@ -1333,11 +1375,11 @@ func (aws *AWS) NodePricing(k models.Key) (*models.Node, error) {
 				BaseGPUPrice:     aws.BaseGPUPrice,
 				UsageType:        usageType,
 				UsesBaseCPUPrice: true,
-			}, fmt.Errorf("Unable to find any Pricing data for \"%s\"", key)
+			}, meta, fmt.Errorf("Unable to find any Pricing data for \"%s\"", key)
 		}
 		return aws.createNode(terms, usageType, k)
 	} else { // Fall back to base pricing if we can't find the key. Base pricing is handled at the costmodel level.
-		return nil, fmt.Errorf("Invalid Pricing Key \"%s\"", key)
+		return nil, meta, fmt.Errorf("Invalid Pricing Key \"%s\"", key)
 
 	}
 }
@@ -1519,8 +1561,20 @@ func (aws *AWS) getAllAddresses() ([]*ec2Types.Address, error) {
 			// Query for first page of volume results
 			resp, err := aws.getAddressesForRegion(context.TODO(), region)
 			if err != nil {
-				errorCh <- err
-				return
+				var awsErr smithy.APIError
+				if errors.As(err, &awsErr) {
+					switch awsErr.ErrorCode() {
+					case "AuthFailure", "InvalidClientTokenId", "UnauthorizedOperation":
+						log.DedupedInfof(5, "Unable to get addresses for region %s due to AWS permissions, error message: %s", r, awsErr.ErrorMessage())
+						return
+					default:
+						errorCh <- err
+						return
+					}
+				} else {
+					errorCh <- err
+					return
+				}
 			}
 			addressCh <- resp
 		}(r)
@@ -1621,8 +1675,20 @@ func (aws *AWS) getAllDisks() ([]*ec2Types.Volume, error) {
 			// Query for first page of volume results
 			resp, err := aws.getDisksForRegion(context.TODO(), region, 1000, nil)
 			if err != nil {
-				errorCh <- err
-				return
+				var awsErr smithy.APIError
+				if errors.As(err, &awsErr) {
+					switch awsErr.ErrorCode() {
+					case "AuthFailure", "InvalidClientTokenId", "UnauthorizedOperation":
+						log.DedupedInfof(5, "Unable to get disks for region %s due to AWS permissions, error message: %s", r, awsErr.ErrorMessage())
+						return
+					default:
+						errorCh <- err
+						return
+					}
+				} else {
+					errorCh <- err
+					return
+				}
 			}
 			volumeCh <- resp
 
@@ -1704,14 +1770,17 @@ func (aws *AWS) isDiskOrphaned(vol *ec2Types.Volume) bool {
 }
 
 func (aws *AWS) GetOrphanedResources() ([]models.OrphanedResource, error) {
-	volumes, err := aws.getAllDisks()
-	if err != nil {
-		return nil, err
-	}
+	volumes, volumesErr := aws.getAllDisks()
+	addresses, addressesErr := aws.getAllAddresses()
 
-	addresses, err := aws.getAllAddresses()
-	if err != nil {
-		return nil, err
+	// If we have any orphaned resources - prioritize returning them over returning errors
+	if len(addresses) == 0 && len(volumes) == 0 {
+		if volumesErr != nil {
+			return nil, volumesErr
+		}
+		if addressesErr != nil {
+			return nil, addressesErr
+		}
 	}
 
 	var orphanedResources []models.OrphanedResource

pkg/cloud/aws/s3connection.go

@@ -5,11 +5,17 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/opencost/opencost/pkg/cloud"
 	"github.com/opencost/opencost/pkg/cloud/config"
 )
 
 type S3Connection struct {
 	S3Configuration
+	ConnectionStatus cloud.ConnectionStatus
+}
+
+func (s3c *S3Connection) GetStatus() cloud.ConnectionStatus {
+	return s3c.ConnectionStatus
 }
 
 func (s3c *S3Connection) Equals(config config.Config) bool {

pkg/cloud/aws/s3selectintegration.go

@@ -0,0 +1,269 @@
+package aws
+
+import (
+	"encoding/csv"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/opencost/opencost/pkg/kubecost"
+	"github.com/opencost/opencost/pkg/log"
+	"github.com/opencost/opencost/pkg/util/timeutil"
+)
+
+const s3SelectDateLayout = "2006-01-02T15:04:05Z"
+
+// S3Object is aliased as "s" in queries
+const s3SelectAccountID = `s."bill/PayerAccountId"`
+
+const s3SelectItemType = `s."lineItem/LineItemType"`
+const s3SelectStartDate = `s."lineItem/UsageStartDate"`
+const s3SelectProductCode = `s."lineItem/ProductCode"`
+const s3SelectResourceID = `s."lineItem/ResourceId"`
+
+const s3SelectIsNode = `SUBSTRING(s."lineItem/ResourceId",1,2) = 'i-'`
+const s3SelectIsVol = `SUBSTRING(s."lineItem/ResourceId", 1, 4) = 'vol-'`
+const s3SelectIsNetwork = `s."lineItem/UsageType" LIKE '%Bytes'`
+
+const s3SelectListCost = `s."lineItem/UnblendedCost"`
+const s3SelectNetCost = `s."lineItem/NetUnblendedCost"`
+
+// These two may be used for Amortized<Net>Cost
+const s3SelectRICost = `s."reservation/EffectiveCost"`
+const s3SelectSPCost = `s."savingsPlan/SavingsPlanEffectiveCost"`
+
+type S3SelectIntegration struct {
+	S3SelectQuerier
+}
+
+func (s3si *S3SelectIntegration) GetCloudCost(
+	start,
+	end time.Time,
+) (*kubecost.CloudCostSetRange, error) {
+	log.Infof(
+		"S3SelectIntegration[%s]: GetCloudCost: %s",
+		s3si.Key(),
+		kubecost.NewWindow(&start, &end).String(),
+	)
+
+	// Set midnight yesterday as last point in time reconciliation data
+	// can be pulled from to ensure complete days of data
+	midnightYesterday := time.Now().In(
+		time.UTC,
+	).Truncate(time.Hour*24).AddDate(0, 0, -1)
+	if end.After(midnightYesterday) {
+		end = midnightYesterday
+	}
+
+	// ccsr to populate with cloudcosts.
+	ccsr, err := kubecost.NewCloudCostSetRange(
+		start,
+		end,
+		timeutil.Day,
+		s3si.Key(),
+	)
+	if err != nil {
+		return nil, err
+	}
+	// acquire S3 client
+	client, err := s3si.GetS3Client()
+	if err != nil {
+		return nil, err
+	}
+	// Acquire query keys
+	queryKeys, err := s3si.GetQueryKeys(start, end, client)
+	if err != nil {
+		return nil, err
+	}
+	// Acquire headers
+	headers, err := s3si.GetHeaders(queryKeys, client)
+	if err != nil {
+		return nil, err
+	}
+	// Exactly what it says on the tin. Though is there a set equivalent
+	// in Go? This seems like a good use case for that.
+	allColumns := map[string]bool{}
+	for _, header := range headers {
+		allColumns[header] = true
+	}
+
+	formattedStart := start.Format("2006-01-02")
+	formattedEnd := end.Format("2006-01-02")
+	selectColumns := []string{
+		s3SelectStartDate,
+		s3SelectAccountID,
+		s3SelectResourceID,
+		s3SelectItemType,
+		s3SelectProductCode,
+		s3SelectIsNode,
+		s3SelectIsVol,
+		s3SelectIsNetwork,
+		s3SelectListCost,
+	}
+	// OC equivalent to KCM env flags relevant at all?
+	// Check for Reservation columns in CUR and query if available
+	checkReservations := allColumns[s3SelectRICost]
+	if checkReservations {
+		selectColumns = append(selectColumns, s3SelectRICost)
+	}
+
+	// Check for Savings Plan Columns in CUR and query if available
+	checkSavingsPlan := allColumns[s3SelectSPCost]
+	if checkSavingsPlan {
+		selectColumns = append(selectColumns, s3SelectSPCost)
+	}
+
+	// Build map of query columns to use for parsing query
+	columnIndexes := map[string]int{}
+	for i, column := range selectColumns {
+		columnIndexes[column] = i
+	}
+	// Build query
+	selectStr := strings.Join(selectColumns, ", ")
+	queryStr := `SELECT %s FROM s3object s
+	WHERE (CAST(s."lineItem/UsageStartDate" AS TIMESTAMP) BETWEEN CAST('%s' AS TIMESTAMP) AND CAST('%s' AS TIMESTAMP))
+	AND s."lineItem/ResourceId" <> ''
+	AND (
+		(
+			s."lineItem/ProductCode" = 'AmazonEC2' AND (
+				SUBSTRING(s."lineItem/ResourceId",1,2) = 'i-'
+				OR SUBSTRING(s."lineItem/ResourceId",1,4) = 'vol-'
+			)
+		)
+		OR s."lineItem/ProductCode" = 'AWSELB'
+       OR s."lineItem/ProductCode" = 'AmazonFSx'
+	)`
+	query := fmt.Sprintf(queryStr, selectStr, formattedStart, formattedEnd)
+
+	processResults := func(reader *csv.Reader) error {
+		_, err2 := reader.Read()
+		if err2 == io.EOF {
+			return nil
+		}
+		for {
+			row, err3 := reader.Read()
+			if err3 == io.EOF {
+				return nil
+			}
+
+			startStr := GetCSVRowValue(row, columnIndexes, s3SelectStartDate)
+			itemAccountID := GetCSVRowValue(row, columnIndexes, s3SelectAccountID)
+			itemProviderID := GetCSVRowValue(row, columnIndexes, s3SelectResourceID)
+			lineItemType := GetCSVRowValue(row, columnIndexes, s3SelectItemType)
+			itemProductCode := GetCSVRowValue(row, columnIndexes, s3SelectProductCode)
+			isNode, _ := strconv.ParseBool(GetCSVRowValue(row, columnIndexes, s3SelectIsNode))
+			isVol, _ := strconv.ParseBool(GetCSVRowValue(row, columnIndexes, s3SelectIsVol))
+			isNetwork, _ := strconv.ParseBool(GetCSVRowValue(row, columnIndexes, s3SelectIsNetwork))
+			var (
+				amortizedCost float64
+				listCost      float64
+				netCost       float64
+			)
+			// Get list and net costs
+			listCost, err = GetCSVRowValueFloat(row, columnIndexes, s3SelectListCost)
+			if err != nil {
+				return err
+			}
+			netCost, err = GetCSVRowValueFloat(row, columnIndexes, s3SelectNetCost)
+			if err != nil {
+				return err
+			}
+
+			// If there is a reservation_reservation_a_r_n on the line item use the awsRIPricingSUMColumn as cost
+			if checkReservations && lineItemType == "DiscountedUsage" {
+				amortizedCost, err = GetCSVRowValueFloat(row, columnIndexes, s3SelectRICost)
+				if err != nil {
+					log.Errorf(err.Error())
+					continue
+				}
+				// If there is a lineItemType of SavingsPlanCoveredUsage use the awsSPPricingSUMColumn
+			} else if checkSavingsPlan && lineItemType == "SavingsPlanCoveredUsage" {
+				amortizedCost, err = GetCSVRowValueFloat(row, columnIndexes, s3SelectSPCost)
+				if err != nil {
+					log.Errorf(err.Error())
+					continue
+				}
+			} else {
+				// Default to listCost
+				amortizedCost = listCost
+			}
+			category := SelectAWSCategory(isNode, isVol, isNetwork, itemProductCode, "")
+			// Retrieve final stanza of product code for ProviderID
+			if itemProductCode == "AWSELB" || itemProductCode == "AmazonFSx" {
+				itemProviderID = ParseARN(itemProviderID)
+			}
+
+			properties := kubecost.CloudCostProperties{}
+			properties.Provider = kubecost.AWSProvider
+			properties.AccountID = itemAccountID
+			properties.Category = category
+			properties.Service = itemProductCode
+			properties.ProviderID = itemProviderID
+
+			itemStart, err := time.Parse(s3SelectDateLayout, startStr)
+			if err != nil {
+				log.Infof(
+					"Unable to parse '%s': '%s'",
+					s3SelectStartDate,
+					err.Error(),
+				)
+				itemStart = time.Now()
+			}
+			itemStart = itemStart.Truncate(time.Hour * 24)
+			itemEnd := itemStart.AddDate(0, 0, 1)
+
+			cc := &kubecost.CloudCost{
+				Properties: &properties,
+				Window:     kubecost.NewWindow(&itemStart, &itemEnd),
+				ListCost: kubecost.CostMetric{
+					Cost: listCost,
+				},
+				NetCost: kubecost.CostMetric{
+					Cost: netCost,
+				},
+				AmortizedNetCost: kubecost.CostMetric{
+					Cost: amortizedCost,
+				},
+				AmortizedCost: kubecost.CostMetric{
+					Cost: amortizedCost,
+				},
+				InvoicedCost: kubecost.CostMetric{
+					Cost: netCost,
+				},
+			}
+			ccsr.LoadCloudCost(cc)
+		}
+	}
+	err = s3si.Query(query, queryKeys, client, processResults)
+	if err != nil {
+		return nil, err
+	}
+
+	return ccsr, nil
+}
+
+func (s3si *S3SelectIntegration) GetHeaders(queryKeys []string, client *s3.Client) ([]string, error) {
+	// Query to grab only header line from file
+	query := "SELECT * FROM S3OBJECT LIMIT 1"
+	var record []string
+
+	proccessheaders := func(reader *csv.Reader) error {
+		var err error
+		record, err = reader.Read()
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
+	// Use only the first query key with assumption that files share schema
+	err := s3si.Query(query, []string{queryKeys[0]}, client, proccessheaders)
+	if err != nil {
+		return nil, err
+	}
+
+	return record, nil
+}

pkg/cloud/aws/s3selectintegration_test.go

@@ -0,0 +1,69 @@
+package aws
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/opencost/opencost/pkg/util/json"
+	"github.com/opencost/opencost/pkg/util/timeutil"
+)
+
+func TestS3Integration_GetCloudCost(t *testing.T) {
+	s3ConfigPath := os.Getenv("S3_CONFIGURATION")
+	if s3ConfigPath == "" {
+		t.Skip("skipping integration test, set environment variable S3_CONFIGURATION")
+	}
+	s3ConfigBin, err := os.ReadFile(s3ConfigPath)
+	if err != nil {
+		t.Fatalf("failed to read config file: %s", err.Error())
+	}
+	var s3Config S3Configuration
+	err = json.Unmarshal(s3ConfigBin, &s3Config)
+	if err != nil {
+		t.Fatalf("failed to unmarshal config from JSON: %s", err.Error())
+	}
+	testCases := map[string]struct {
+		integration *S3SelectIntegration
+		start       time.Time
+		end         time.Time
+		expected    bool
+	}{
+		// No CUR data is expected within 2 days of now
+		"too_recent_window": {
+			integration: &S3SelectIntegration{
+				S3SelectQuerier: S3SelectQuerier{
+					S3Connection: S3Connection{
+						S3Configuration: s3Config,
+					},
+				},
+			},
+			end:      time.Now(),
+			start:    time.Now().Add(-timeutil.Day),
+			expected: true,
+		},
+		// CUR data should be available
+		"last week window": {
+			integration: &S3SelectIntegration{
+				S3SelectQuerier: S3SelectQuerier{
+					S3Connection: S3Connection{
+						S3Configuration: s3Config,
+					},
+				},
+			},
+			end:      time.Now().Add(-7 * timeutil.Day),
+			start:    time.Now().Add(-8 * timeutil.Day),
+			expected: false,
+		},
+	}
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual, err := testCase.integration.GetCloudCost(testCase.start, testCase.end)
+			if err != nil {
+				t.Errorf("Other error during testing %s", err)
+			} else if actual.IsEmpty() != testCase.expected {
+				t.Errorf("Incorrect result, actual emptiness: %t, expected: %t", actual.IsEmpty(), testCase.expected)
+			}
+		})
+	}
+}

pkg/cloud/aws/s3selectquerier.go

@@ -12,12 +12,14 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	s3Types "github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/opencost/opencost/pkg/cloud"
 	"github.com/opencost/opencost/pkg/cloud/config"
 	"github.com/opencost/opencost/pkg/util/stringutil"
 )
 
 type S3SelectQuerier struct {
 	S3Connection
+	connectionStatus cloud.ConnectionStatus
 }
 
 func (s3sq *S3SelectQuerier) Equals(config config.Config) bool {

pkg/cloud/azure/billingexportparser.go

@@ -44,6 +44,9 @@ func (brv *BillingRowValues) IsCompute(category string) bool {
 	if category == kubecost.NetworkCategory && brv.MeterCategory == "Virtual Network" {
 		return true
 	}
+	if category == kubecost.NetworkCategory && brv.MeterCategory == "Bandwidth" {
+		return true
+	}
 	return false
 }
 
@@ -265,7 +268,7 @@ func AzureSetProviderID(abv *BillingRowValues) string {
 		return fmt.Sprintf("%v", value2)
 	}
 
-	if category == kubecost.StorageCategory {
+	if category == kubecost.StorageCategory || (category == kubecost.NetworkCategory && abv.MeterCategory == "Bandwidth") {
 		if value2, ok2 := abv.Tags["creationSource"]; ok2 {
 			creationSource := fmt.Sprintf("%v", value2)
 			return strings.TrimPrefix(creationSource, "aks-")

pkg/cloud/azure/provider.go

@@ -208,7 +208,7 @@ func getRegions(service string, subscriptionsClient subscriptions.Client, provid
 						if loc, ok := allLocations[displName]; ok {
 							supLocations[loc] = displName
 						} else {
-							log.Warnf("unsupported cloud region %q", loc)
+							log.Warnf("unsupported cloud region %q", displName)
 						}
 					}
 					break
@@ -226,7 +226,7 @@ func getRegions(service string, subscriptionsClient subscriptions.Client, provid
 						if loc, ok := allLocations[displName]; ok {
 							supLocations[loc] = displName
 						} else {
-							log.Warnf("unsupported cloud region %q", loc)
+							log.Warnf("unsupported cloud region %q", displName)
 						}
 					}
 					break
@@ -1079,7 +1079,7 @@ func (az *Azure) AllNodePricing() (interface{}, error) {
 }
 
 // NodePricing returns Azure pricing data for a single node
-func (az *Azure) NodePricing(key models.Key) (*models.Node, error) {
+func (az *Azure) NodePricing(key models.Key) (*models.Node, models.PricingMetadata, error) {
 	az.DownloadPricingDataLock.RLock()
 	defer az.DownloadPricingDataLock.RUnlock()
 	pricingDataExists := true
@@ -1088,9 +1088,11 @@ func (az *Azure) NodePricing(key models.Key) (*models.Node, error) {
 		log.DedupedWarningf(1, "Unable to download Azure pricing data")
 	}
 
+	meta := models.PricingMetadata{}
+
 	azKey, ok := key.(*azureKey)
 	if !ok {
-		return nil, fmt.Errorf("azure: NodePricing: key is of type %T", key)
+		return nil, meta, fmt.Errorf("azure: NodePricing: key is of type %T", key)
 	}
 	config, _ := az.GetConfig()
 
@@ -1105,7 +1107,7 @@ func (az *Azure) NodePricing(key models.Key) (*models.Node, error) {
 			if azKey.isValidGPUNode() {
 				n.Node.GPU = "1" // TODO: support multiple GPUs
 			}
-			return n.Node, nil
+			return n.Node, meta, nil
 		}
 		log.Infof("[Info] found spot instance, trying to get retail price for %s: %s, ", spotFeatures, azKey)
 		spotCost, err := getRetailPrice(region, instance, config.CurrencyCode, true)
@@ -1124,7 +1126,7 @@ func (az *Azure) NodePricing(key models.Key) (*models.Node, error) {
 			az.addPricing(spotFeatures, &AzurePricing{
 				Node: spotNode,
 			})
-			return spotNode, nil
+			return spotNode, meta, nil
 		}
 	}
 
@@ -1136,13 +1138,13 @@ func (az *Azure) NodePricing(key models.Key) (*models.Node, error) {
 			if azKey.isValidGPUNode() {
 				n.Node.GPU = azKey.GetGPUCount()
 			}
-			return n.Node, nil
+			return n.Node, meta, nil
 		}
 		log.DedupedWarningf(5, "No pricing data found for node %s from key %s", azKey, azKey.Features())
 	}
 	c, err := az.GetConfig()
 	if err != nil {
-		return nil, fmt.Errorf("No default pricing data available")
+		return nil, meta, fmt.Errorf("No default pricing data available")
 	}
 
 	// GPU Node
@@ -1153,7 +1155,7 @@ func (az *Azure) NodePricing(key models.Key) (*models.Node, error) {
 			UsesBaseCPUPrice: true,
 			GPUCost:          c.GPU,
 			GPU:              azKey.GetGPUCount(),
-		}, nil
+		}, meta, nil
 	}
 
 	// Serverless Node. This is an Azure Container Instance, and no pods can be
@@ -1163,7 +1165,7 @@ func (az *Azure) NodePricing(key models.Key) (*models.Node, error) {
 		return &models.Node{
 			VCPUCost: "0",
 			RAMCost:  "0",
-		}, nil
+		}, meta, nil
 	}
 
 	// Regular Node
@@ -1171,7 +1173,7 @@ func (az *Azure) NodePricing(key models.Key) (*models.Node, error) {
 		VCPUCost:         c.CPU,
 		RAMCost:          c.RAM,
 		UsesBaseCPUPrice: true,
-	}, nil
+	}, meta, nil
 }
 
 // Stubbed NetworkPricing for Azure. Pull directly from azure.json for now
@@ -1415,12 +1417,28 @@ func (az *Azure) findCostForDisk(d *compute.Disk) (float64, error) {
 		storageClass = AzureDiskStandardStorageClass
 	}
 
-	key := *d.Location + "," + storageClass
+	loc := ""
+	if d.Location != nil {
+		loc = *d.Location
+	}
+	key := loc + "," + storageClass
 
+	if p, ok := az.Pricing[key]; !ok || p == nil {
+		return 0.0, fmt.Errorf("failed to find pricing for key: %s", key)
+	}
+	if az.Pricing[key].PV == nil {
+		return 0.0, fmt.Errorf("pricing for key '%s' has nil PV", key)
+	}
 	diskPricePerGBHour, err := strconv.ParseFloat(az.Pricing[key].PV.Cost, 64)
 	if err != nil {
 		return 0.0, fmt.Errorf("error converting to float: %s", err)
 	}
+	if d.DiskProperties == nil {
+		return 0.0, fmt.Errorf("disk properties are nil")
+	}
+	if d.DiskSizeGB == nil {
+		return 0.0, fmt.Errorf("disk size is nil")
+	}
 	cost := diskPricePerGBHour * timeutil.HoursPerMonth * float64(*d.DiskSizeGB)
 
 	return cost, nil

pkg/cloud/azure/provider_test.go

@@ -1,12 +1,15 @@
 package azure
 
 import (
+	"fmt"
 	"testing"
 
+	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute"
 	"github.com/Azure/azure-sdk-for-go/services/preview/commerce/mgmt/2015-06-01-preview/commerce"
 	"github.com/stretchr/testify/require"
 
 	"github.com/opencost/opencost/pkg/cloud/models"
+	"github.com/opencost/opencost/pkg/util/mathutil"
 )
 
 func TestParseAzureSubscriptionID(t *testing.T) {
@@ -95,3 +98,145 @@ func TestConvertMeterToPricings(t *testing.T) {
 		require.Equal(t, expected, results)
 	})
 }
+
+func TestAzure_findCostForDisk(t *testing.T) {
+	var loc string = "location"
+	var size int32 = 1
+
+	az := &Azure{
+		Pricing: map[string]*AzurePricing{
+			"location,nil": nil,
+			"location,nilpv": {
+				PV: nil,
+			},
+			"location,ssd": {
+				PV: &models.PV{
+					Cost: "1",
+				},
+			},
+		},
+	}
+
+	testCases := []struct {
+		name   string
+		disk   *compute.Disk
+		exp    float64
+		expErr error
+	}{
+		{
+			"disk is nil",
+			nil,
+			0.0,
+			fmt.Errorf("disk is empty"),
+		},
+		{
+			"nil location",
+			&compute.Disk{
+				Location: nil,
+				Sku: &compute.DiskSku{
+					Name: "ssd",
+				},
+				DiskProperties: &compute.DiskProperties{
+					DiskSizeGB: &size,
+				},
+			},
+			0.0,
+			fmt.Errorf("failed to find pricing for key: ,ssd"),
+		},
+		{
+			"nil disk properties",
+			&compute.Disk{
+				Location: &loc,
+				Sku: &compute.DiskSku{
+					Name: "ssd",
+				},
+				DiskProperties: nil,
+			},
+			0.0,
+			fmt.Errorf("disk properties are nil"),
+		},
+		{
+			"nil disk size",
+			&compute.Disk{
+				Location: &loc,
+				Sku: &compute.DiskSku{
+					Name: "ssd",
+				},
+				DiskProperties: &compute.DiskProperties{
+					DiskSizeGB: nil,
+				},
+			},
+			0.0,
+			fmt.Errorf("disk size is nil"),
+		},
+		{
+			"sku does not exist",
+			&compute.Disk{
+				Location: &loc,
+				Sku: &compute.DiskSku{
+					Name: "doesnotexist",
+				},
+				DiskProperties: &compute.DiskProperties{
+					DiskSizeGB: &size,
+				},
+			},
+			0.0,
+			fmt.Errorf("failed to find pricing for key: location,doesnotexist"),
+		},
+		{
+			"pricing is nil",
+			&compute.Disk{
+				Sku: &compute.DiskSku{
+					Name: "nil",
+				},
+				DiskProperties: &compute.DiskProperties{
+					DiskSizeGB: &size,
+				},
+			},
+			0.0,
+			fmt.Errorf("failed to find pricing for key: location,nil"),
+		},
+		{
+			"pricing.PV is nil",
+			&compute.Disk{
+				Sku: &compute.DiskSku{
+					Name: "nilpv",
+				},
+				DiskProperties: &compute.DiskProperties{
+					DiskSizeGB: &size,
+				},
+			},
+			0.0,
+			fmt.Errorf("pricing for key 'location,nilpv' has nil PV"),
+		},
+		{
+			"valid (ssd)",
+			&compute.Disk{
+				Location: &loc,
+				Sku: &compute.DiskSku{
+					Name: "ssd",
+				},
+				DiskProperties: &compute.DiskProperties{
+					DiskSizeGB: &size,
+				},
+			},
+			730.0,
+			nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			act, actErr := az.findCostForDisk(tc.disk)
+			if actErr != nil && tc.expErr == nil {
+				t.Fatalf("unexpected error: %s", actErr)
+			}
+			if tc.expErr != nil && actErr == nil {
+				t.Fatalf("missing expected error: %s", tc.expErr)
+			}
+			if !mathutil.Approximately(tc.exp, act) {
+				t.Fatalf("expected value %f; got %f", tc.exp, act)
+			}
+		})
+	}
+}

pkg/cloud/azure/storageconnection.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/opencost/opencost/pkg/cloud"
 	cloudconfig "github.com/opencost/opencost/pkg/cloud/config"
 	"github.com/opencost/opencost/pkg/log"
 )
@@ -15,6 +16,11 @@ import (
 // StorageConnection provides access to Azure Storage
 type StorageConnection struct {
 	StorageConfiguration
+	ConnectionStatus cloud.ConnectionStatus
+}
+
+func (sc *StorageConnection) GetStatus() cloud.ConnectionStatus {
+	return sc.ConnectionStatus
 }
 
 func (sc *StorageConnection) Equals(config cloudconfig.Config) bool {

pkg/cloud/gcp/bigqueryquerier.go

@@ -13,6 +13,10 @@ type BigQueryQuerier struct {
 	ConnectionStatus cloud.ConnectionStatus
 }
 
+func (bqq *BigQueryQuerier) GetStatus() cloud.ConnectionStatus {
+	return bqq.ConnectionStatus
+}
+
 func (bqq *BigQueryQuerier) Equals(config cloudconfig.Config) bool {
 	thatConfig, ok := config.(*BigQueryQuerier)
 	if !ok {

pkg/cloud/gcp/provider.go

@@ -37,6 +37,7 @@ import (
 
 const GKE_GPU_TAG = "cloud.google.com/gke-accelerator"
 const BigqueryUpdateType = "bigqueryupdate"
+const BillingAPIURLFmt = "https://cloudbilling.googleapis.com/v1/services/6F81-5844-456A/skus?key=%s&currencyCode=%s"
 
 const (
 	GCPHourlyPublicIPCost = 0.01
@@ -152,7 +153,7 @@ func (gcp *GCP) GetLocalStorageQuery(window, offset time.Duration, rate bool, us
 	}
 	fmtWindow := timeutil.DurationString(window)
 
-	return fmt.Sprintf(fmtQuery, env.GetPromClusterFilter(), baseMetric, fmtWindow, fmtOffset, env.GetPromClusterLabel(), localStorageCost)
+	return fmt.Sprintf(fmtQuery, baseMetric, env.GetPromClusterFilter(), fmtWindow, fmtOffset, env.GetPromClusterLabel(), localStorageCost)
 }
 
 func (gcp *GCP) GetConfig() (*models.CustomPricing, error) {
@@ -627,7 +628,7 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]models.Key, pvKeys m
 		if err == io.EOF {
 			break
 		} else if err != nil {
-			return nil, "", fmt.Errorf("Error parsing GCP pricing page: %s", err)
+			return nil, "", fmt.Errorf("error parsing GCP pricing page: %s", err)
 		}
 		if t == "skus" {
 			_, err := dec.Token() // consumes [
@@ -760,19 +761,19 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]models.Key, pvKeys m
 				partialCPUMap["e2micro"] = 0.25
 				partialCPUMap["e2small"] = 0.5
 				partialCPUMap["e2medium"] = 1
-				/*
-					var partialCPU float64
-					if strings.ToLower(instanceType) == "f1micro" {
-						partialCPU = 0.2
-					} else if strings.ToLower(instanceType) == "g1small" {
-						partialCPU = 0.5
-					}
-				*/
+
+				if (instanceType == "ram" || instanceType == "cpu") && strings.Contains(strings.ToUpper(product.Description), "T2D AMD") {
+					instanceType = "t2dstandard"
+				}
+				if (instanceType == "ram" || instanceType == "cpu") && strings.Contains(strings.ToUpper(product.Description), "T2A ARM") {
+					instanceType = "t2astandard"
+				}
+
 				var gpuType string
 				for matchnum, group := range nvidiaGPURegex.FindStringSubmatch(product.Description) {
 					if matchnum == 1 {
 						gpuType = strings.ToLower(strings.Join(strings.Split(group, " "), "-"))
-						log.Debug("GPU type found: " + gpuType)
+						log.Debugf("GCP Billing API: GPU type found: '%s'", gpuType)
 					}
 				}
 
@@ -826,16 +827,15 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]models.Key, pvKeys m
 						// (E.g., SKU "2013-37B4-22EA")
 						// and are excluded from cost computations
 						if hourlyPrice == 0 {
-							log.Infof("Excluding reserved GPU SKU #%s", product.SKUID)
+							log.Debugf("GCP Billing API: excluding reserved GPU SKU #%s", product.SKUID)
 							continue
 						}
 
 						for k, key := range inputKeys {
 							if key.GPUType() == gpuType+","+usageType {
 								if region == strings.Split(k, ",")[0] {
-									log.Infof("Matched GPU to node in region \"%s\"", region)
-									log.Debugf("PRODUCT DESCRIPTION: %s", product.Description)
 									matchedKey := key.Features()
+									log.Debugf("GCP Billing API: matched GPU to node: %s: %s", matchedKey, product.Description)
 									if pl, ok := gcpPricingList[matchedKey]; ok {
 										pl.Node.GPUName = gpuType
 										pl.Node.GPUCost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64)
@@ -848,7 +848,6 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]models.Key, pvKeys m
 										}
 										gcpPricingList[matchedKey] = product
 									}
-									log.Infof("Added data for " + matchedKey)
 								}
 							}
 						}
@@ -856,14 +855,18 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]models.Key, pvKeys m
 						_, ok := inputKeys[candidateKey]
 						_, ok2 := inputKeys[candidateKeyGPU]
 						if ok || ok2 {
-							lastRateIndex := len(product.PricingInfo[0].PricingExpression.TieredRates) - 1
 							var nanos float64
 							var unitsBaseCurrency int
-							if lastRateIndex > -1 && len(product.PricingInfo) > 0 {
-								nanos = product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Nanos
-								unitsBaseCurrency, err = strconv.Atoi(product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Units)
-								if err != nil {
-									return nil, "", fmt.Errorf("error parsing base unit price for instance: %w", err)
+							if len(product.PricingInfo) > 0 {
+								lastRateIndex := len(product.PricingInfo[0].PricingExpression.TieredRates) - 1
+								if lastRateIndex >= 0 {
+									nanos = product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Nanos
+									unitsBaseCurrency, err = strconv.Atoi(product.PricingInfo[0].PricingExpression.TieredRates[lastRateIndex].UnitPrice.Units)
+									if err != nil {
+										return nil, "", fmt.Errorf("error parsing base unit price for instance: %w", err)
+									}
+								} else {
+									continue
 								}
 							} else {
 								continue
@@ -877,69 +880,73 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]models.Key, pvKeys m
 								continue
 							} else if strings.Contains(strings.ToUpper(product.Description), "RAM") {
 								if instanceType == "custom" {
-									log.Debug("RAM custom sku is: " + product.Name)
+									log.Debugf("GCP Billing API: RAM custom sku '%s'", product.Name)
 								}
 								if _, ok := gcpPricingList[candidateKey]; ok {
+									log.Debugf("GCP Billing API: key '%s': RAM price: %f", candidateKey, hourlyPrice)
 									gcpPricingList[candidateKey].Node.RAMCost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64)
 								} else {
-									product = &GCPPricing{}
-									product.Node = &models.Node{
+									log.Debugf("GCP Billing API: key '%s': RAM price: %f", candidateKey, hourlyPrice)
+									pricing := &GCPPricing{}
+									pricing.Node = &models.Node{
 										RAMCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 									}
 									partialCPU, pcok := partialCPUMap[instanceType]
 									if pcok {
-										product.Node.VCPU = fmt.Sprintf("%f", partialCPU)
+										pricing.Node.VCPU = fmt.Sprintf("%f", partialCPU)
 									}
-									product.Node.UsageType = usageType
-									gcpPricingList[candidateKey] = product
+									pricing.Node.UsageType = usageType
+									gcpPricingList[candidateKey] = pricing
 								}
 								if _, ok := gcpPricingList[candidateKeyGPU]; ok {
-									log.Infof("Adding RAM %f for %s", hourlyPrice, candidateKeyGPU)
+									log.Debugf("GCP Billing API: key '%s': RAM price: %f", candidateKeyGPU, hourlyPrice)
 									gcpPricingList[candidateKeyGPU].Node.RAMCost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64)
 								} else {
-									log.Infof("Adding RAM %f for %s", hourlyPrice, candidateKeyGPU)
-									product = &GCPPricing{}
-									product.Node = &models.Node{
+									log.Debugf("GCP Billing API: key '%s': RAM price: %f", candidateKeyGPU, hourlyPrice)
+									pricing := &GCPPricing{}
+									pricing.Node = &models.Node{
 										RAMCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 									}
 									partialCPU, pcok := partialCPUMap[instanceType]
 									if pcok {
-										product.Node.VCPU = fmt.Sprintf("%f", partialCPU)
+										pricing.Node.VCPU = fmt.Sprintf("%f", partialCPU)
 									}
-									product.Node.UsageType = usageType
-									gcpPricingList[candidateKeyGPU] = product
+									pricing.Node.UsageType = usageType
+									gcpPricingList[candidateKeyGPU] = pricing
 								}
-								break
 							} else {
 								if _, ok := gcpPricingList[candidateKey]; ok {
+									log.Debugf("GCP Billing API: key '%s': CPU price: %f", candidateKey, hourlyPrice)
 									gcpPricingList[candidateKey].Node.VCPUCost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64)
 								} else {
-									product = &GCPPricing{}
-									product.Node = &models.Node{
+									log.Debugf("GCP Billing API: key '%s': CPU price: %f", candidateKey, hourlyPrice)
+									pricing := &GCPPricing{}
+									pricing.Node = &models.Node{
 										VCPUCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 									}
 									partialCPU, pcok := partialCPUMap[instanceType]
 									if pcok {
-										product.Node.VCPU = fmt.Sprintf("%f", partialCPU)
+										pricing.Node.VCPU = fmt.Sprintf("%f", partialCPU)
 									}
-									product.Node.UsageType = usageType
-									gcpPricingList[candidateKey] = product
+									pricing.Node.UsageType = usageType
+									gcpPricingList[candidateKey] = pricing
 								}
 								if _, ok := gcpPricingList[candidateKeyGPU]; ok {
+									log.Debugf("GCP Billing API: key '%s': CPU price: %f", candidateKeyGPU, hourlyPrice)
 									gcpPricingList[candidateKeyGPU].Node.VCPUCost = strconv.FormatFloat(hourlyPrice, 'f', -1, 64)
 								} else {
-									product = &GCPPricing{}
-									product.Node = &models.Node{
+									log.Debugf("GCP Billing API: key '%s': CPU price: %f", candidateKeyGPU, hourlyPrice)
+									pricing := &GCPPricing{}
+									pricing.Node = &models.Node{
 										VCPUCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 									}
 									partialCPU, pcok := partialCPUMap[instanceType]
 									if pcok {
-										product.Node.VCPU = fmt.Sprintf("%f", partialCPU)
+										pricing.Node.VCPU = fmt.Sprintf("%f", partialCPU)
 									}
-									product.Node.UsageType = usageType
-									gcpPricingList[candidateKeyGPU] = product
+									pricing.Node.UsageType = usageType
+									gcpPricingList[candidateKeyGPU] = pricing
 								}
-								break
 							}
 						}
 					}
@@ -962,13 +969,19 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]models.Key, pvKeys m
 	return gcpPricingList, nextPageToken, nil
 }
 
+func (gcp *GCP) getBillingAPIURL(apiKey, currencyCode string) string {
+	return fmt.Sprintf(BillingAPIURLFmt, apiKey, currencyCode)
+}
+
 func (gcp *GCP) parsePages(inputKeys map[string]models.Key, pvKeys map[string]models.PVKey) (map[string]*GCPPricing, error) {
 	var pages []map[string]*GCPPricing
 	c, err := gcp.GetConfig()
 	if err != nil {
 		return nil, err
 	}
-	url := "https://cloudbilling.googleapis.com/v1/services/6F81-5844-456A/skus?key=" + gcp.APIKey + "&currencyCode=" + c.CurrencyCode
+
+	url := gcp.getBillingAPIURL(gcp.APIKey, c.CurrencyCode)
+
 	log.Infof("Fetch GCP Billing Data from URL: %s", url)
 	var parsePagesHelper func(string) error
 	parsePagesHelper = func(pageToken string) error {
@@ -1019,6 +1032,7 @@ func (gcp *GCP) parsePages(inputKeys map[string]models.Key, pvKeys map[string]mo
 			}
 		}
 	}
+
 	log.Debugf("ALL PAGES: %+v", returnPages)
 	for k, v := range returnPages {
 		if v.Node != nil {
@@ -1539,25 +1553,57 @@ func (gcp *GCP) isValidPricingKey(key models.Key) bool {
 }
 
 // NodePricing returns GCP pricing data for a single node
-func (gcp *GCP) NodePricing(key models.Key) (*models.Node, error) {
+func (gcp *GCP) NodePricing(key models.Key) (*models.Node, models.PricingMetadata, error) {
+	meta := models.PricingMetadata{}
+
+	c, err := gcp.Config.GetCustomPricingData()
+	if err != nil {
+		meta.Warnings = append(meta.Warnings, fmt.Sprintf("failed to detect currency: %s", err))
+	} else {
+		meta.Currency = c.CurrencyCode
+	}
+
 	if n, ok := gcp.getPricing(key); ok {
 		log.Debugf("Returning pricing for node %s: %+v from SKU %s", key, n.Node, n.Name)
+
+		// Add pricing URL, but redact the key (hence, "***"")
+		meta.Source = fmt.Sprintf("Downloaded pricing from %s", gcp.getBillingAPIURL("***", c.CurrencyCode))
+
 		n.Node.BaseCPUPrice = gcp.BaseCPUPrice
-		return n.Node, nil
+
+		return n.Node, meta, nil
 	} else if ok := gcp.isValidPricingKey(key); ok {
+		meta.Warnings = append(meta.Warnings, fmt.Sprintf("No pricing found, but key is valid: %s", key.Features()))
+
 		err := gcp.DownloadPricingData()
 		if err != nil {
-			return nil, fmt.Errorf("Download pricing data failed: %s", err.Error())
+			log.Warnf("no pricing data found for %s", key.Features())
+
+			meta.Warnings = append(meta.Warnings, "Failed to download pricing data")
+
+			return nil, meta, fmt.Errorf("failed to download pricing data: %w", err)
 		}
 		if n, ok := gcp.getPricing(key); ok {
 			log.Debugf("Returning pricing for node %s: %+v from SKU %s", key, n.Node, n.Name)
+
+			// Add pricing URL, but redact the key (hence, "***"")
+			meta.Source = fmt.Sprintf("Downloaded pricing from %s", gcp.getBillingAPIURL("***", c.CurrencyCode))
+
 			n.Node.BaseCPUPrice = gcp.BaseCPUPrice
-			return n.Node, nil
+
+			return n.Node, meta, nil
 		}
-		log.Warnf("no pricing data found for %s: %s", key.Features(), key)
-		return nil, fmt.Errorf("Warning: no pricing data found for %s", key)
+
+		log.Warnf("no pricing data found for %s", key.Features())
+
+		meta.Warnings = append(meta.Warnings, "Failed to find pricing after downloading data, but key is valid")
+
+		return nil, meta, fmt.Errorf("failed to find pricing data: %s", key.Features())
 	}
-	return nil, fmt.Errorf("Warning: no pricing data found for %s", key)
+
+	meta.Warnings = append(meta.Warnings, fmt.Sprintf("No pricing found, and key is not valid: %s", key.Features()))
+
+	return nil, meta, fmt.Errorf("no pricing data found for %s", key.Features())
 }
 
 func (gcp *GCP) ServiceAccountStatus() *models.ServiceAccountStatus {

pkg/cloud/gcp/provider_test.go

@@ -2,7 +2,8 @@ package gcp
 
 import (
 	"bytes"
-	"io/ioutil"
+	"encoding/json"
+	"os"
 	"reflect"
 	"testing"
 
@@ -118,169 +119,80 @@ func TestGetUsageType(t *testing.T) {
 	}
 }
 
-// tests basic parsing of GCP pricing API responses
-// Load a reader object on a portion of a GCP api response
-// Confirm that the resting *GCP object contains the correctly parsed pricing info
-func TestParsePage(t *testing.T) {
+func TestKeyFeatures(t *testing.T) {
+	type testCase struct {
+		key *gcpKey
+		exp string
+	}
 
-	gcpSkuString := `
-	{
-		"skus": [
-			{
-				"name": "services/6F81-5844-456A/skus/039F-D0DA-4055",
-				"skuId": "039F-D0DA-4055",
-				"description": "Nvidia Tesla A100 GPU running in Americas",
-				"category": {
-				  "serviceDisplayName": "Compute Engine",
-				  "resourceFamily": "Compute",
-				  "resourceGroup": "GPU",
-				  "usageType": "OnDemand"
+	testCases := []testCase{
+		{
+			key: &gcpKey{
+				Labels: map[string]string{
+					"node.kubernetes.io/instance-type": "n2-standard-4",
+					"topology.kubernetes.io/region":    "us-east1",
+				},
+			},
+			exp: "us-east1,n2standard,ondemand",
+		},
+		{
+			key: &gcpKey{
+				Labels: map[string]string{
+					"node.kubernetes.io/instance-type": "e2-standard-8",
+					"topology.kubernetes.io/region":    "us-west1",
+					"cloud.google.com/gke-preemptible": "true",
 				},
-				"serviceRegions": [
-				  "us-central1",
-				  "us-east1",
-				  "us-west1"
-				],
-				"pricingInfo": [
-				  {
-					"summary": "",
-					"pricingExpression": {
-					  "usageUnit": "h",
-					  "displayQuantity": 1,
-					  "tieredRates": [
-						{
-						  "startUsageAmount": 0,
-						  "unitPrice": {
-							"currencyCode": "USD",
-							"units": "2",
-							"nanos": 933908000
-						  }
-						}
-					  ],
-					  "usageUnitDescription": "hour",
-					  "baseUnit": "s",
-					  "baseUnitDescription": "second",
-					  "baseUnitConversionFactor": 3600
-					},
-					"currencyConversionRate": 1,
-					"effectiveTime": "2023-03-24T10:52:50.681Z"
-				  }
-				],
-				"serviceProviderName": "Google",
-				"geoTaxonomy": {
-				  "type": "MULTI_REGIONAL",
-				  "regions": [
-					"us-central1",
-					"us-east1",
-					"us-west1"
-				  ]
-				}
 			},
-			{
-				"name": "services/6F81-5844-456A/skus/2390-DCAF-DA38",
-				"skuId": "2390-DCAF-DA38",
-				"description": "A2 Instance Ram running in Americas",
-				"category": {
-				  "serviceDisplayName": "Compute Engine",
-				  "resourceFamily": "Compute",
-				  "resourceGroup": "RAM",
-				  "usageType": "OnDemand"
+			exp: "us-west1,e2standard,preemptible",
+		},
+		{
+			key: &gcpKey{
+				Labels: map[string]string{
+					"node.kubernetes.io/instance-type": "a2-highgpu-1g",
+					"cloud.google.com/gke-gpu":         "true",
+					"cloud.google.com/gke-accelerator": "nvidia-tesla-a100",
+					"topology.kubernetes.io/region":    "us-central1",
 				},
-				"serviceRegions": [
-				  "us-central1",
-				  "us-east1",
-				  "us-west1"
-				],
-				"pricingInfo": [
-				  {
-					"summary": "",
-					"pricingExpression": {
-					  "usageUnit": "GiBy.h",
-					  "displayQuantity": 1,
-					  "tieredRates": [
-						{
-						  "startUsageAmount": 0,
-						  "unitPrice": {
-							"currencyCode": "USD",
-							"units": "0",
-							"nanos": 4237000
-						  }
-						}
-					  ],
-					  "usageUnitDescription": "gibibyte hour",
-					  "baseUnit": "By.s",
-					  "baseUnitDescription": "byte second",
-					  "baseUnitConversionFactor": 3865470566400
-					},
-					"currencyConversionRate": 1,
-					"effectiveTime": "2023-03-24T10:52:50.681Z"
-				  }
-				],
-				"serviceProviderName": "Google",
-				"geoTaxonomy": {
-				  "type": "MULTI_REGIONAL",
-				  "regions": [
-					"us-central1",
-					"us-east1",
-					"us-west1"
-				  ]
-				}
 			},
-			{
-				"name": "services/6F81-5844-456A/skus/2922-40C5-B19F",
-				"skuId": "2922-40C5-B19F",
-				"description": "A2 Instance Core running in Americas",
-				"category": {
-				  "serviceDisplayName": "Compute Engine",
-				  "resourceFamily": "Compute",
-				  "resourceGroup": "CPU",
-				  "usageType": "OnDemand"
+			exp: "us-central1,a2highgpu,ondemand,gpu",
+		},
+		{
+			key: &gcpKey{
+				Labels: map[string]string{
+					"node.kubernetes.io/instance-type": "t2d-standard-1",
+					"topology.kubernetes.io/region":    "asia-southeast1",
 				},
-				"serviceRegions": [
-				  "us-central1",
-				  "us-east1",
-				  "us-west1"
-				],
-				"pricingInfo": [
-				  {
-					"summary": "",
-					"pricingExpression": {
-					  "usageUnit": "h",
-					  "displayQuantity": 1,
-					  "tieredRates": [
-						{
-						  "startUsageAmount": 0,
-						  "unitPrice": {
-							"currencyCode": "USD",
-							"units": "0",
-							"nanos": 31611000
-						  }
-						}
-					  ],
-					  "usageUnitDescription": "hour",
-					  "baseUnit": "s",
-					  "baseUnitDescription": "second",
-					  "baseUnitConversionFactor": 3600
-					},
-					"currencyConversionRate": 1,
-					"effectiveTime": "2023-03-24T10:52:50.681Z"
-				  }
-				],
-				"serviceProviderName": "Google",
-				"geoTaxonomy": {
-				  "type": "MULTI_REGIONAL",
-				  "regions": [
-					"us-central1",
-					"us-east1",
-					"us-west1"
-				  ]
-				}
+			},
+			exp: "asia-southeast1,t2dstandard,ondemand",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.exp, func(t *testing.T) {
+			act := tc.key.Features()
+			if act != tc.exp {
+				t.Errorf("expected '%s'; got '%s'", tc.exp, act)
 			}
-		],
-			"nextPageToken": "APKCS1HVa0YpwgyTFbqbJ1eGwzKZmsPwLqzMZPTSNia5ck1Hc54Tx_Kz3oBxwSnRIdGVxXoSPdf-XlDpyNBf4QuxKcIEgtrQ1LDLWAgZowI0ns7HjrGta2s="
-		}
-	`
-	reader := ioutil.NopCloser(bytes.NewBufferString(gcpSkuString))
+		})
+	}
+}
+
+// tests basic parsing of GCP pricing API responses
+// Load a reader object on a portion of a GCP api response
+// Confirm that the resting *GCP object contains the correctly parsed pricing info
+func TestParsePage(t *testing.T) {
+	// NOTE: SKUs here are copied directly from GCP Billing API. Some of them
+	// are in currency IDR, which relates directly to ticket GTM-52, for which
+	// some of this work was done. So if the prices look huge... don't panic.
+	// The only thing we're testing here is that, given these instance types
+	// and regions and prices, those same prices get set appropriately into
+	// the returned pricing map.
+	skuFilePath := "./test/skus.json"
+	fileBytes, err := os.ReadFile(skuFilePath)
+	if err != nil {
+		t.Fatalf("failed to open file '%s': %s", skuFilePath, err)
+	}
+	reader := bytes.NewReader(fileBytes)
 
 	testGcp := &GCP{}
 
@@ -293,6 +205,24 @@ func TestParsePage(t *testing.T) {
 				"topology.kubernetes.io/region":    "us-central1",
 			},
 		},
+		"us-central1,e2medium,ondemand": &gcpKey{
+			Labels: map[string]string{
+				"node.kubernetes.io/instance-type": "e2-medium",
+				"topology.kubernetes.io/region":    "us-central1",
+			},
+		},
+		"us-central1,e2standard,ondemand": &gcpKey{
+			Labels: map[string]string{
+				"node.kubernetes.io/instance-type": "e2-standard",
+				"topology.kubernetes.io/region":    "us-central1",
+			},
+		},
+		"asia-southeast1,t2dstandard,ondemand": &gcpKey{
+			Labels: map[string]string{
+				"node.kubernetes.io/instance-type": "t2d-standard-1",
+				"topology.kubernetes.io/region":    "asia-southeast1",
+			},
+		},
 	}
 
 	pvKeys := map[string]models.PVKey{}
@@ -361,9 +291,61 @@ func TestParsePage(t *testing.T) {
 				UsageType:        "ondemand",
 			},
 		},
+		"us-central1,e2medium,ondemand": {
+			Node: &models.Node{
+				VCPU:             "1.000000",
+				VCPUCost:         "327.173848364",
+				RAMCost:          "43.85294978",
+				UsesBaseCPUPrice: false,
+				UsageType:        "ondemand",
+			},
+		},
+		"us-central1,e2medium,ondemand,gpu": {
+			Node: &models.Node{
+				VCPU:             "1.000000",
+				VCPUCost:         "327.173848364",
+				RAMCost:          "43.85294978",
+				UsesBaseCPUPrice: false,
+				UsageType:        "ondemand",
+			},
+		},
+		"us-central1,e2standard,ondemand": {
+			Node: &models.Node{
+				VCPUCost:         "327.173848364",
+				RAMCost:          "43.85294978",
+				UsesBaseCPUPrice: false,
+				UsageType:        "ondemand",
+			},
+		},
+		"us-central1,e2standard,ondemand,gpu": {
+			Node: &models.Node{
+				VCPUCost:         "327.173848364",
+				RAMCost:          "43.85294978",
+				UsesBaseCPUPrice: false,
+				UsageType:        "ondemand",
+			},
+		},
+		"asia-southeast1,t2dstandard,ondemand": {
+			Node: &models.Node{
+				VCPUCost:         "508.934997455",
+				RAMCost:          "68.204999658",
+				UsesBaseCPUPrice: false,
+				UsageType:        "ondemand",
+			},
+		},
+		"asia-southeast1,t2dstandard,ondemand,gpu": {
+			Node: &models.Node{
+				VCPUCost:         "508.934997455",
+				RAMCost:          "68.204999658",
+				UsesBaseCPUPrice: false,
+				UsageType:        "ondemand",
+			},
+		},
 	}
 
 	if !reflect.DeepEqual(actualPrices, expectedActualPrices) {
-		t.Fatalf("error parsing GCP prices. parsed %v but expected %v", actualPrices, expectedActualPrices)
+		act, _ := json.Marshal(actualPrices)
+		exp, _ := json.Marshal(expectedActualPrices)
+		t.Errorf("error parsing GCP prices: parsed \n%s\n expected \n%s\n", string(act), string(exp))
 	}
 }

pkg/cloud/gcp/test/skus.json

@@ -0,0 +1,319 @@
+{
+    "skus": [
+        {
+            "name": "services/6F81-5844-456A/skus/039F-D0DA-4055",
+            "skuId": "039F-D0DA-4055",
+            "description": "Nvidia Tesla A100 GPU running in Americas",
+            "category": {
+              "serviceDisplayName": "Compute Engine",
+              "resourceFamily": "Compute",
+              "resourceGroup": "GPU",
+              "usageType": "OnDemand"
+            },
+            "serviceRegions": [
+              "us-central1",
+              "us-east1",
+              "us-west1"
+            ],
+            "pricingInfo": [
+              {
+                "summary": "",
+                "pricingExpression": {
+                  "usageUnit": "h",
+                  "displayQuantity": 1,
+                  "tieredRates": [
+                    {
+                      "startUsageAmount": 0,
+                      "unitPrice": {
+                        "currencyCode": "USD",
+                        "units": "2",
+                        "nanos": 933908000
+                      }
+                    }
+                  ],
+                  "usageUnitDescription": "hour",
+                  "baseUnit": "s",
+                  "baseUnitDescription": "second",
+                  "baseUnitConversionFactor": 3600
+                },
+                "currencyConversionRate": 1,
+                "effectiveTime": "2023-03-24T10:52:50.681Z"
+              }
+            ],
+            "serviceProviderName": "Google",
+            "geoTaxonomy": {
+              "type": "MULTI_REGIONAL",
+              "regions": [
+                "us-central1",
+                "us-east1",
+                "us-west1"
+              ]
+            }
+        },
+        {
+            "name": "services/6F81-5844-456A/skus/2390-DCAF-DA38",
+            "skuId": "2390-DCAF-DA38",
+            "description": "A2 Instance Ram running in Americas",
+            "category": {
+              "serviceDisplayName": "Compute Engine",
+              "resourceFamily": "Compute",
+              "resourceGroup": "RAM",
+              "usageType": "OnDemand"
+            },
+            "serviceRegions": [
+              "us-central1",
+              "us-east1",
+              "us-west1"
+            ],
+            "pricingInfo": [
+              {
+                "summary": "",
+                "pricingExpression": {
+                  "usageUnit": "GiBy.h",
+                  "displayQuantity": 1,
+                  "tieredRates": [
+                    {
+                      "startUsageAmount": 0,
+                      "unitPrice": {
+                        "currencyCode": "USD",
+                        "units": "0",
+                        "nanos": 4237000
+                      }
+                    }
+                  ],
+                  "usageUnitDescription": "gibibyte hour",
+                  "baseUnit": "By.s",
+                  "baseUnitDescription": "byte second",
+                  "baseUnitConversionFactor": 3865470566400
+                },
+                "currencyConversionRate": 1,
+                "effectiveTime": "2023-03-24T10:52:50.681Z"
+              }
+            ],
+            "serviceProviderName": "Google",
+            "geoTaxonomy": {
+              "type": "MULTI_REGIONAL",
+              "regions": [
+                "us-central1",
+                "us-east1",
+                "us-west1"
+              ]
+            }
+        },
+        {
+            "name": "services/6F81-5844-456A/skus/2922-40C5-B19F",
+            "skuId": "2922-40C5-B19F",
+            "description": "A2 Instance Core running in Americas",
+            "category": {
+              "serviceDisplayName": "Compute Engine",
+              "resourceFamily": "Compute",
+              "resourceGroup": "CPU",
+              "usageType": "OnDemand"
+            },
+            "serviceRegions": [
+              "us-central1",
+              "us-east1",
+              "us-west1"
+            ],
+            "pricingInfo": [
+              {
+                "summary": "",
+                "pricingExpression": {
+                  "usageUnit": "h",
+                  "displayQuantity": 1,
+                  "tieredRates": [
+                    {
+                      "startUsageAmount": 0,
+                      "unitPrice": {
+                        "currencyCode": "USD",
+                        "units": "0",
+                        "nanos": 31611000
+                      }
+                    }
+                  ],
+                  "usageUnitDescription": "hour",
+                  "baseUnit": "s",
+                  "baseUnitDescription": "second",
+                  "baseUnitConversionFactor": 3600
+                },
+                "currencyConversionRate": 1,
+                "effectiveTime": "2023-03-24T10:52:50.681Z"
+              }
+            ],
+            "serviceProviderName": "Google",
+            "geoTaxonomy": {
+              "type": "MULTI_REGIONAL",
+              "regions": [
+                "us-central1",
+                "us-east1",
+                "us-west1"
+              ]
+            }
+        },
+        {
+            "name": "services/6F81-5844-456A/skus/4756-01E4-0F32",
+            "skuId": "4756-01E4-0F32",
+            "description": "T2D AMD Instance Ram running in Singapore",
+            "category": {
+                "serviceDisplayName": "Compute Engine",
+                "resourceFamily": "Compute",
+                "resourceGroup": "RAM",
+                "usageType": "OnDemand"
+            },
+            "serviceRegions": [
+                "asia-southeast1"
+            ],
+            "pricingInfo": [
+                {
+                    "summary": "",
+                    "pricingExpression": {
+                        "usageUnit": "GiBy.h",
+                        "usageUnitDescription": "gibibyte hour",
+                        "baseUnit": "By.s",
+                        "displayQuantity": 1,
+                        "tieredRates": [
+                            {
+                                "startUsageAmount": 0,
+                                "unitPrice": {
+                                    "currencyCode": "IDR",
+                                    "units": "68",
+                                    "nanos": 204999658
+                                }
+                            }
+                        ]
+                    },
+                    "currencyConversionRate": 14999.999925,
+                    "EffectiveTime": "2023-08-10T22:49:22.905126Z"
+                }
+            ],
+            "serviceProviderName": "Google",
+            "node": null,
+            "pv": null
+        },
+        {
+            "name": "services/6F81-5844-456A/skus/9E37-EAF4-1576",
+            "skuId": "9E37-EAF4-1576",
+            "description": "T2D AMD Instance Core running in Singapore",
+            "category": {
+                "serviceDisplayName": "Compute Engine",
+                "resourceFamily": "Compute",
+                "resourceGroup": "CPU",
+                "usageType": "OnDemand"
+            },
+            "serviceRegions": [
+                "asia-southeast1"
+            ],
+            "pricingInfo": [
+                {
+                    "summary": "",
+                    "pricingExpression": {
+                        "usageUnit": "h",
+                        "usageUnitDescription": "hour",
+                        "baseUnit": "s",
+                        "displayQuantity": 1,
+                        "tieredRates": [
+                            {
+                                "startUsageAmount": 0,
+                                "unitPrice": {
+                                    "currencyCode": "IDR",
+                                    "units": "508",
+                                    "nanos": 934997455
+                                }
+                            }
+                        ]
+                    },
+                    "currencyConversionRate": 14999.999925,
+                    "EffectiveTime": "2023-08-10T22:49:22.905126Z"
+                }
+            ],
+            "serviceProviderName": "Google",
+            "node": null,
+            "pv": null
+        },
+        {
+            "name": "services/6F81-5844-456A/skus/CF4E-A0C7-E3BF",
+            "skuId": "CF4E-A0C7-E3BF",
+            "description": "E2 Instance Core running in Americas",
+            "category": {
+                "serviceDisplayName": "Compute Engine",
+                "resourceFamily": "Compute",
+                "resourceGroup": "CPU",
+                "usageType": "OnDemand"
+            },
+            "serviceRegions": [
+                "us-central1",
+                "us-east1",
+                "us-west1"
+            ],
+            "pricingInfo": [
+                {
+                    "summary": "",
+                    "pricingExpression": {
+                        "usageUnit": "h",
+                        "usageUnitDescription": "hour",
+                        "baseUnit": "s",
+                        "displayQuantity": 1,
+                        "tieredRates": [
+                            {
+                                "startUsageAmount": 0,
+                                "unitPrice": {
+                                    "currencyCode": "IDR",
+                                    "units": "327",
+                                    "nanos": 173848364
+                                }
+                            }
+                        ]
+                    },
+                    "currencyConversionRate": 14999.999925,
+                    "EffectiveTime": "2023-08-09T07:28:37.555408Z"
+                }
+            ],
+            "serviceProviderName": "Google",
+            "node": null,
+            "pv": null
+        },
+        {
+            "name": "services/6F81-5844-456A/skus/F449-33EC-A5EF",
+            "skuId": "F449-33EC-A5EF",
+            "description": "E2 Instance Ram running in Americas",
+            "category": {
+                "serviceDisplayName": "Compute Engine",
+                "resourceFamily": "Compute",
+                "resourceGroup": "RAM",
+                "usageType": "OnDemand"
+            },
+            "serviceRegions": [
+                "us-central1",
+                "us-east1",
+                "us-west1"
+            ],
+            "pricingInfo": [
+                {
+                    "summary": "",
+                    "pricingExpression": {
+                        "usageUnit": "GiBy.h",
+                        "usageUnitDescription": "gibibyte hour",
+                        "baseUnit": "By.s",
+                        "displayQuantity": 1,
+                        "tieredRates": [
+                            {
+                                "startUsageAmount": 0,
+                                "unitPrice": {
+                                    "currencyCode": "IDR",
+                                    "units": "43",
+                                    "nanos": 852949780
+                                }
+                            }
+                        ]
+                    },
+                    "currencyConversionRate": 14999.999925,
+                    "EffectiveTime": "2023-08-09T07:28:37.555408Z"
+                }
+            ],
+            "serviceProviderName": "Google",
+            "node": null,
+            "pv": null
+        }
+    ],
+    "nextPageToken": "APKCS1HVa0YpwgyTFbqbJ1eGwzKZmsPwLqzMZPTSNia5ck1Hc54Tx_Kz3oBxwSnRIdGVxXoSPdf-XlDpyNBf4QuxKcIEgtrQ1LDLWAgZowI0ns7HjrGta2s="
+}

pkg/cloud/models/models.go

@@ -273,7 +273,7 @@ type Provider interface {
 	GetAddresses() ([]byte, error)
 	GetDisks() ([]byte, error)
 	GetOrphanedResources() ([]OrphanedResource, error)
-	NodePricing(Key) (*Node, error)
+	NodePricing(Key) (*Node, PricingMetadata, error)
 	PVPricing(PVKey) (*PV, error)
 	NetworkPricing() (*Network, error)           // TODO: add key interface arg for dynamic price fetching
 	LoadBalancerPricing() (*LoadBalancer, error) // TODO: add key interface arg for dynamic price fetching

pkg/cloud/models/pricing.go

@@ -0,0 +1,7 @@
+package models
+
+type PricingMetadata struct {
+	Currency string   `json:"currency"`
+	Source   string   `json:"source"`
+	Warnings []string `json:"warnings,omitempty"`
+}

pkg/cloud/provider/csvprovider.go

@@ -228,9 +228,10 @@ func (k *csvKey) ID() string {
 	return k.ProviderID
 }
 
-func (c *CSVProvider) NodePricing(key models.Key) (*models.Node, error) {
+func (c *CSVProvider) NodePricing(key models.Key) (*models.Node, models.PricingMetadata, error) {
 	c.DownloadPricingDataLock.RLock()
 	defer c.DownloadPricingDataLock.RUnlock()
+	meta := models.PricingMetadata{}
 	var node *models.Node
 	if p, ok := c.Pricing[key.ID()]; ok {
 		node = &models.Node{
@@ -277,9 +278,9 @@ func (c *CSVProvider) NodePricing(key models.Key) (*models.Node, error) {
 			}
 			node.Cost = fmt.Sprintf("%f", nc+totalCost)
 		}
-		return node, nil
+		return node, meta, nil
 	} else {
-		return nil, fmt.Errorf("Unable to find Node matching `%s`:`%s`", key.ID(), key.Features())
+		return nil, meta, fmt.Errorf("Unable to find Node matching `%s`:`%s`", key.ID(), key.Features())
 	}
 }
 

pkg/cloud/provider/customprovider.go

@@ -172,10 +172,12 @@ func (cp *CustomProvider) AllNodePricing() (interface{}, error) {
 	return cp.Pricing, nil
 }
 
-func (cp *CustomProvider) NodePricing(key models.Key) (*models.Node, error) {
+func (cp *CustomProvider) NodePricing(key models.Key) (*models.Node, models.PricingMetadata, error) {
 	cp.DownloadPricingDataLock.RLock()
 	defer cp.DownloadPricingDataLock.RUnlock()
 
+	meta := models.PricingMetadata{}
+
 	k := key.Features()
 	var gpuCount string
 	if _, ok := cp.Pricing[k]; !ok {
@@ -205,7 +207,7 @@ func (cp *CustomProvider) NodePricing(key models.Key) (*models.Node, error) {
 		RAMCost:  ramCost,
 		GPUCost:  gpuCost,
 		GPU:      gpuCount,
-	}, nil
+	}, meta, nil
 }
 
 func (cp *CustomProvider) DownloadPricingData() error {

pkg/cloud/scaleway/provider.go

@@ -132,10 +132,12 @@ func (k *scalewayKey) ID() string {
 	return ""
 }
 
-func (c *Scaleway) NodePricing(key models.Key) (*models.Node, error) {
+func (c *Scaleway) NodePricing(key models.Key) (*models.Node, models.PricingMetadata, error) {
 	c.DownloadPricingDataLock.RLock()
 	defer c.DownloadPricingDataLock.RUnlock()
 
+	meta := models.PricingMetadata{}
+
 	// There is only the zone and the instance ID in the providerID, hence we must use the features
 	split := strings.Split(key.Features(), ",")
 	if pricing, ok := c.Pricing[split[0]]; ok {
@@ -151,12 +153,12 @@ func (c *Scaleway) NodePricing(key models.Key) (*models.Node, error) {
 				InstanceType: split[1],
 				Region:       split[0],
 				GPUName:      key.GPUType(),
-			}, nil
+			}, meta, nil
 
 		}
 
 	}
-	return nil, fmt.Errorf("Unable to find node pricing matching thes features `%s`", key.Features())
+	return nil, meta, fmt.Errorf("Unable to find node pricing matching thes features `%s`", key.Features())
 }
 
 func (c *Scaleway) LoadBalancerPricing() (*models.LoadBalancer, error) {

pkg/cmd/costmodel/costmodel.go

@@ -55,7 +55,8 @@ func Execute(opts *CostModelOpts) error {
 func StartExportWorker(ctx context.Context, model costmodel.AllocationModel) error {
 	exportPath := env.GetExportCSVFile()
 	if exportPath == "" {
-		return fmt.Errorf("%s is not set, skipping CSV exporter", exportPath)
+		log.Infof("%s is not set, CSV export is disabled", env.ExportCSVFile)
+		return nil
 	}
 	fm, err := filemanager.NewFileManager(exportPath)
 	if err != nil {

pkg/costmodel/aggregation.go

@@ -2112,17 +2112,30 @@ func (a *Accesses) AggregateCostModelHandler(w http.ResponseWriter, r *http.Requ
 // ParseAggregationProperties attempts to parse and return aggregation properties
 // encoded under the given key. If none exist, or if parsing fails, an error
 // is returned with empty AllocationProperties.
-func ParseAggregationProperties(qp httputil.QueryParams, key string) ([]string, error) {
+func ParseAggregationProperties(aggregations []string) ([]string, error) {
 	aggregateBy := []string{}
-	for _, agg := range qp.GetList(key, ",") {
-		aggregate := strings.TrimSpace(agg)
-		if aggregate != "" {
-			if prop, err := kubecost.ParseProperty(aggregate); err == nil {
-				aggregateBy = append(aggregateBy, string(prop))
-			} else if strings.HasPrefix(aggregate, "label:") {
-				aggregateBy = append(aggregateBy, aggregate)
-			} else if strings.HasPrefix(aggregate, "annotation:") {
-				aggregateBy = append(aggregateBy, aggregate)
+	// In case of no aggregation option, aggregate to the container, with a key Cluster/Node/Namespace/Pod/Container
+	if len(aggregations) == 0 {
+		aggregateBy = []string{
+			kubecost.AllocationClusterProp,
+			kubecost.AllocationNodeProp,
+			kubecost.AllocationNamespaceProp,
+			kubecost.AllocationPodProp,
+			kubecost.AllocationContainerProp,
+		}
+	} else if len(aggregations) == 1 && aggregations[0] == "all" {
+		aggregateBy = []string{}
+	} else {
+		for _, agg := range aggregations {
+			aggregate := strings.TrimSpace(agg)
+			if aggregate != "" {
+				if prop, err := kubecost.ParseProperty(aggregate); err == nil {
+					aggregateBy = append(aggregateBy, string(prop))
+				} else if strings.HasPrefix(aggregate, "label:") {
+					aggregateBy = append(aggregateBy, aggregate)
+				} else if strings.HasPrefix(aggregate, "annotation:") {
+					aggregateBy = append(aggregateBy, aggregate)
+				}
 			}
 		}
 	}
@@ -2154,7 +2167,8 @@ func (a *Accesses) ComputeAllocationHandlerSummary(w http.ResponseWriter, r *htt
 	// aggregate results. Some fields allow a sub-field, which is distinguished
 	// with a colon; e.g. "label:app".
 	// Examples: "namespace", "namespace,label:app"
-	aggregateBy, err := ParseAggregationProperties(qp, "aggregate")
+	aggregations := qp.GetList("aggregate", ",")
+	aggregateBy, err := ParseAggregationProperties(aggregations)
 	if err != nil {
 		http.Error(w, fmt.Sprintf("Invalid 'aggregate' parameter: %s", err), http.StatusBadRequest)
 	}
@@ -2235,7 +2249,8 @@ func (a *Accesses) ComputeAllocationHandler(w http.ResponseWriter, r *http.Reque
 	// aggregate results. Some fields allow a sub-field, which is distinguished
 	// with a colon; e.g. "label:app".
 	// Examples: "namespace", "namespace,label:app"
-	aggregateBy, err := ParseAggregationProperties(qp, "aggregate")
+	aggregations := qp.GetList("aggregate", ",")
+	aggregateBy, err := ParseAggregationProperties(aggregations)
 	if err != nil {
 		http.Error(w, fmt.Sprintf("Invalid 'aggregate' parameter: %s", err), http.StatusBadRequest)
 	}
@@ -2267,7 +2282,7 @@ func (a *Accesses) ComputeAllocationHandler(w http.ResponseWriter, r *http.Reque
 	// include aggregated labels/annotations if true
 	includeAggregatedMetadata := qp.GetBool("includeAggregatedMetadata", false)
 
-	asr, err := a.Model.QueryAllocation(window, resolution, step, aggregateBy, includeIdle, idleByNode, includeProportionalAssetResourceCosts, includeAggregatedMetadata)
+	asr, err := a.Model.QueryAllocation(window, resolution, step, aggregateBy, includeIdle, idleByNode, includeProportionalAssetResourceCosts, includeAggregatedMetadata, accumulateBy)
 	if err != nil {
 		if strings.Contains(strings.ToLower(err.Error()), "bad request") {
 			WriteError(w, BadRequest(err.Error()))
@@ -2278,16 +2293,6 @@ func (a *Accesses) ComputeAllocationHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	// Accumulate, if requested
-	if accumulateBy != kubecost.AccumulateOptionNone {
-		asr, err = asr.Accumulate(accumulateBy)
-		if err != nil {
-			log.Errorf("error accumulating by %v: %s", accumulateBy, err)
-			WriteError(w, InternalServerError(fmt.Errorf("error accumulating by %v: %s", accumulateBy, err).Error()))
-			return
-		}
-	}
-
 	w.Write(WrapData(asr, nil))
 }
 

pkg/costmodel/aggregation_test.go

@@ -3,6 +3,7 @@ package costmodel
 import (
 	"testing"
 
+	"github.com/opencost/opencost/pkg/kubecost"
 	"github.com/opencost/opencost/pkg/util"
 )
 
@@ -193,3 +194,40 @@ func TestScaleHourlyCostData(t *testing.T) {
 		}
 	}
 }
+
+func TestParseAggregationProperties_Default(t *testing.T) {
+	got, err := ParseAggregationProperties([]string{})
+	expected := []string{
+		kubecost.AllocationClusterProp,
+		kubecost.AllocationNodeProp,
+		kubecost.AllocationNamespaceProp,
+		kubecost.AllocationPodProp,
+		kubecost.AllocationContainerProp,
+	}
+
+	if err != nil {
+		t.Fatalf("TestParseAggregationPropertiesDefault: unexpected error: %s", err)
+	}
+
+	if len(expected) != len(got) {
+		t.Fatalf("TestParseAggregationPropertiesDefault: expected length of %d, got: %d", len(expected), len(got))
+	}
+
+	for i := range got {
+		if got[i] != expected[i] {
+			t.Fatalf("TestParseAggregationPropertiesDefault: expected[i] should be %s, got[i]:%s", expected[i], got[i])
+		}
+	}
+}
+
+func TestParseAggregationProperties_All(t *testing.T) {
+	got, err := ParseAggregationProperties([]string{"all"})
+
+	if err != nil {
+		t.Fatalf("TestParseAggregationPropertiesDefault: unexpected error: %s", err)
+	}
+
+	if len(got) != 0 {
+		t.Fatalf("TestParseAggregationPropertiesDefault: expected length of 0, got: %d", len(got))
+	}
+}

pkg/costmodel/allocation.go

@@ -55,7 +55,7 @@ const (
 	queryFmtPodsWithReplicaSetOwner     = `sum(avg_over_time(kube_pod_owner{owner_kind="ReplicaSet", %s}[%s])) by (pod, owner_name, namespace ,%s)`
 	queryFmtReplicaSetsWithoutOwners    = `avg(avg_over_time(kube_replicaset_owner{owner_kind="<none>", owner_name="<none>", %s}[%s])) by (replicaset, namespace, %s)`
 	queryFmtReplicaSetsWithRolloutOwner = `avg(avg_over_time(kube_replicaset_owner{owner_kind="Rollout", %s}[%s])) by (replicaset, namespace, owner_kind, owner_name, %s)`
-	queryFmtLBCostPerHr                 = `avg(avg_over_time(kubecost_load_balancer_cost{%s}[%s])) by (namespace, service_name, %s)`
+	queryFmtLBCostPerHr                 = `avg(avg_over_time(kubecost_load_balancer_cost{%s}[%s])) by (namespace, service_name, ingress_ip, %s)`
 	queryFmtLBActiveMins                = `count(kubecost_load_balancer_cost{%s}) by (namespace, service_name, %s)[%s:%s]`
 	queryFmtOldestSample                = `min_over_time(timestamp(group(node_cpu_hourly_cost{%s}))[%s:%s])`
 	queryFmtNewestSample                = `max_over_time(timestamp(group(node_cpu_hourly_cost{%s}))[%s:%s])`
@@ -84,7 +84,7 @@ const (
 	// the resolution, to make sure the irate always has two points to query
 	// in case the Prom scrape duration has been reduced to be equal to the
 	// ETL resolution.
-	queryFmtCPUUsageMaxSubquery = `max(max_over_time(irate(container_cpu_usage_seconds_total{container_name!="POD", container_name!="", %s}[%s])[%s:%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
+	queryFmtCPUUsageMaxSubquery = `max(max_over_time(irate(container_cpu_usage_seconds_total{container!="POD", container!="", %s}[%s])[%s:%s])) by (container, pod_name, pod, namespace, instance, %s)`
 )
 
 // Constants for Network Cost Subtype
@@ -279,6 +279,9 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	result.Errors = errors
 	result.Warnings = warnings
 
+	// Convert any NaNs to 0 to avoid JSON marshaling issues and avoid cascading NaN appearances elsewhere
+	result.SanitizeNaN()
+
 	return result, nil
 }
 
@@ -647,14 +650,14 @@ func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Dur
 	// a PVC, we get time running there, so this is only inaccurate
 	// for short-lived, unmounted PVs.)
 	pvMap := map[pvKey]*pv{}
-	buildPVMap(resolution, pvMap, resPVCostPerGiBHour, resPVActiveMins)
+	buildPVMap(resolution, pvMap, resPVCostPerGiBHour, resPVActiveMins, window)
 	applyPVBytes(pvMap, resPVBytes)
 
 	// Build out the map of all PVCs with time running, bytes requested,
 	// and connect to the correct PV from pvMap. (If no PV exists, that
 	// is noted, but does not result in any allocation/cost.)
 	pvcMap := map[pvcKey]*pvc{}
-	buildPVCMap(resolution, pvcMap, pvMap, resPVCInfo)
+	buildPVCMap(resolution, pvcMap, pvMap, resPVCInfo, window)
 	applyPVCBytesRequested(pvcMap, resPVCBytesRequested)
 
 	// Build out the relationships of pods to their PVCs. This step
@@ -671,7 +674,7 @@ func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Dur
 	applyUnmountedPVs(window, podMap, pvMap, pvcMap)
 
 	lbMap := make(map[serviceKey]*lbCost)
-	getLoadBalancerCosts(lbMap, resLBCostPerHr, resLBActiveMins, resolution)
+	getLoadBalancerCosts(lbMap, resLBCostPerHr, resLBActiveMins, resolution, window)
 	applyLoadBalancersToPods(window, podMap, lbMap, allocsByService)
 
 	// Build out a map of Nodes with resource costs, discounts, and node types

pkg/costmodel/allocation_helpers.go

@@ -18,7 +18,15 @@ import (
 
 // This is a bit of a hack to work around garbage data from cadvisor
 // Ideally you cap each pod to the max CPU on its node, but that involves a bit more complexity, as it it would need to be done when allocations joins with asset data.
-const MAX_CPU_CAP = 512
+const CPU_SANITY_LIMIT = 512
+
+// Sanity Limit for PV usage, set to 10 PB, in bytes for now
+const KiB = 1024.0
+const MiB = 1024.0 * KiB
+const GiB = 1024.0 * MiB
+const TiB = 1024.0 * GiB
+const PiB = 1024.0 * TiB
+const PV_USAGE_SANITY_LIMIT_BYTES = 10.0 * PiB
 
 /* Pod Helpers */
 
@@ -156,7 +164,7 @@ func applyPodResults(window kubecost.Window, resolution time.Duration, podMap ma
 
 		}
 
-		allocStart, allocEnd := calculateStartEndFromIsRunning(res, resolution, window)
+		allocStart, allocEnd := calculateStartAndEnd(res, resolution, window)
 		if allocStart.IsZero() || allocEnd.IsZero() {
 			continue
 		}
@@ -231,7 +239,7 @@ func applyCPUCoresAllocated(podMap map[podKey]*pod, resCPUCoresAllocated []*prom
 			}
 
 			cpuCores := res.Values[0].Value
-			if cpuCores > MAX_CPU_CAP {
+			if cpuCores > CPU_SANITY_LIMIT {
 				log.Infof("[WARNING] Very large cpu allocation, clamping to %f", res.Values[0].Value*(thisPod.Allocations[container].Minutes()/60.0))
 				cpuCores = 0.0
 			}
@@ -292,7 +300,7 @@ func applyCPUCoresRequested(podMap map[podKey]*pod, resCPUCoresRequested []*prom
 			if thisPod.Allocations[container].CPUCores() < res.Values[0].Value {
 				thisPod.Allocations[container].CPUCoreHours = res.Values[0].Value * (thisPod.Allocations[container].Minutes() / 60.0)
 			}
-			if thisPod.Allocations[container].CPUCores() > MAX_CPU_CAP {
+			if thisPod.Allocations[container].CPUCores() > CPU_SANITY_LIMIT {
 				log.Infof("[WARNING] Very large cpu allocation, clamping! to %f", res.Values[0].Value*(thisPod.Allocations[container].Minutes()/60.0))
 				thisPod.Allocations[container].CPUCoreHours = res.Values[0].Value * (thisPod.Allocations[container].Minutes() / 60.0)
 			}
@@ -347,7 +355,7 @@ func applyCPUCoresUsedAvg(podMap map[podKey]*pod, resCPUCoresUsedAvg []*prom.Que
 			}
 
 			thisPod.Allocations[container].CPUCoreUsageAverage = res.Values[0].Value
-			if res.Values[0].Value > MAX_CPU_CAP {
+			if res.Values[0].Value > CPU_SANITY_LIMIT {
 				log.Infof("[WARNING] Very large cpu USAGE, dropping outlier")
 				thisPod.Allocations[container].CPUCoreUsageAverage = 0.0
 			}
@@ -960,7 +968,7 @@ func applyLabels(podMap map[podKey]*pod, nodeLabels map[nodeKey]map[string]strin
 
 			alloc.Properties.Labels = allocLabels
 			alloc.Properties.NamespaceLabels = nsLabels
-			
+
 		}
 	}
 }
@@ -1335,14 +1343,15 @@ func applyServicesToPods(podMap map[podKey]*pod, podLabels map[podKey]map[string
 	}
 }
 
-func getLoadBalancerCosts(lbMap map[serviceKey]*lbCost, resLBCost, resLBActiveMins []*prom.QueryResult, resolution time.Duration) {
+func getLoadBalancerCosts(lbMap map[serviceKey]*lbCost, resLBCost, resLBActiveMins []*prom.QueryResult, resolution time.Duration, window kubecost.Window) {
 	for _, res := range resLBActiveMins {
 		serviceKey, err := resultServiceKey(res, env.GetPromClusterLabel(), "namespace", "service_name")
 		if err != nil || len(res.Values) == 0 {
 			continue
 		}
 
-		lbStart, lbEnd := calculateStartAndEnd(res, resolution)
+		// load balancers have interpolation for costs, we don't need to offset the resolution
+		lbStart, lbEnd := calculateStartAndEnd(res, resolution, window)
 		if lbStart.IsZero() || lbEnd.IsZero() {
 			log.Warnf("CostModel.ComputeAllocation: pvc %s has no running time", serviceKey)
 		}
@@ -1358,11 +1367,33 @@ func getLoadBalancerCosts(lbMap map[serviceKey]*lbCost, resLBCost, resLBActiveMi
 		if err != nil {
 			continue
 		}
+
+		// get the ingress IP to determine if this is a private LB
+		ip, err := res.GetString("ingress_ip")
+		if err != nil {
+			log.Warnf("error getting ingress ip for key %s: %v, skipping", serviceKey, err)
+			// do not count the time that the service was being created or deleted
+			// ingress IP will be empty string
+			// only add cost to allocation when external IP is provisioned
+			if ip == "" {
+				continue
+			}
+		}
+
 		// Apply cost as price-per-hour * hours
 		if lb, ok := lbMap[serviceKey]; ok {
 			lbPricePerHr := res.Values[0].Value
-			hours := lb.End.Sub(lb.Start).Hours()
-			lb.TotalCost += lbPricePerHr * hours
+			// interpolate any missing data
+			resolutionHours := resolution.Hours()
+			resultHours := lb.End.Sub(lb.Start).Hours()
+			scaleFactor := (resolutionHours + resultHours) / resultHours
+
+			// after scaling, we can adjust the timings to reflect the interpolated data
+			lb.End = lb.End.Add(resolution)
+
+			lb.TotalCost += lbPricePerHr * resultHours * scaleFactor
+			lb.Ip = ip
+			lb.Private = privateIPCheck(ip)
 		} else {
 			log.DedupedWarningf(20, "CostModel: found minutes for key that does not exist: %s", serviceKey)
 		}
@@ -1407,6 +1438,23 @@ func applyLoadBalancersToPods(window kubecost.Window, podMap map[podKey]*pod, lb
 			alloc.LoadBalancerCost += lb.TotalCost * hours / totalHours
 		}
 
+		for _, alloc := range allocs {
+			if alloc.LoadBalancers == nil {
+				alloc.LoadBalancers = kubecost.LbAllocations{}
+			}
+
+			if _, found := alloc.LoadBalancers[sKey.String()]; found {
+				alloc.LoadBalancers[sKey.String()].Cost += alloc.LoadBalancerCost
+			} else {
+				alloc.LoadBalancers[sKey.String()] = &kubecost.LbAllocation{
+					Service: sKey.Namespace + "/" + sKey.Service,
+					Cost:    alloc.LoadBalancerCost,
+					Private: lb.Private,
+					Ip:      lb.Ip,
+				}
+			}
+		}
+
 		// If there was no overlap apply to Unmounted pod
 		if len(allocHours) == 0 {
 			pod := getUnmountedPodForCluster(window, podMap, sKey.Cluster)
@@ -1728,7 +1776,7 @@ func (cm *CostModel) getNodePricing(nodeMap map[nodeKey]*nodePricing, nodeKey no
 
 /* PV/PVC Helpers */
 
-func buildPVMap(resolution time.Duration, pvMap map[pvKey]*pv, resPVCostPerGiBHour, resPVActiveMins []*prom.QueryResult) {
+func buildPVMap(resolution time.Duration, pvMap map[pvKey]*pv, resPVCostPerGiBHour, resPVActiveMins []*prom.QueryResult, window kubecost.Window) {
 	for _, result := range resPVActiveMins {
 		key, err := resultPVKey(result, env.GetPromClusterLabel(), "persistentvolume")
 		if err != nil {
@@ -1736,7 +1784,7 @@ func buildPVMap(resolution time.Duration, pvMap map[pvKey]*pv, resPVCostPerGiBHo
 			continue
 		}
 
-		pvStart, pvEnd := calculateStartAndEnd(result, resolution)
+		pvStart, pvEnd := calculateStartAndEnd(result, resolution, window)
 		if pvStart.IsZero() || pvEnd.IsZero() {
 			log.Warnf("CostModel.ComputeAllocation: pv %s has no running time", key)
 		}
@@ -1780,11 +1828,17 @@ func applyPVBytes(pvMap map[pvKey]*pv, resPVBytes []*prom.QueryResult) {
 			continue
 		}
 
-		pvMap[key].Bytes = res.Values[0].Value
+		pvBytesUsed := res.Values[0].Value
+		if pvBytesUsed < PV_USAGE_SANITY_LIMIT_BYTES {
+			pvMap[key].Bytes = pvBytesUsed
+		} else {
+			pvMap[key].Bytes = 0
+			log.Warnf("PV usage exceeds sanity limit, clamping to zero")
+		}
 	}
 }
 
-func buildPVCMap(resolution time.Duration, pvcMap map[pvcKey]*pvc, pvMap map[pvKey]*pv, resPVCInfo []*prom.QueryResult) {
+func buildPVCMap(resolution time.Duration, pvcMap map[pvcKey]*pvc, pvMap map[pvKey]*pv, resPVCInfo []*prom.QueryResult, window kubecost.Window) {
 	for _, res := range resPVCInfo {
 		cluster, err := res.GetString(env.GetPromClusterLabel())
 		if err != nil {
@@ -1805,7 +1859,7 @@ func buildPVCMap(resolution time.Duration, pvcMap map[pvcKey]*pvc, pvMap map[pvK
 		pvKey := newPVKey(cluster, volume)
 		pvcKey := newPVCKey(cluster, namespace, name)
 
-		pvcStart, pvcEnd := calculateStartAndEnd(res, resolution)
+		pvcStart, pvcEnd := calculateStartAndEnd(res, resolution, window)
 		if pvcStart.IsZero() || pvcEnd.IsZero() {
 			log.Warnf("CostModel.ComputeAllocation: pvc %s has no running time", pvcKey)
 		}
@@ -1906,6 +1960,7 @@ func buildPodPVCMap(podPVCMap map[podKey][]*pvc, pvMap map[pvKey]*pv, pvcMap map
 		}
 	}
 }
+
 func applyPVCsToPods(window kubecost.Window, podMap map[podKey]*pod, podPVCMap map[podKey][]*pvc, pvcMap map[pvcKey]*pvc) {
 	// Because PVCs can be shared among pods, the respective pv cost
 	// needs to be evenly distributed to those pods based on time
@@ -1951,12 +2006,20 @@ func applyPVCsToPods(window kubecost.Window, podMap map[podKey]*pod, podPVCMap m
 
 		pvc, ok := pvcMap[thisPVCKey]
 		if !ok {
-			log.DedupedWarningf(5, "Missing pvc with key %s", thisPVCKey)
+			log.Warnf("Allocation: Compute: applyPVCsToPods: missing pvc with key %s", thisPVCKey)
+			continue
+		}
+		if pvc == nil {
+			log.Warnf("Allocation: Compute: applyPVCsToPods: nil pvc with key %s", thisPVCKey)
 			continue
 		}
 
 		// Determine coefficients for each pvc-pod relation.
-		sharedPVCCostCoefficients := getPVCCostCoefficients(intervals, pvc)
+		sharedPVCCostCoefficients, err := getPVCCostCoefficients(intervals, pvc)
+		if err != nil {
+			log.Warnf("Allocation: Compute: applyPVCsToPods: getPVCCostCoefficients: %s", err)
+			continue
+		}
 
 		// Distribute pvc costs to Allocations
 		for thisPodKey, coeffComponents := range sharedPVCCostCoefficients {
@@ -1988,8 +2051,9 @@ func applyPVCsToPods(window kubecost.Window, podMap map[podKey]*pod, podPVCMap m
 					Cluster: pvc.Volume.Cluster,
 					Name:    pvc.Volume.Name,
 				}
+
 				// Both Cost and byteHours should be multiplied by the coef and divided by count
-				// so that you if all allocations with a given pv key are summed the result of those
+				// so that if all allocations with a given pv key are summed the result of those
 				// would be equal to the values of the original pv
 				count := float64(len(pod.Allocations))
 				alloc.PVs[pvKey] = &kubecost.PVAllocation{
@@ -2139,98 +2203,35 @@ func getUnmountedPodForNamespace(window kubecost.Window, podMap map[podKey]*pod,
 	return thisPod
 }
 
-func calculateStartAndEnd(result *prom.QueryResult, resolution time.Duration) (time.Time, time.Time) {
+func calculateStartAndEnd(result *prom.QueryResult, resolution time.Duration, window kubecost.Window) (time.Time, time.Time) {
+	// Start and end for a range vector are pulled from the timestamps of the
+	// first and final values in the range. There is no "offsetting" required
+	// of the start or the end, as we used to do. If you query for a duration
+	// of time that is divisible by the given resolution, and set the end time
+	// to be precisely the end of the window, Prometheus should give all the
+	// relevant timestamps.
+	//
+	// E.g. avg(kube_pod_container_status_running{}) by (pod, namespace)[1h:1m]
+	// with time=01:00:00 will return, for a pod running the entire time,
+	// 61 timestamps where the first is 00:00:00 and the last is 01:00:00.
 	s := time.Unix(int64(result.Values[0].Timestamp), 0).UTC()
-	// subtract resolution from start time to cover full time period
-	s = s.Add(-resolution)
 	e := time.Unix(int64(result.Values[len(result.Values)-1].Timestamp), 0).UTC()
-	return s, e
-}
 
-// calculateStartEndFromIsRunning Calculates the start and end of a prom result when the values of the datum are 0 for not running and 1 for running
-// the coeffs are used to adjust the start and end when the value is not equal to 1 or 0, which means that pod came up or went down in that window.
-func calculateStartEndFromIsRunning(result *prom.QueryResult, resolution time.Duration, window kubecost.Window) (time.Time, time.Time) {
-	// start and end are the timestamps of the first and last
-	// minutes the pod was running, respectively. We subtract one resolution
-	// from start because this point will actually represent the end
-	// of the first minute. We don't subtract from end because it
-	// already represents the end of the last minute.
-	var start, end time.Time
-	startAdjustmentCoeff, endAdjustmentCoeff := 1.0, 1.0
-	for _, datum := range result.Values {
-		t := time.Unix(int64(datum.Timestamp), 0)
-
-		if start.IsZero() && datum.Value > 0 && window.Contains(t) {
-			// Set the start timestamp to the earliest non-zero timestamp
-			start = t
-
-			// Record adjustment coefficient, i.e. the portion of the start
-			// timestamp to "ignore". That is, sometimes the value will be
-			// 0.5, meaning that we should discount the time running by
-			// half of the resolution the timestamp stands for.
-			startAdjustmentCoeff = (1.0 - datum.Value)
-		}
-
-		if datum.Value > 0 && window.Contains(t) {
-			// Set the end timestamp to the latest non-zero timestamp
-			end = t
-
-			// Record adjustment coefficient, i.e. the portion of the end
-			// timestamp to "ignore". (See explanation above for start.)
-			endAdjustmentCoeff = (1.0 - datum.Value)
-		}
-	}
-
-	// Do not attempt to adjust start if it is zero
-	if !start.IsZero() {
-		// Adjust timestamps according to the resolution and the adjustment
-		// coefficients, as described above. That is, count the start timestamp
-		// from the beginning of the resolution, not the end. Then "reduce" the
-		// start and end by the correct amount, in the case that the "running"
-		// value of the first or last timestamp was not a full 1.0.
-		start = start.Add(-resolution)
-		// Note: the *100 and /100 are necessary because Duration is an int, so
-		// 0.5, for instance, will be truncated, resulting in no adjustment.
-		start = start.Add(time.Duration(startAdjustmentCoeff*100) * resolution / time.Duration(100))
-		end = end.Add(-time.Duration(endAdjustmentCoeff*100) * resolution / time.Duration(100))
-
-		// Ensure that the start is always within the window, adjusting
-		// for the occasions where start falls 1m before the query window.
-		// NOTE: window here will always be closed (so no need to nil check
-		// "start").
-		// TODO:CLEANUP revisit query methodology to figure out why this is
-		// happening on occasion
-		if start.Before(*window.Start()) {
-			start = *window.Start()
-		}
-	}
-
-	// do not attempt to adjust end if it is zero
-	if !end.IsZero() {
-		// If there is only one point with a value <= 0.5 that the start and
-		// end timestamps both share, then we will enter this case because at
-		// least half of a resolution will be subtracted from both the start
-		// and the end. If that is the case, then add back half of each side
-		// so that the pod is said to run for half a resolution total.
-		// e.g. For resolution 1m and a value of 0.5 at one timestamp, we'll
-		//      end up with end == start and each coeff == 0.5. In
-		//      that case, add 0.25m to each side, resulting in 0.5m duration.
-		if !end.After(start) {
-			start = start.Add(-time.Duration(50*startAdjustmentCoeff) * resolution / time.Duration(100))
-			end = end.Add(time.Duration(50*endAdjustmentCoeff) * resolution / time.Duration(100))
-		}
-
-		// Ensure that the allocEnf is always within the window, adjusting
-		// for the occasions where end falls 1m after the query window. This
-		// has not ever happened, but is symmetrical with the start check
-		// above.
-		// NOTE: window here will always be closed (so no need to nil check
-		// "end").
-		// TODO:CLEANUP revisit query methodology to figure out why this is
-		// happening on occasion
-		if end.After(*window.End()) {
-			end = *window.End()
-		}
-	}
-	return start, end
+	// The only corner-case here is what to do if you only get one timestamp.
+	// This dilemma still requires the use of the resolution, and can be
+	// clamped using the window. In this case, we want to honor the existence
+	// of the pod by giving "one resolution" worth of duration, half on each
+	// side of the given timestamp.
+	if s.Equal(e) {
+		s = s.Add(-1 * resolution / time.Duration(2))
+		e = e.Add(resolution / time.Duration(2))
+	}
+	if s.Before(*window.Start()) {
+		s = *window.Start()
+	}
+	if e.After(*window.End()) {
+		e = *window.End()
+	}
+
+	return s, e
 }

pkg/costmodel/allocation_helpers_test.go

@@ -2,11 +2,12 @@ package costmodel
 
 import (
 	"fmt"
+	"testing"
+	"time"
+
 	"github.com/opencost/opencost/pkg/kubecost"
 	"github.com/opencost/opencost/pkg/prom"
 	"github.com/opencost/opencost/pkg/util"
-	"testing"
-	"time"
 )
 
 const Ki = 1024
@@ -271,6 +272,9 @@ func TestBuildPVMap(t *testing.T) {
 						"persistentvolume": "pv1",
 					},
 					Values: []*util.Vector{
+						{
+							Timestamp: startFloat,
+						},
 						{
 							Timestamp: startFloat + (hour * 6),
 						},
@@ -288,6 +292,9 @@ func TestBuildPVMap(t *testing.T) {
 						"persistentvolume": "pv2",
 					},
 					Values: []*util.Vector{
+						{
+							Timestamp: startFloat,
+						},
 						{
 							Timestamp: startFloat + (hour * 6),
 						},
@@ -308,6 +315,9 @@ func TestBuildPVMap(t *testing.T) {
 						"persistentvolume": "pv3",
 					},
 					Values: []*util.Vector{
+						{
+							Timestamp: startFloat + (hour * 6),
+						},
 						{
 							Timestamp: startFloat + (hour * 12),
 						},
@@ -322,6 +332,9 @@ func TestBuildPVMap(t *testing.T) {
 						"persistentvolume": "pv4",
 					},
 					Values: []*util.Vector{
+						{
+							Timestamp: startFloat,
+						},
 						{
 							Timestamp: startFloat + (hour * 6),
 						},
@@ -341,7 +354,7 @@ func TestBuildPVMap(t *testing.T) {
 	for name, testCase := range testCases {
 		t.Run(name, func(t *testing.T) {
 			pvMap := make(map[pvKey]*pv)
-			buildPVMap(testCase.resolution, pvMap, testCase.resultsPVCostPerGiBHour, testCase.resultsActiveMinutes)
+			buildPVMap(testCase.resolution, pvMap, testCase.resultsPVCostPerGiBHour, testCase.resultsActiveMinutes, window)
 			if len(pvMap) != len(testCase.expected) {
 				t.Errorf("pv map does not have the expected length %d : %d", len(pvMap), len(testCase.expected))
 			}
@@ -352,7 +365,7 @@ func TestBuildPVMap(t *testing.T) {
 					t.Errorf("pv map is missing key %s", thisPVKey)
 				}
 				if !actualPV.equal(expectedPV) {
-					t.Errorf("pv does not match with key %s", thisPVKey)
+					t.Errorf("pv does not match with key %s: %s != %s", thisPVKey, kubecost.NewClosedWindow(actualPV.Start, actualPV.End), kubecost.NewClosedWindow(expectedPV.Start, expectedPV.End))
 				}
 			}
 		})
@@ -455,6 +468,9 @@ func TestCalculateStartAndEnd(t *testing.T) {
 			expectedEnd:   windowStart.Add(time.Hour),
 			result: &prom.QueryResult{
 				Values: []*util.Vector{
+					{
+						Timestamp: startFloat,
+					},
 					{
 						Timestamp: startFloat + (minute * 60),
 					},
@@ -467,6 +483,9 @@ func TestCalculateStartAndEnd(t *testing.T) {
 			expectedEnd:   windowStart.Add(time.Hour),
 			result: &prom.QueryResult{
 				Values: []*util.Vector{
+					{
+						Timestamp: startFloat,
+					},
 					{
 						Timestamp: startFloat + (minute * 30),
 					},
@@ -478,8 +497,8 @@ func TestCalculateStartAndEnd(t *testing.T) {
 		},
 		"15 minute resolution, 45 minute window": {
 			resolution:    time.Minute * 15,
-			expectedStart: windowStart.Add(time.Minute * -15),
-			expectedEnd:   windowStart.Add(time.Minute * 30),
+			expectedStart: windowStart,
+			expectedEnd:   windowStart.Add(time.Minute * 45),
 			result: &prom.QueryResult{
 				Values: []*util.Vector{
 					{
@@ -491,6 +510,60 @@ func TestCalculateStartAndEnd(t *testing.T) {
 					{
 						Timestamp: startFloat + (minute * 30),
 					},
+					{
+						Timestamp: startFloat + (minute * 45),
+					},
+				},
+			},
+		},
+		"1 minute resolution, 5 minute window": {
+			resolution:    time.Minute,
+			expectedStart: windowStart.Add(time.Minute * 15),
+			expectedEnd:   windowStart.Add(time.Minute * 20),
+			result: &prom.QueryResult{
+				Values: []*util.Vector{
+					{
+						Timestamp: startFloat + (minute * 15),
+					},
+					{
+						Timestamp: startFloat + (minute * 16),
+					},
+					{
+						Timestamp: startFloat + (minute * 17),
+					},
+					{
+						Timestamp: startFloat + (minute * 18),
+					},
+					{
+						Timestamp: startFloat + (minute * 19),
+					},
+					{
+						Timestamp: startFloat + (minute * 20),
+					},
+				},
+			},
+		},
+		"1 minute resolution, 1 minute window": {
+			resolution:    time.Minute,
+			expectedStart: windowStart.Add(time.Minute * 14).Add(time.Second * 30),
+			expectedEnd:   windowStart.Add(time.Minute * 15).Add(time.Second * 30),
+			result: &prom.QueryResult{
+				Values: []*util.Vector{
+					{
+						Timestamp: startFloat + (minute * 15),
+					},
+				},
+			},
+		},
+		"1 minute resolution, 1 minute window, at window start": {
+			resolution:    time.Minute,
+			expectedStart: windowStart,
+			expectedEnd:   windowStart.Add(time.Second * 30),
+			result: &prom.QueryResult{
+				Values: []*util.Vector{
+					{
+						Timestamp: startFloat,
+					},
 				},
 			},
 		},
@@ -498,12 +571,12 @@ func TestCalculateStartAndEnd(t *testing.T) {
 
 	for name, testCase := range testCases {
 		t.Run(name, func(t *testing.T) {
-			start, end := calculateStartAndEnd(testCase.result, testCase.resolution)
+			start, end := calculateStartAndEnd(testCase.result, testCase.resolution, window)
 			if !start.Equal(testCase.expectedStart) {
-				t.Errorf("start to not match expected %v : %v", start, testCase.expectedStart)
+				t.Errorf("start does not match: expected %v; got %v", testCase.expectedStart, start)
 			}
 			if !end.Equal(testCase.expectedEnd) {
-				t.Errorf("end to not match expected %v : %v", end, testCase.expectedEnd)
+				t.Errorf("end does not match: expected %v; got %v", testCase.expectedEnd, end)
 			}
 		})
 	}

pkg/costmodel/allocation_types.go

@@ -211,4 +211,6 @@ type lbCost struct {
 	TotalCost float64
 	Start     time.Time
 	End       time.Time
+	Private   bool
+	Ip        string
 }

pkg/costmodel/assets.go

@@ -84,7 +84,7 @@ func (cm *CostModel) ComputeAssets(start, end time.Time) (*kubecost.AssetSet, er
 			e = end
 		}
 
-		loadBalancer := kubecost.NewLoadBalancer(lb.Name, lb.Cluster, lb.ProviderID, s, e, kubecost.NewWindow(&start, &end))
+		loadBalancer := kubecost.NewLoadBalancer(lb.Name, lb.Cluster, lb.ProviderID, s, e, kubecost.NewWindow(&start, &end), lb.Private, lb.Ip)
 		cm.PropertiesFromCluster(loadBalancer.Properties)
 		loadBalancer.Cost = lb.Cost
 		assetSet.Insert(loadBalancer, nil)

pkg/costmodel/cluster.go

@@ -2,6 +2,7 @@ package costmodel
 
 import (
 	"fmt"
+	"net"
 	"strconv"
 	"time"
 
@@ -142,12 +143,6 @@ type DiskIdentifier struct {
 }
 
 func ClusterDisks(client prometheus.Client, provider models.Provider, start, end time.Time) (map[DiskIdentifier]*Disk, error) {
-	// Query for the duration between start and end
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		return nil, fmt.Errorf("illegal duration value for %s", kubecost.NewClosedWindow(start, end))
-	}
-
 	// Start from the time "end", querying backwards
 	t := end
 
@@ -162,6 +157,10 @@ func ClusterDisks(client prometheus.Client, provider models.Provider, start, end
 		log.DedupedWarningf(3, "ClusterDisks(): Configured ETL resolution (%d seconds) is below the 60 seconds threshold. Overriding with 1 minute.", int(resolution.Seconds()))
 	}
 
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		return nil, fmt.Errorf("illegal duration value for %s", kubecost.NewClosedWindow(start, end))
+	}
 	// hourlyToCumulative is a scaling factor that, when multiplied by an hourly
 	// value, converts it to a cumulative value; i.e.
 	// [$/hr] * [min/res]*[hr/min] = [$/res]
@@ -255,7 +254,7 @@ func ClusterDisks(client prometheus.Client, provider models.Provider, start, end
 		diskMap[key].ClaimNamespace = claimNamespace
 	}
 
-	pvCosts(diskMap, resolution, resActiveMins, resPVSize, resPVCost, resPVUsedAvg, resPVUsedMax, resPVCInfo, provider)
+	pvCosts(diskMap, resolution, resActiveMins, resPVSize, resPVCost, resPVUsedAvg, resPVUsedMax, resPVCInfo, provider, kubecost.NewClosedWindow(start, end))
 
 	for _, result := range resLocalStorageCost {
 		cluster, err := result.GetString(env.GetPromClusterLabel())
@@ -548,12 +547,6 @@ func costTimesMinute(activeDataMap map[NodeIdentifier]activeData, costMap map[No
 }
 
 func ClusterNodes(cp models.Provider, client prometheus.Client, start, end time.Time) (map[NodeIdentifier]*Node, error) {
-	// Query for the duration between start and end
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		return nil, fmt.Errorf("illegal duration value for %s", kubecost.NewClosedWindow(start, end))
-	}
-
 	// Start from the time "end", querying backwards
 	t := end
 
@@ -568,14 +561,19 @@ func ClusterNodes(cp models.Provider, client prometheus.Client, start, end time.
 		log.DedupedWarningf(3, "ClusterNodes(): Configured ETL resolution (%d seconds) is below the 60 seconds threshold. Overriding with 1 minute.", int(resolution.Seconds()))
 	}
 
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		return nil, fmt.Errorf("illegal duration value for %s", kubecost.NewClosedWindow(start, end))
+	}
+
 	requiredCtx := prom.NewNamedContext(client, prom.ClusterContextName)
 	optionalCtx := prom.NewNamedContext(client, prom.ClusterOptionalContextName)
 
 	queryNodeCPUHourlyCost := fmt.Sprintf(`avg(avg_over_time(node_cpu_hourly_cost{%s}[%s])) by (%s, node, instance_type, provider_id)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-  queryNodeCPUCoresCapacity := fmt.Sprintf(`avg(avg_over_time(kube_node_status_capacity_cpu_cores{%s}[%s])) by (%s, node)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryNodeCPUCoresCapacity := fmt.Sprintf(`avg(avg_over_time(kube_node_status_capacity_cpu_cores{%s}[%s])) by (%s, node)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	queryNodeCPUCoresAllocatable := fmt.Sprintf(`avg(avg_over_time(kube_node_status_allocatable_cpu_cores{%s}[%s])) by (%s, node)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	queryNodeRAMHourlyCost := fmt.Sprintf(`avg(avg_over_time(node_ram_hourly_cost{%s}[%s])) by (%s, node, instance_type, provider_id) / 1024 / 1024 / 1024`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
-  queryNodeRAMBytesCapacity := fmt.Sprintf(`avg(avg_over_time(kube_node_status_capacity_memory_bytes{%s}[%s])) by (%s, node)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryNodeRAMBytesCapacity := fmt.Sprintf(`avg(avg_over_time(kube_node_status_capacity_memory_bytes{%s}[%s])) by (%s, node)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	queryNodeRAMBytesAllocatable := fmt.Sprintf(`avg(avg_over_time(kube_node_status_allocatable_memory_bytes{%s}[%s])) by (%s, node)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	queryNodeGPUCount := fmt.Sprintf(`avg(avg_over_time(node_gpu_count{%s}[%s])) by (%s, node, provider_id)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	queryNodeGPUHourlyCost := fmt.Sprintf(`avg(avg_over_time(node_gpu_hourly_cost{%s}[%s])) by (%s, node, instance_type, provider_id)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
@@ -632,7 +630,7 @@ func ClusterNodes(cp models.Provider, client prometheus.Client, start, end time.
 		return nil, requiredCtx.ErrorCollection()
 	}
 
-	activeDataMap := buildActiveDataMap(resActiveMins, resolution)
+	activeDataMap := buildActiveDataMap(resActiveMins, resolution, kubecost.NewClosedWindow(start, end))
 
 	gpuCountMap := buildGPUCountMap(resNodeGPUCount)
 	preemptibleMap := buildPreemptibleMap(resIsSpot)
@@ -717,14 +715,11 @@ type LoadBalancer struct {
 	Start      time.Time
 	End        time.Time
 	Minutes    float64
+	Private    bool
+	Ip         string
 }
 
 func ClusterLoadBalancers(client prometheus.Client, start, end time.Time) (map[LoadBalancerIdentifier]*LoadBalancer, error) {
-	// Query for the duration between start and end
-	durStr := timeutil.DurationString(end.Sub(start))
-	if durStr == "" {
-		return nil, fmt.Errorf("illegal duration value for %s", kubecost.NewClosedWindow(start, end))
-	}
 
 	// Start from the time "end", querying backwards
 	t := end
@@ -740,6 +735,12 @@ func ClusterLoadBalancers(client prometheus.Client, start, end time.Time) (map[L
 		log.DedupedWarningf(3, "ClusterLoadBalancers(): Configured ETL resolution (%d seconds) is below the 60 seconds threshold. Overriding with 1 minute.", int(resolution.Seconds()))
 	}
 
+	// Query for the duration between start and end
+	durStr := timeutil.DurationString(end.Sub(start))
+	if durStr == "" {
+		return nil, fmt.Errorf("illegal duration value for %s", kubecost.NewClosedWindow(start, end))
+	}
+
 	ctx := prom.NewNamedContext(client, prom.ClusterContextName)
 
 	queryLBCost := fmt.Sprintf(`avg(avg_over_time(kubecost_load_balancer_cost{%s}[%s])) by (namespace, service_name, %s, ingress_ip)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
@@ -829,6 +830,12 @@ func ClusterLoadBalancers(client prometheus.Client, start, end time.Time) (map[L
 			continue
 		}
 
+		providerID, err := result.GetString("ingress_ip")
+		if err != nil {
+			log.DedupedWarningf(5, "ClusterLoadBalancers: LB cost data missing ingress_ip")
+			// only update asset cost when an actual IP was returned
+			continue
+		}
 		key := LoadBalancerIdentifier{
 			Cluster:   cluster,
 			Namespace: namespace,
@@ -838,16 +845,34 @@ func ClusterLoadBalancers(client prometheus.Client, start, end time.Time) (map[L
 		// Apply cost as price-per-hour * hours
 		if lb, ok := loadBalancerMap[key]; ok {
 			lbPricePerHr := result.Values[0].Value
-			hrs := lb.Minutes / 60.0
+
+			// interpolate any missing data
+			resultMins := lb.Minutes
+			scaleFactor := (resultMins + resolution.Minutes()) / resultMins
+
+			hrs := (lb.Minutes * scaleFactor) / 60.0
 			lb.Cost += lbPricePerHr * hrs
+
+			if lb.Ip != "" && lb.Ip != providerID {
+				log.DedupedWarningf(5, "ClusterLoadBalancers: multiple IPs per load balancer not supported, using most recent IP")
+			}
+			lb.Ip = providerID
+
+			lb.Private = privateIPCheck(providerID)
 		} else {
-			log.DedupedWarningf(20, "ClusterLoadBalancers: found minutes for key that does not exist: %s", key)
+			log.DedupedWarningf(20, "ClusterLoadBalancers: found minutes for key that does not exist: %v", key)
 		}
 	}
 
 	return loadBalancerMap, nil
 }
 
+// Check if an ip is private.
+func privateIPCheck(ip string) bool {
+	ipAddress := net.ParseIP(ip)
+	return ipAddress.IsPrivate()
+}
+
 // ComputeClusterCosts gives the cumulative and monthly-rate cluster costs over a window of time for all clusters.
 func (a *Accesses) ComputeClusterCosts(client prometheus.Client, provider models.Provider, window, offset time.Duration, withBreakdown bool) (map[string]*ClusterCosts, error) {
 	if window < 10*time.Minute {
@@ -859,8 +884,6 @@ func (a *Accesses) ComputeClusterCosts(client prometheus.Client, provider models
 
 	mins := end.Sub(start).Minutes()
 
-	windowStr := timeutil.DurationString(window)
-
 	// minsPerResolution determines accuracy and resource use for the following
 	// queries. Smaller values (higher resolution) result in better accuracy,
 	// but more expensive queries, and vice-a-versa.
@@ -872,6 +895,8 @@ func (a *Accesses) ComputeClusterCosts(client prometheus.Client, provider models
 		log.DedupedWarningf(3, "ComputeClusterCosts(): Configured ETL resolution (%d seconds) is below the 60 seconds threshold. Overriding with 1 minute.", int(resolution.Seconds()))
 	}
 
+	windowStr := timeutil.DurationString(window)
+
 	// hourlyToCumulative is a scaling factor that, when multiplied by an hourly
 	// value, converts it to a cumulative value; i.e.
 	// [$/hr] * [min/res]*[hr/min] = [$/res]
@@ -1314,7 +1339,7 @@ func ClusterCostsOverTime(cli prometheus.Client, provider models.Provider, start
 	}, nil
 }
 
-func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActiveMins, resPVSize, resPVCost, resPVUsedAvg, resPVUsedMax, resPVCInfo []*prom.QueryResult, cp models.Provider) {
+func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActiveMins, resPVSize, resPVCost, resPVUsedAvg, resPVUsedMax, resPVCInfo []*prom.QueryResult, cp models.Provider, window kubecost.Window) {
 	for _, result := range resActiveMins {
 		cluster, err := result.GetString(env.GetPromClusterLabel())
 		if err != nil {
@@ -1339,11 +1364,9 @@ func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActi
 				Breakdown: &ClusterCostsBreakdown{},
 			}
 		}
-		s := time.Unix(int64(result.Values[0].Timestamp), 0)
-		e := time.Unix(int64(result.Values[len(result.Values)-1].Timestamp), 0)
-		mins := e.Sub(s).Minutes()
 
-		// TODO niko/assets if mins >= threshold, interpolate for missing data?
+		s, e := calculateStartAndEnd(result, resolution, window)
+		mins := e.Sub(s).Minutes()
 
 		diskMap[key].End = e
 		diskMap[key].Start = s
@@ -1418,6 +1441,7 @@ func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActi
 				Breakdown: &ClusterCostsBreakdown{},
 			}
 		}
+
 		diskMap[key].Cost = cost * (diskMap[key].Bytes / 1024 / 1024 / 1024) * (diskMap[key].Minutes / 60)
 		providerID, _ := result.GetString("provider_id") // just put the providerID set up here, it's the simplest query.
 		if providerID != "" {

pkg/costmodel/cluster_helpers.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/cloud/provider"
+	"github.com/opencost/opencost/pkg/kubecost"
 
 	"github.com/opencost/opencost/pkg/env"
 	"github.com/opencost/opencost/pkg/log"
@@ -527,7 +528,7 @@ type activeData struct {
 	minutes float64
 }
 
-func buildActiveDataMap(resActiveMins []*prom.QueryResult, resolution time.Duration) map[NodeIdentifier]activeData {
+func buildActiveDataMap(resActiveMins []*prom.QueryResult, resolution time.Duration, window kubecost.Window) map[NodeIdentifier]activeData {
 
 	m := make(map[NodeIdentifier]activeData)
 
@@ -555,8 +556,7 @@ func buildActiveDataMap(resActiveMins []*prom.QueryResult, resolution time.Durat
 			continue
 		}
 
-		s := time.Unix(int64(result.Values[0].Timestamp), 0)
-		e := time.Unix(int64(result.Values[len(result.Values)-1].Timestamp), 0)
+		s, e := calculateStartAndEnd(result, resolution, window)
 		mins := e.Sub(s).Minutes()
 
 		// TODO niko/assets if mins >= threshold, interpolate for missing data?

pkg/costmodel/cluster_helpers_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/opencost/opencost/pkg/cloud/provider"
 	"github.com/opencost/opencost/pkg/config"
+	"github.com/opencost/opencost/pkg/kubecost"
 	"github.com/opencost/opencost/pkg/prom"
 	"github.com/opencost/opencost/pkg/util"
 
@@ -891,6 +892,12 @@ func TestBuildGPUCostMap(t *testing.T) {
 
 func TestAssetCustompricing(t *testing.T) {
 
+	windowStart := time.Date(2020, time.April, 13, 0, 0, 0, 0, time.UTC)
+	windowEnd := windowStart.Add(time.Hour)
+	window := kubecost.NewClosedWindow(windowStart, windowEnd)
+
+	startTimestamp := float64(windowStart.Unix())
+
 	nodePromResult := []*prom.QueryResult{
 		{
 			Metric: map[string]interface{}{
@@ -901,7 +908,7 @@ func TestAssetCustompricing(t *testing.T) {
 			},
 			Values: []*util.Vector{
 				{
-					Timestamp: 0,
+					Timestamp: startTimestamp,
 					Value:     0.5,
 				},
 			},
@@ -917,7 +924,7 @@ func TestAssetCustompricing(t *testing.T) {
 			},
 			Values: []*util.Vector{
 				{
-					Timestamp: 0,
+					Timestamp: startTimestamp,
 					Value:     1.0,
 				},
 			},
@@ -933,7 +940,7 @@ func TestAssetCustompricing(t *testing.T) {
 			},
 			Values: []*util.Vector{
 				{
-					Timestamp: 0,
+					Timestamp: startTimestamp,
 					Value:     1073741824.0,
 				},
 			},
@@ -949,11 +956,11 @@ func TestAssetCustompricing(t *testing.T) {
 			},
 			Values: []*util.Vector{
 				{
-					Timestamp: 0,
+					Timestamp: startTimestamp,
 					Value:     1.0,
 				},
 				{
-					Timestamp: 3600.0,
+					Timestamp: startTimestamp + (60.0 * 60.0),
 					Value:     1.0,
 				},
 			},
@@ -969,11 +976,11 @@ func TestAssetCustompricing(t *testing.T) {
 			},
 			Values: []*util.Vector{
 				{
-					Timestamp: 0,
+					Timestamp: startTimestamp,
 					Value:     1.0,
 				},
 				{
-					Timestamp: 3600.0,
+					Timestamp: startTimestamp + (60.0 * 60.0),
 					Value:     1.0,
 				},
 			},
@@ -989,11 +996,11 @@ func TestAssetCustompricing(t *testing.T) {
 			},
 			Values: []*util.Vector{
 				{
-					Timestamp: 0,
+					Timestamp: startTimestamp,
 					Value:     1.0,
 				},
 				{
-					Timestamp: 3600.0,
+					Timestamp: startTimestamp + (60.0 * 60.0),
 					Value:     1.0,
 				},
 			},
@@ -1010,7 +1017,7 @@ func TestAssetCustompricing(t *testing.T) {
 			},
 			Values: []*util.Vector{
 				{
-					Timestamp: 0,
+					Timestamp: startTimestamp,
 					Value:     1.0,
 				},
 			},
@@ -1081,7 +1088,7 @@ func TestAssetCustompricing(t *testing.T) {
 			gpuResult := gpuMap[nodeKey]
 
 			diskMap := map[DiskIdentifier]*Disk{}
-			pvCosts(diskMap, time.Hour, pvMinsPromResult, pvSizePromResult, pvCostPromResult, pvAvgUsagePromResult, pvMaxUsagePromResult, pvInfoPromResult, testProvider)
+			pvCosts(diskMap, time.Hour, pvMinsPromResult, pvSizePromResult, pvCostPromResult, pvAvgUsagePromResult, pvMaxUsagePromResult, pvInfoPromResult, testProvider, window)
 
 			diskResult := diskMap[DiskIdentifier{"cluster1", "pvc1"}].Cost
 

pkg/costmodel/costmodel.go

@@ -500,11 +500,9 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, cp costAnalyze
 				// for the units of memory and CPU.
 				ramRequestBytes := container.Resources.Requests.Memory().Value()
 
-				// Because RAM (and CPU) information isn't coming from Prometheus, it won't
-				// have a timestamp associated with it. We need to provide a timestamp,
-				// otherwise the vector op that gets applied to take the max of usage
-				// and request won't work properly and will only take into account
-				// usage.
+				// Because information on container RAM & CPU requests isn't
+				// coming from Prometheus, it won't have a timestamp associated
+				// with it. We need to provide a timestamp.
 				RAMReqV := []*util.Vector{
 					{
 						Value:     float64(ramRequestBytes),
@@ -582,8 +580,25 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, cp costAnalyze
 					ClusterID:       clusterID,
 					ClusterName:     cm.ClusterMap.NameFor(clusterID),
 				}
-				costs.CPUAllocation = getContainerAllocation(costs.CPUReq, costs.CPUUsed, "CPU")
-				costs.RAMAllocation = getContainerAllocation(costs.RAMReq, costs.RAMUsed, "RAM")
+
+				var cpuReq, cpuUse *util.Vector
+				if len(costs.CPUReq) > 0 {
+					cpuReq = costs.CPUReq[0]
+				}
+				if len(costs.CPUUsed) > 0 {
+					cpuUse = costs.CPUUsed[0]
+				}
+				costs.CPUAllocation = getContainerAllocation(cpuReq, cpuUse, "CPU")
+
+				var ramReq, ramUse *util.Vector
+				if len(costs.RAMReq) > 0 {
+					ramReq = costs.RAMReq[0]
+				}
+				if len(costs.RAMUsed) > 0 {
+					ramUse = costs.RAMUsed[0]
+				}
+				costs.RAMAllocation = getContainerAllocation(ramReq, ramUse, "RAM")
+
 				if filterNamespace == "" {
 					containerNameCost[newKey] = costs
 				} else if costs.Namespace == filterNamespace {
@@ -650,8 +665,25 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, cp costAnalyze
 				ClusterID:       c.ClusterID,
 				ClusterName:     cm.ClusterMap.NameFor(c.ClusterID),
 			}
-			costs.CPUAllocation = getContainerAllocation(costs.CPUReq, costs.CPUUsed, "CPU")
-			costs.RAMAllocation = getContainerAllocation(costs.RAMReq, costs.RAMUsed, "RAM")
+
+			var cpuReq, cpuUse *util.Vector
+			if len(costs.CPUReq) > 0 {
+				cpuReq = costs.CPUReq[0]
+			}
+			if len(costs.CPUUsed) > 0 {
+				cpuUse = costs.CPUUsed[0]
+			}
+			costs.CPUAllocation = getContainerAllocation(cpuReq, cpuUse, "CPU")
+
+			var ramReq, ramUse *util.Vector
+			if len(costs.RAMReq) > 0 {
+				ramReq = costs.RAMReq[0]
+			}
+			if len(costs.RAMUsed) > 0 {
+				ramUse = costs.RAMUsed[0]
+			}
+			costs.RAMAllocation = getContainerAllocation(ramReq, ramUse, "RAM")
+
 			if filterNamespace == "" {
 				containerNameCost[key] = costs
 				missingContainers[key] = costs
@@ -828,32 +860,62 @@ func findDeletedNodeInfo(cli prometheusClient.Client, missingNodes map[string]*c
 	return nil
 }
 
-func getContainerAllocation(req []*util.Vector, used []*util.Vector, allocationType string) []*util.Vector {
-	// The result of the normalize operation will be a new []*util.Vector to replace the requests
-	allocationOp := func(r *util.Vector, x *float64, y *float64) bool {
-		if x != nil && y != nil {
-			x1 := *x
-			if math.IsNaN(x1) {
-				log.Warnf("NaN value found during %s allocation calculation for requests.", allocationType)
-				x1 = 0.0
-			}
-			y1 := *y
-			if math.IsNaN(y1) {
-				log.Warnf("NaN value found during %s allocation calculation for used.", allocationType)
-				y1 = 0.0
-			}
-
-			r.Value = math.Max(x1, y1)
-		} else if x != nil {
-			r.Value = *x
-		} else if y != nil {
-			r.Value = *y
+// getContainerAllocation takes the max between request and usage. This function
+// returns a slice containing a single element describing the container's
+// allocation.
+//
+// Additionally, the timestamp of the allocation will be the highest value
+// timestamp between the two vectors. This mitigates situations where
+// Timestamp=0. This should have no effect on the metrics emitted by the
+// CostModelMetricsEmitter
+func getContainerAllocation(req *util.Vector, used *util.Vector, allocationType string) []*util.Vector {
+	var result []*util.Vector
+
+	if req != nil && used != nil {
+		x1 := req.Value
+		if math.IsNaN(x1) {
+			log.Warnf("NaN value found during %s allocation calculation for requests.", allocationType)
+			x1 = 0.0
+		}
+		y1 := used.Value
+		if math.IsNaN(y1) {
+			log.Warnf("NaN value found during %s allocation calculation for used.", allocationType)
+			y1 = 0.0
+		}
+		result = []*util.Vector{
+			{
+				Value:     math.Max(x1, y1),
+				Timestamp: math.Max(req.Timestamp, used.Timestamp),
+			},
+		}
+		if result[0].Value == 0 && result[0].Timestamp == 0 {
+			log.Warnf("No request or usage data found during %s allocation calculation. Setting allocation to 0.", allocationType)
+		}
+	} else if req != nil {
+		result = []*util.Vector{
+			{
+				Value:     req.Value,
+				Timestamp: req.Timestamp,
+			},
+		}
+	} else if used != nil {
+		result = []*util.Vector{
+			{
+				Value:     used.Value,
+				Timestamp: used.Timestamp,
+			},
+		}
+	} else {
+		log.Warnf("No request or usage data found during %s allocation calculation. Setting allocation to 0.", allocationType)
+		result = []*util.Vector{
+			{
+				Value:     0,
+				Timestamp: float64(time.Now().UTC().Unix()),
+			},
 		}
-
-		return true
 	}
 
-	return util.ApplyVectorOp(req, used, allocationOp)
+	return result
 }
 
 func addPVData(cache clustercache.ClusterCache, pvClaimMapping map[string]*PersistentVolumeClaimData, cloud costAnalyzerCloud.Provider) error {
@@ -973,7 +1035,7 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 
 		pmd.TotalNodes++
 
-		cnode, err := cp.NodePricing(cp.GetKey(nodeLabels, n))
+		cnode, _, err := cp.NodePricing(cp.GetKey(nodeLabels, n))
 		if err != nil {
 			log.Infof("Error getting node pricing. Error: %s", err.Error())
 			if cnode != nil {
@@ -2299,17 +2361,19 @@ func measureTimeAsync(start time.Time, threshold time.Duration, name string, ch
 	}
 }
 
-func (cm *CostModel) QueryAllocation(window kubecost.Window, resolution, step time.Duration, aggregate []string, includeIdle, idleByNode, includeProportionalAssetResourceCosts, includeAggregatedMetadata bool) (*kubecost.AllocationSetRange, error) {
+func (cm *CostModel) QueryAllocation(window kubecost.Window, resolution, step time.Duration, aggregate []string, includeIdle, idleByNode, includeProportionalAssetResourceCosts, includeAggregatedMetadata bool, accumulateBy kubecost.AccumulateOption) (*kubecost.AllocationSetRange, error) {
 	// Validate window is legal
 	if window.IsOpen() || window.IsNegative() {
 		return nil, fmt.Errorf("illegal window: %s", window)
 	}
 
+	var totalsStore kubecost.TotalsStore
 	// Idle is required for proportional asset costs
 	if includeProportionalAssetResourceCosts {
 		if !includeIdle {
 			return nil, errors.New("bad request - includeIdle must be set true if includeProportionalAssetResourceCosts is true")
 		}
+		totalsStore = kubecost.NewMemoryTotalsStore()
 	}
 
 	// Begin with empty response
@@ -2319,6 +2383,7 @@ func (cm *CostModel) QueryAllocation(window kubecost.Window, resolution, step ti
 	// appending each to the response.
 	stepStart := *window.Start()
 	stepEnd := stepStart.Add(step)
+	var isAzure bool
 	for window.End().After(stepStart) {
 		allocSet, err := cm.ComputeAllocation(stepStart, stepEnd, resolution)
 		if err != nil {
@@ -2331,6 +2396,25 @@ func (cm *CostModel) QueryAllocation(window kubecost.Window, resolution, step ti
 				return nil, fmt.Errorf("error computing assets for %s: %w", kubecost.NewClosedWindow(stepStart, stepEnd), err)
 			}
 
+			if includeProportionalAssetResourceCosts {
+
+				// AKS is a special case - there can be a maximum of 2
+				// load balancers (1 public and 1 private) in an AKS cluster
+				// therefore, when calculating PARCs for load balancers,
+				// we must know if this is an AKS cluster
+				for _, node := range assetSet.Nodes {
+					if _, found := node.Labels["label_kubernetes_azure_com_cluster"]; found {
+						isAzure = true
+						break
+					}
+				}
+
+				_, err := kubecost.UpdateAssetTotalsStore(totalsStore, assetSet)
+				if err != nil {
+					log.Errorf("ETL: error updating asset resource totals for %s: %s", assetSet.Window, err)
+				}
+			}
+
 			idleSet, err := computeIdleAllocations(allocSet, assetSet, true)
 			if err != nil {
 				return nil, fmt.Errorf("error computing idle allocations for %s: %w", kubecost.NewClosedWindow(stepStart, stepEnd), err)
@@ -2360,6 +2444,110 @@ func (cm *CostModel) QueryAllocation(window kubecost.Window, resolution, step ti
 		return nil, fmt.Errorf("error aggregating for %s: %w", window, err)
 	}
 
+	// Accumulate, if requested
+	if accumulateBy != kubecost.AccumulateOptionNone {
+		asr, err = asr.Accumulate(accumulateBy)
+		if err != nil {
+			log.Errorf("error accumulating by %v: %s", accumulateBy, err)
+			return nil, fmt.Errorf("error accumulating by %v: %s", accumulateBy, err)
+		}
+
+		// when accumulating and returning PARCs, we need the totals for the
+		// accumulated windows to accurately compute a fraction
+		if includeProportionalAssetResourceCosts {
+			assetSet, err := cm.ComputeAssets(*asr.Window().Start(), *asr.Window().End())
+			if err != nil {
+				return nil, fmt.Errorf("error computing assets for %s: %w", kubecost.NewClosedWindow(*asr.Window().Start(), *asr.Window().End()), err)
+			}
+
+			_, err = kubecost.UpdateAssetTotalsStore(totalsStore, assetSet)
+			if err != nil {
+				log.Errorf("ETL: error updating asset resource totals for %s: %s", kubecost.NewClosedWindow(*asr.Window().Start(), *asr.Window().End()), err)
+			}
+
+		}
+	}
+
+	if includeProportionalAssetResourceCosts {
+
+		for _, as := range asr.Allocations {
+			totalStoreByNode, ok := totalsStore.GetAssetTotalsByNode(as.Start(), as.End())
+			if !ok {
+				log.Errorf("unable to locate allocation totals for node for window %v - %v", as.Start(), as.End())
+				return nil, fmt.Errorf("unable to locate allocation totals for node for window %v - %v", as.Start(), as.End())
+			}
+
+			totalStoreByCluster, ok := totalsStore.GetAssetTotalsByCluster(as.Start(), as.End())
+			if !ok {
+				log.Errorf("unable to locate allocation totals for cluster for window %v - %v", as.Start(), as.End())
+				return nil, fmt.Errorf("unable to locate allocation totals for cluster for window %v - %v", as.Start(), as.End())
+			}
+
+			var totalPublicLbCost, totalPrivateLbCost float64
+			if isAzure {
+				// loop through all assetTotals, adding all load balancer costs by public and private
+				for _, tot := range totalStoreByNode {
+					if tot.PrivateLoadBalancer {
+						totalPrivateLbCost += tot.LoadBalancerCost
+					} else {
+						totalPublicLbCost += tot.LoadBalancerCost
+					}
+				}
+			}
+
+			// loop through each allocation set, using total cost from totals store
+			for _, alloc := range as.Allocations {
+				for rawKey, parc := range alloc.ProportionalAssetResourceCosts {
+
+					key := strings.TrimSuffix(strings.ReplaceAll(rawKey, ",", "/"), "/")
+					// for each parc , check the totals store for each
+					// on a totals hit, set the corresponding total and calculate percentage
+					var totals *kubecost.AssetTotals
+					if totalsLoc, found := totalStoreByCluster[key]; found {
+						totals = totalsLoc
+					}
+
+					if totalsLoc, found := totalStoreByNode[key]; found {
+						totals = totalsLoc
+					}
+
+					if totals == nil {
+						log.Errorf("unable to locate asset totals for allocation %s", key)
+						return nil, fmt.Errorf("unable to locate allocation totals for allocation")
+
+					}
+
+					parc.CPUTotalCost = totals.CPUCost
+					parc.GPUTotalCost = totals.GPUCost
+					parc.RAMTotalCost = totals.RAMCost
+					parc.PVTotalCost = totals.PersistentVolumeCost
+					if !isAzure {
+						parc.LoadBalancerTotalCost = totals.LoadBalancerCost
+					} else if len(alloc.LoadBalancers) > 0 {
+						// Azure is a special case - use computed totals above
+						// use the lbAllocations in the object to determine if
+						// this PARC is a public or private load balancer
+						// then set the total accordingly
+						// AKS only has 1 public and 1 private load balancer
+
+						lbAlloc, found := alloc.LoadBalancers[key]
+						if found {
+							if lbAlloc.Private {
+								parc.LoadBalancerTotalCost = totalPrivateLbCost
+							} else {
+								parc.LoadBalancerTotalCost = totalPublicLbCost
+							}
+						}
+					}
+
+					kubecost.ComputePercentages(&parc)
+					alloc.ProportionalAssetResourceCosts[rawKey] = parc
+				}
+			}
+
+		}
+	}
+
 	return asr, nil
 }
 
@@ -2373,10 +2561,10 @@ func computeIdleAllocations(allocSet *kubecost.AllocationSet, assetSet *kubecost
 
 	if idleByNode {
 		allocTotals = kubecost.ComputeAllocationTotals(allocSet, kubecost.AllocationNodeProp)
-		assetTotals = kubecost.ComputeAssetTotals(assetSet, kubecost.AssetNodeProp)
+		assetTotals = kubecost.ComputeAssetTotals(assetSet, true)
 	} else {
 		allocTotals = kubecost.ComputeAllocationTotals(allocSet, kubecost.AllocationClusterProp)
-		assetTotals = kubecost.ComputeAssetTotals(assetSet, kubecost.AssetClusterProp)
+		assetTotals = kubecost.ComputeAssetTotals(assetSet, false)
 	}
 
 	start, end := *allocSet.Window.Start(), *allocSet.Window.End()

pkg/costmodel/costmodel_test.go

@@ -2,6 +2,8 @@ package costmodel
 
 import (
 	"testing"
+
+	"github.com/opencost/opencost/pkg/util"
 )
 
 func Test_CostData_GetController_CronJob(t *testing.T) {
@@ -65,3 +67,151 @@ func Test_CostData_GetController_CronJob(t *testing.T) {
 		})
 	}
 }
+
+func Test_getContainerAllocation(t *testing.T) {
+	cases := []struct {
+		name string
+		cd   CostData
+
+		expectedCPUAllocation []*util.Vector
+		expectedRAMAllocation []*util.Vector
+	}{
+		{
+			name: "Requests greater than usage",
+			cd: CostData{
+				CPUReq:  []*util.Vector{{Value: 1.0, Timestamp: 1686929350}},
+				CPUUsed: []*util.Vector{{Value: .01, Timestamp: 1686929350}},
+				RAMReq:  []*util.Vector{{Value: 10000000, Timestamp: 1686929350}},
+				RAMUsed: []*util.Vector{{Value: 5500000, Timestamp: 1686929350}},
+			},
+
+			expectedCPUAllocation: []*util.Vector{{Value: 1.0, Timestamp: 1686929350}},
+			expectedRAMAllocation: []*util.Vector{{Value: 10000000, Timestamp: 1686929350}},
+		},
+		{
+			name: "Requests less than usage",
+			cd: CostData{
+				CPUReq:  []*util.Vector{{Value: 1.0, Timestamp: 1686929350}},
+				CPUUsed: []*util.Vector{{Value: 2.2, Timestamp: 1686929350}},
+				RAMReq:  []*util.Vector{{Value: 10000000, Timestamp: 1686929350}},
+				RAMUsed: []*util.Vector{{Value: 75000000, Timestamp: 1686929350}},
+			},
+
+			expectedCPUAllocation: []*util.Vector{{Value: 2.2, Timestamp: 1686929350}},
+			expectedRAMAllocation: []*util.Vector{{Value: 75000000, Timestamp: 1686929350}},
+		},
+		{
+			// Expected behavior for getContainerAllocation is to always use the
+			// highest Timestamp value. The significance of 10 seconds comes
+			// from the current default in ApplyVectorOp() in
+			// pkg/util/vector.go.
+			name: "Mismatched timestamps less than 10 seconds apart",
+			cd: CostData{
+				CPUReq:  []*util.Vector{{Value: 1.0, Timestamp: 1686929354}},
+				CPUUsed: []*util.Vector{{Value: .01, Timestamp: 1686929350}},
+				RAMReq:  []*util.Vector{{Value: 10000000, Timestamp: 1686929354}},
+				RAMUsed: []*util.Vector{{Value: 5500000, Timestamp: 1686929350}},
+			},
+
+			expectedCPUAllocation: []*util.Vector{{Value: 1.0, Timestamp: 1686929354}},
+			expectedRAMAllocation: []*util.Vector{{Value: 10000000, Timestamp: 1686929354}},
+		},
+		{
+			// Expected behavior for getContainerAllocation is to always use the
+			// hightest Timestamp value. The significance of 10 seconds comes
+			// from the current default in ApplyVectorOp() in
+			// pkg/util/vector.go.
+			name: "Mismatched timestamps greater than 10 seconds apart",
+			cd: CostData{
+				CPUReq:  []*util.Vector{{Value: 1.0, Timestamp: 1686929399}},
+				CPUUsed: []*util.Vector{{Value: .01, Timestamp: 1686929350}},
+				RAMReq:  []*util.Vector{{Value: 10000000, Timestamp: 1686929399}},
+				RAMUsed: []*util.Vector{{Value: 5500000, Timestamp: 1686929350}},
+			},
+
+			expectedCPUAllocation: []*util.Vector{{Value: 1.0, Timestamp: 1686929399}},
+			expectedRAMAllocation: []*util.Vector{{Value: 10000000, Timestamp: 1686929399}},
+		},
+		{
+			name: "Requests has no values",
+			cd: CostData{
+				CPUReq:  []*util.Vector{{Value: 0, Timestamp: 0}},
+				CPUUsed: []*util.Vector{{Value: .01, Timestamp: 1686929350}},
+				RAMReq:  []*util.Vector{{Value: 0, Timestamp: 0}},
+				RAMUsed: []*util.Vector{{Value: 5500000, Timestamp: 1686929350}},
+			},
+
+			expectedCPUAllocation: []*util.Vector{{Value: .01, Timestamp: 1686929350}},
+			expectedRAMAllocation: []*util.Vector{{Value: 5500000, Timestamp: 1686929350}},
+		},
+		{
+			name: "Usage has no values",
+			cd: CostData{
+				CPUReq:  []*util.Vector{{Value: 1.0, Timestamp: 1686929350}},
+				CPUUsed: []*util.Vector{{Value: 0, Timestamp: 0}},
+				RAMReq:  []*util.Vector{{Value: 10000000, Timestamp: 1686929350}},
+				RAMUsed: []*util.Vector{{Value: 0, Timestamp: 0}},
+			},
+
+			expectedCPUAllocation: []*util.Vector{{Value: 1.0, Timestamp: 1686929350}},
+			expectedRAMAllocation: []*util.Vector{{Value: 10000000, Timestamp: 1686929350}},
+		},
+		{
+			// WRN Log should be thrown
+			name: "Both have no values",
+			cd: CostData{
+				CPUReq:  []*util.Vector{{Value: 0, Timestamp: 0}},
+				CPUUsed: []*util.Vector{{Value: 0, Timestamp: 0}},
+				RAMReq:  []*util.Vector{{Value: 0, Timestamp: 0}},
+				RAMUsed: []*util.Vector{{Value: 0, Timestamp: 0}},
+			},
+
+			expectedCPUAllocation: []*util.Vector{{Value: 0, Timestamp: 0}},
+			expectedRAMAllocation: []*util.Vector{{Value: 0, Timestamp: 0}},
+		},
+		{
+			name: "Requests is Nil",
+			cd: CostData{
+				CPUReq:  []*util.Vector{nil},
+				CPUUsed: []*util.Vector{{Value: .01, Timestamp: 1686929350}},
+				RAMReq:  []*util.Vector{nil},
+				RAMUsed: []*util.Vector{{Value: 5500000, Timestamp: 1686929350}},
+			},
+
+			expectedCPUAllocation: []*util.Vector{{Value: .01, Timestamp: 1686929350}},
+			expectedRAMAllocation: []*util.Vector{{Value: 5500000, Timestamp: 1686929350}},
+		},
+		{
+			name: "Usage is nil",
+			cd: CostData{
+				CPUReq:  []*util.Vector{{Value: 1.0, Timestamp: 1686929350}},
+				CPUUsed: []*util.Vector{nil},
+				RAMReq:  []*util.Vector{{Value: 10000000, Timestamp: 1686929350}},
+				RAMUsed: []*util.Vector{nil},
+			},
+
+			expectedCPUAllocation: []*util.Vector{{Value: 1.0, Timestamp: 1686929350}},
+			expectedRAMAllocation: []*util.Vector{{Value: 10000000, Timestamp: 1686929350}},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			cpuAllocation := getContainerAllocation(c.cd.CPUReq[0], c.cd.CPUUsed[0], "CPU")
+			ramAllocation := getContainerAllocation(c.cd.RAMReq[0], c.cd.RAMUsed[0], "RAM")
+
+			if cpuAllocation[0].Value != c.expectedCPUAllocation[0].Value {
+				t.Errorf("CPU Allocation mismatch. Expected Value: %f. Got: %f", cpuAllocation[0].Value, c.expectedCPUAllocation[0].Value)
+			}
+			if cpuAllocation[0].Timestamp != c.expectedCPUAllocation[0].Timestamp {
+				t.Errorf("CPU Allocation mismatch. Expected Timestamp: %f. Got: %f", cpuAllocation[0].Timestamp, c.expectedCPUAllocation[0].Timestamp)
+			}
+			if ramAllocation[0].Value != c.expectedRAMAllocation[0].Value {
+				t.Errorf("RAM Allocation mismatch. Expected Value: %f. Got: %f", ramAllocation[0].Value, c.expectedRAMAllocation[0].Value)
+			}
+			if ramAllocation[0].Timestamp != c.expectedRAMAllocation[0].Timestamp {
+				t.Errorf("RAM Allocation mismatch. Expected Timestamp: %f. Got: %f", ramAllocation[0].Timestamp, c.expectedRAMAllocation[0].Timestamp)
+			}
+		})
+	}
+}

pkg/costmodel/intervals.go

@@ -1,6 +1,7 @@
 package costmodel
 
 import (
+	"fmt"
 	"sort"
 	"time"
 
@@ -79,12 +80,18 @@ func getIntervalPointsFromWindows(windows map[podKey]kubecost.Window) IntervalPo
 // getPVCCostCoefficients gets a coefficient which represents the scale
 // factor that each PVC in a pvcIntervalMap and corresponding slice of
 // IntervalPoints intervals uses to calculate a cost for that PVC's PV.
-func getPVCCostCoefficients(intervals IntervalPoints, thisPVC *pvc) map[podKey][]CoefficientComponent {
+func getPVCCostCoefficients(intervals IntervalPoints, thisPVC *pvc) (map[podKey][]CoefficientComponent, error) {
 	// pvcCostCoefficientMap has a format such that the individual coefficient
 	// components are preserved for testing purposes.
 	pvcCostCoefficientMap := make(map[podKey][]CoefficientComponent)
 
 	pvcWindow := kubecost.NewWindow(&thisPVC.Start, &thisPVC.End)
+	pvcWindowDurationMinutes := pvcWindow.Duration().Minutes()
+	if pvcWindowDurationMinutes <= 0.0 {
+		// Protect against Inf and NaN issues that would be caused by dividing
+		// by zero later on.
+		return nil, fmt.Errorf("detected PVC with window of zero duration: %s/%s/%s", thisPVC.Cluster, thisPVC.Namespace, thisPVC.Name)
+	}
 
 	unmountedKey := getUnmountedPodKey(thisPVC.Cluster)
 
@@ -97,22 +104,25 @@ func getPVCCostCoefficients(intervals IntervalPoints, thisPVC *pvc) map[podKey][
 	for _, point := range intervals {
 		// If the current point happens at a later time than the previous point
 		if !point.Time.Equal(currentTime) {
+			// If there are active keys, attribute one unit of proportion to
+			// each active key.
 			for key := range activeKeys {
 				pvcCostCoefficientMap[key] = append(
 					pvcCostCoefficientMap[key],
 					CoefficientComponent{
-						Time:       point.Time.Sub(currentTime).Minutes() / pvcWindow.Duration().Minutes(),
+						Time:       point.Time.Sub(currentTime).Minutes() / pvcWindowDurationMinutes,
 						Proportion: 1.0 / float64(len(activeKeys)),
 					},
 				)
 
 			}
+
 			// If there are no active keys attribute all cost to the unmounted pv
 			if len(activeKeys) == 0 {
 				pvcCostCoefficientMap[unmountedKey] = append(
 					pvcCostCoefficientMap[unmountedKey],
 					CoefficientComponent{
-						Time:       point.Time.Sub(currentTime).Minutes() / pvcWindow.Duration().Minutes(),
+						Time:       point.Time.Sub(currentTime).Minutes() / pvcWindowDurationMinutes,
 						Proportion: 1.0,
 					},
 				)
@@ -142,8 +152,7 @@ func getPVCCostCoefficients(intervals IntervalPoints, thisPVC *pvc) map[podKey][
 			},
 		)
 	}
-
-	return pvcCostCoefficientMap
+	return pvcCostCoefficientMap, nil
 }
 
 // getCoefficientFromComponents takes the components of a PVC-pod PV cost coefficient

pkg/costmodel/intervals_test.go

@@ -1,6 +1,7 @@
 package costmodel
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
 	"time"
@@ -150,26 +151,47 @@ func TestGetIntervalPointsFromWindows(t *testing.T) {
 }
 
 func TestGetPVCCostCoefficients(t *testing.T) {
+	pod1Key := newPodKey("cluster1", "namespace1", "pod1")
+	pod2Key := newPodKey("cluster1", "namespace1", "pod2")
+	pod3Key := newPodKey("cluster1", "namespace1", "pod3")
+	pod4Key := newPodKey("cluster1", "namespace1", "pod4")
+	ummountedPodKey := newPodKey("cluster1", kubecost.UnmountedSuffix, kubecost.UnmountedSuffix)
+
 	pvc1 := &pvc{
-		Bytes:     0,
+		Bytes:     100 * 1024 * 1024 * 1024,
 		Name:      "pvc1",
 		Cluster:   "cluster1",
 		Namespace: "namespace1",
 		Start:     time.Date(2021, 2, 19, 8, 0, 0, 0, time.UTC),
 		End:       time.Date(2021, 2, 19, 9, 0, 0, 0, time.UTC),
 	}
-	pod1Key := newPodKey("cluster1", "namespace1", "pod1")
-	pod2Key := newPodKey("cluster1", "namespace1", "pod2")
-	pod3Key := newPodKey("cluster1", "namespace1", "pod3")
-	pod4Key := newPodKey("cluster1", "namespace1", "pod4")
-	ummountedPodKey := newPodKey("cluster1", kubecost.UnmountedSuffix, kubecost.UnmountedSuffix)
+
+	pvc2 := &pvc{
+		Bytes:     100 * 1024 * 1024 * 1024,
+		Name:      "pvc2",
+		Cluster:   "cluster1",
+		Namespace: "namespace1",
+		Start:     time.Date(2021, 2, 19, 8, 0, 0, 0, time.UTC),
+		End:       time.Date(2021, 2, 19, 9, 0, 0, 0, time.UTC),
+	}
+
+	pvc3 := &pvc{
+		Bytes:     100 * 1024 * 1024 * 1024,
+		Name:      "pvc3",
+		Cluster:   "cluster1",
+		Namespace: "namespace1",
+		Start:     time.Date(2021, 2, 19, 8, 0, 0, 0, time.UTC),
+		End:       time.Date(2021, 2, 19, 8, 0, 0, 0, time.UTC),
+	}
 
 	cases := []struct {
 		name           string
 		pvc            *pvc
 		pvcIntervalMap map[podKey]kubecost.Window
 		intervals      []IntervalPoint
+		resolution     time.Duration
 		expected       map[podKey][]CoefficientComponent
+		expError       error
 	}{
 		{
 			name: "four pods w/ various overlaps",
@@ -184,6 +206,7 @@ func TestGetPVCCostCoefficients(t *testing.T) {
 				NewIntervalPoint(time.Date(2021, 2, 19, 9, 0, 0, 0, time.UTC), "end", pod3Key),
 				NewIntervalPoint(time.Date(2021, 2, 19, 9, 0, 0, 0, time.UTC), "end", pod1Key),
 			},
+			expError: nil,
 			expected: map[podKey][]CoefficientComponent{
 				pod1Key: {
 					{0.5, 0.25},
@@ -212,6 +235,7 @@ func TestGetPVCCostCoefficients(t *testing.T) {
 				NewIntervalPoint(time.Date(2021, 2, 19, 8, 30, 0, 0, time.UTC), "end", pod1Key),
 				NewIntervalPoint(time.Date(2021, 2, 19, 9, 0, 0, 0, time.UTC), "end", pod2Key),
 			},
+			expError: nil,
 			expected: map[podKey][]CoefficientComponent{
 				pod1Key: {
 					{1.0, 0.5},
@@ -230,6 +254,7 @@ func TestGetPVCCostCoefficients(t *testing.T) {
 				NewIntervalPoint(time.Date(2021, 2, 19, 9, 0, 0, 0, time.UTC), "end", pod1Key),
 				NewIntervalPoint(time.Date(2021, 2, 19, 9, 0, 0, 0, time.UTC), "end", pod2Key),
 			},
+			expError: nil,
 			expected: map[podKey][]CoefficientComponent{
 				pod1Key: {
 					{0.5, 0.5},
@@ -249,6 +274,7 @@ func TestGetPVCCostCoefficients(t *testing.T) {
 				NewIntervalPoint(time.Date(2021, 2, 19, 8, 0, 0, 0, time.UTC), "start", pod1Key),
 				NewIntervalPoint(time.Date(2021, 2, 19, 9, 0, 0, 0, time.UTC), "end", pod1Key),
 			},
+			expError: nil,
 			expected: map[podKey][]CoefficientComponent{
 				pod1Key: {
 					{1.0, 1.0},
@@ -264,6 +290,7 @@ func TestGetPVCCostCoefficients(t *testing.T) {
 				NewIntervalPoint(time.Date(2021, 2, 19, 8, 45, 0, 0, time.UTC), "start", pod2Key),
 				NewIntervalPoint(time.Date(2021, 2, 19, 9, 0, 0, 0, time.UTC), "end", pod2Key),
 			},
+			expError: nil,
 			expected: map[podKey][]CoefficientComponent{
 				pod1Key: {
 					{1.0, 0.25},
@@ -283,6 +310,7 @@ func TestGetPVCCostCoefficients(t *testing.T) {
 				NewIntervalPoint(time.Date(2021, 2, 19, 8, 15, 0, 0, time.UTC), "start", pod1Key),
 				NewIntervalPoint(time.Date(2021, 2, 19, 8, 45, 0, 0, time.UTC), "end", pod1Key),
 			},
+			expError: nil,
 			expected: map[podKey][]CoefficientComponent{
 				pod1Key: {
 					{1.0, 0.5},
@@ -293,11 +321,50 @@ func TestGetPVCCostCoefficients(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "back to back pods, full coverage",
+			pvc:  pvc2,
+			intervals: []IntervalPoint{
+				NewIntervalPoint(time.Date(2021, 2, 19, 8, 0, 0, 0, time.UTC), "start", pod1Key),
+				NewIntervalPoint(time.Date(2021, 2, 19, 8, 30, 0, 0, time.UTC), "start", pod2Key),
+				NewIntervalPoint(time.Date(2021, 2, 19, 8, 30, 0, 0, time.UTC), "end", pod1Key),
+				NewIntervalPoint(time.Date(2021, 2, 19, 9, 0, 0, 0, time.UTC), "end", pod2Key),
+			},
+			expError: nil,
+			expected: map[podKey][]CoefficientComponent{
+				pod1Key: {
+					{1.0, 0.5},
+				},
+				pod2Key: {
+					{1.0, 0.5},
+				},
+			},
+		},
+		{
+			name: "zero duration",
+			pvc:  pvc3,
+			intervals: []IntervalPoint{
+				NewIntervalPoint(time.Date(2021, 2, 19, 8, 0, 0, 0, time.UTC), "start", pod1Key),
+				NewIntervalPoint(time.Date(2021, 2, 19, 8, 0, 0, 0, time.UTC), "end", pod1Key),
+			},
+			expError: fmt.Errorf("detected PVC with window of zero duration: %s/%s/%s", "cluster1", "namespace1", "pvc3"),
+			expected: nil,
+		},
 	}
 
 	for _, testCase := range cases {
 		t.Run(testCase.name, func(t *testing.T) {
-			result := getPVCCostCoefficients(testCase.intervals, testCase.pvc)
+			result, err := getPVCCostCoefficients(testCase.intervals, testCase.pvc)
+			if err != nil {
+				if testCase.expError == nil {
+					t.Errorf("getPVCCostCoefficients failed: got unexpected error: %v", err)
+				}
+				return
+			}
+
+			if err == nil && testCase.expError != nil {
+				t.Errorf("getPVCCostCoefficients failed: did not get expected error: %v", testCase.expError)
+			}
 
 			if !reflect.DeepEqual(result, testCase.expected) {
 				t.Errorf("getPVCCostCoefficients test failed: %s: Got %+v but expected %+v", testCase.name, result, testCase.expected)

pkg/costmodel/metrics.go

@@ -187,7 +187,7 @@ func initCostModelMetrics(clusterCache clustercache.ClusterCache, provider model
 		spotGv = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Name: "kubecost_node_is_spot",
 			Help: "kubecost_node_is_spot Cloud provider info about node preemptibility",
-		}, []string{"instance", "node", "instance_type", "region", "provider_id"})
+		}, []string{"instance", "node", "instance_type", "region", "provider_id", "arch"})
 		if _, disabled := disabledMetrics["kubecost_node_is_spot"]; !disabled {
 			toRegisterGV = append(toRegisterGV, spotGv)
 		}
@@ -513,7 +513,7 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 
 				totalCost := cpu*cpuCost + ramCost*(ram/1024/1024/1024) + gpu*gpuCost
 
-				labelKey := getKeyFromLabelStrings(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID)
+				labelKey := getKeyFromLabelStrings(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID, node.ArchType)
 
 				avgCosts, ok := nodeCostAverages[labelKey]
 
@@ -558,9 +558,9 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 				nodeCostAverages[labelKey] = avgCosts
 
 				if node.IsSpot() {
-					cmme.NodeSpotRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(1.0)
+					cmme.NodeSpotRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID, node.ArchType).Set(1.0)
 				} else {
-					cmme.NodeSpotRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(0.0)
+					cmme.NodeSpotRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID, node.ArchType).Set(0.0)
 				}
 				nodeSeen[labelKey] = true
 			}
@@ -666,45 +666,48 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 				pvSeen[labelKey] = true
 			}
 
+			// Remove metrics on Nodes/LoadBalancers/Containers/PVs that no
+			// longer exist
 			for labelString, seen := range nodeSeen {
 				if !seen {
 					log.Debugf("Removing %s from nodes", labelString)
 					labels := getLabelStringsFromKey(labelString)
+
 					ok := cmme.NodeTotalPriceRecorder.DeleteLabelValues(labels...)
 					if ok {
 						log.Debugf("removed %s from totalprice", labelString)
 					} else {
-						log.Infof("FAILURE TO REMOVE %s from totalprice", labelString)
+						log.Errorf("FAILURE TO REMOVE %s from totalprice", labelString)
 					}
 					ok = cmme.NodeSpotRecorder.DeleteLabelValues(labels...)
 					if ok {
 						log.Debugf("removed %s from spot records", labelString)
 					} else {
-						log.Infof("FAILURE TO REMOVE %s from spot records", labelString)
+						log.Errorf("FAILURE TO REMOVE %s from spot records", labelString)
 					}
 					ok = cmme.CPUPriceRecorder.DeleteLabelValues(labels...)
 					if ok {
 						log.Debugf("removed %s from cpuprice", labelString)
 					} else {
-						log.Infof("FAILURE TO REMOVE %s from cpuprice", labelString)
+						log.Errorf("FAILURE TO REMOVE %s from cpuprice", labelString)
 					}
 					ok = cmme.GPUPriceRecorder.DeleteLabelValues(labels...)
 					if ok {
 						log.Debugf("removed %s from gpuprice", labelString)
 					} else {
-						log.Infof("FAILURE TO REMOVE %s from gpuprice", labelString)
+						log.Errorf("FAILURE TO REMOVE %s from gpuprice", labelString)
 					}
 					ok = cmme.GPUCountRecorder.DeleteLabelValues(labels...)
 					if ok {
 						log.Debugf("removed %s from gpucount", labelString)
 					} else {
-						log.Infof("FAILURE TO REMOVE %s from gpucount", labelString)
+						log.Errorf("FAILURE TO REMOVE %s from gpucount", labelString)
 					}
 					ok = cmme.RAMPriceRecorder.DeleteLabelValues(labels...)
 					if ok {
 						log.Debugf("removed %s from ramprice", labelString)
 					} else {
-						log.Infof("FAILURE TO REMOVE %s from ramprice", labelString)
+						log.Errorf("FAILURE TO REMOVE %s from ramprice", labelString)
 					}
 					delete(nodeSeen, labelString)
 					delete(nodeCostAverages, labelString)
@@ -717,7 +720,7 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 					labels := getLabelStringsFromKey(labelString)
 					ok := cmme.LBCostRecorder.DeleteLabelValues(labels...)
 					if !ok {
-						log.Warnf("Metric emission: failed to delete LoadBalancer with labels: %v", labels)
+						log.Errorf("Metric emission: failed to delete LoadBalancer with labels: %v", labels)
 					}
 					delete(loadBalancerSeen, labelString)
 				} else {
@@ -727,9 +730,18 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 			for labelString, seen := range containerSeen {
 				if !seen {
 					labels := getLabelStringsFromKey(labelString)
-					cmme.RAMAllocationRecorder.DeleteLabelValues(labels...)
-					cmme.CPUAllocationRecorder.DeleteLabelValues(labels...)
-					cmme.GPUAllocationRecorder.DeleteLabelValues(labels...)
+					ok := cmme.RAMAllocationRecorder.DeleteLabelValues(labels...)
+					if !ok {
+						log.Errorf("Metric emission: failed to delete RAMAllocation with labels: %v", labels)
+					}
+					ok = cmme.CPUAllocationRecorder.DeleteLabelValues(labels...)
+					if !ok {
+						log.Errorf("Metric emission: failed to delete CPUAllocation with labels: %v", labels)
+					}
+					ok = cmme.GPUAllocationRecorder.DeleteLabelValues(labels...)
+					if !ok {
+						log.Errorf("Metric emission: failed to delete GPUAllocation with labels: %v", labels)
+					}
 					delete(containerSeen, labelString)
 				} else {
 					containerSeen[labelString] = false
@@ -738,7 +750,10 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 			for labelString, seen := range pvSeen {
 				if !seen {
 					labels := getLabelStringsFromKey(labelString)
-					cmme.PersistentVolumePriceRecorder.DeleteLabelValues(labels...)
+					ok := cmme.PersistentVolumePriceRecorder.DeleteLabelValues(labels...)
+					if !ok {
+						log.Errorf("Metric emission: failed to delete PVPrice with labels: %v", labels)
+					}
 					delete(pvSeen, labelString)
 				} else {
 					pvSeen[labelString] = false
@@ -747,7 +762,10 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 			for labelString, seen := range pvcSeen {
 				if !seen {
 					labels := getLabelStringsFromKey(labelString)
-					cmme.PVAllocationRecorder.DeleteLabelValues(labels...)
+					ok := cmme.PVAllocationRecorder.DeleteLabelValues(labels...)
+					if !ok {
+						log.Errorf("Metric emission: failed to delete PVAllocation with labels: %v", labels)
+					}
 					delete(pvcSeen, labelString)
 				} else {
 					pvcSeen[labelString] = false

pkg/costmodel/router.go

@@ -1533,8 +1533,9 @@ func Initialize(additionalConfigWatchers ...*watcher.ConfigMapWatcher) *Accesses
 			Password:    env.GetDBBasicAuthUserPassword(),
 			BearerToken: env.GetDBBearerToken(),
 		},
-		QueryConcurrency: queryConcurrency,
-		QueryLogFile:     "",
+		QueryConcurrency:  queryConcurrency,
+		QueryLogFile:      "",
+		HeaderXScopeOrgId: env.GetPrometheusHeaderXScopeOrgId(),
 	})
 	if err != nil {
 		log.Fatalf("Failed to create prometheus client, Error: %v", err)

pkg/env/costmodelenv.go

@@ -92,6 +92,8 @@ const (
 	PrometheusRetryOnRateLimitMaxRetriesEnvVar  = "PROMETHEUS_RETRY_ON_RATE_LIMIT_MAX_RETRIES"
 	PrometheusRetryOnRateLimitDefaultWaitEnvVar = "PROMETHEUS_RETRY_ON_RATE_LIMIT_DEFAULT_WAIT"
 
+	PrometheusHeaderXScopeOrgIdEnvVar = "PROMETHEUS_HEADER_X_SCOPE_ORGID"
+
 	IngestPodUIDEnvVar = "INGEST_POD_UID"
 
 	ETLReadOnlyMode = "ETL_READ_ONLY"
@@ -162,6 +164,15 @@ func GetPrometheusRetryOnRateLimitDefaultWait() time.Duration {
 	return GetDuration(PrometheusRetryOnRateLimitDefaultWaitEnvVar, 100*time.Millisecond)
 }
 
+// GetPrometheusHeaderXScopeOrgId returns the default value for X-Scope-OrgID header used for requests in Mimir/Cortex-Tenant API.
+// To use Mimir(or Cortex-Tenant) instead of Prometheus add variable from cluster settings:
+// "PROMETHEUS_HEADER_X_SCOPE_ORGID": "my-cluster-name"
+// Then set Prometheus URL to prometheus API endpoint:
+// "PROMETHEUS_SERVER_ENDPOINT": "http://mimir-url/prometheus/"
+func GetPrometheusHeaderXScopeOrgId() string {
+	return Get(PrometheusHeaderXScopeOrgIdEnvVar, "")
+}
+
 // GetPrometheusQueryOffset returns the time.Duration to offset all prometheus queries by. NOTE: This env var is applied
 // to all non-range queries made via our query context. This should only be applied when there is a significant delay in
 // data arriving in the target prom db. For example, if supplying a thanos or cortex querier for the prometheus server, using

pkg/filter21/allocation/fields.go

@@ -25,7 +25,8 @@ const (
 // Filtering based on label aliases (team, department, etc.) should be a
 // responsibility of the query handler. By the time it reaches this
 // structured representation, we shouldn't have to be aware of what is
-// aliased to what.
+// aliased to what. The aliases correspond to either a label or annotation,
+// defined by the user.
 type AllocationAlias string
 
 const (

pkg/filter21/asset/fields.go

@@ -0,0 +1,35 @@
+package asset
+
+// AssetField is an enum that represents Asset-specific fields that can be
+// filtered on (namespace, label, etc.)
+type AssetField string
+
+// If you add a AssetField, make sure to update field maps to return the correct
+// Asset value does not enforce exhaustive pattern matching on "enum" types.
+const (
+	FieldName       AssetField = "name"
+	FieldType       AssetField = "assetType"
+	FieldCategory   AssetField = "category"
+	FieldClusterID  AssetField = "cluster"
+	FieldProject    AssetField = "project"
+	FieldProvider   AssetField = "provider"
+	FieldProviderID AssetField = "providerID"
+	FieldAccount    AssetField = "account"
+	FieldService    AssetField = "service"
+	FieldLabel      AssetField = "label"
+)
+
+// AssetAlias represents an alias field type for assets.
+// Filtering based on label aliases (team, department, etc.) should be a
+// responsibility of the query handler. By the time it reaches this
+// structured representation, we shouldn't have to be aware of what is
+// aliased to what.
+type AssetAlias string
+
+const (
+	DepartmentProp  AssetAlias = "department"
+	EnvironmentProp AssetAlias = "environment"
+	OwnerProp       AssetAlias = "owner"
+	ProductProp     AssetAlias = "product"
+	TeamProp        AssetAlias = "team"
+)

pkg/filter21/asset/parser.go

@@ -0,0 +1,50 @@
+package asset
+
+import "github.com/opencost/opencost/pkg/filter21/ast"
+
+// a slice of all the asset field instances the lexer should recognize as
+// valid left-hand comparators
+var assetFilterFields []*ast.Field = []*ast.Field{
+	ast.NewField(FieldType),
+	ast.NewField(FieldName),
+	ast.NewField(FieldCategory),
+	ast.NewField(FieldClusterID),
+	ast.NewField(FieldProject),
+	ast.NewField(FieldProvider),
+	ast.NewField(FieldProviderID),
+	ast.NewField(FieldAccount),
+	ast.NewField(FieldService),
+	ast.NewMapField(FieldLabel),
+	ast.NewAliasField(DepartmentProp),
+	ast.NewAliasField(EnvironmentProp),
+	ast.NewAliasField(ProductProp),
+	ast.NewAliasField(OwnerProp),
+	ast.NewAliasField(TeamProp),
+}
+
+// fieldMap is a lazily loaded mapping from AllocationField to ast.Field
+var fieldMap map[AssetField]*ast.Field
+
+// DefaultFieldByName returns only default allocation filter fields by name.
+func DefaultFieldByName(field AssetField) *ast.Field {
+	if fieldMap == nil {
+		fieldMap = make(map[AssetField]*ast.Field, len(assetFilterFields))
+		for _, f := range assetFilterFields {
+			ff := *f
+			fieldMap[AssetField(ff.Name)] = &ff
+		}
+	}
+
+	if af, ok := fieldMap[field]; ok {
+		afcopy := *af
+		return &afcopy
+	}
+
+	return nil
+}
+
+// NewAssetFilterParser creates a new `ast.FilterParser` implementation
+// which uses asset specific fields
+func NewAssetFilterParser() ast.FilterParser {
+	return ast.NewFilterParser(assetFilterFields)
+}

pkg/filter21/ast/walker.go

@@ -236,6 +236,9 @@ func Clone(filter FilterNode) FilterNode {
 	var currentOps *util.Stack[FilterGroup] = util.NewStack[FilterGroup]()
 
 	PreOrderTraversal(filter, func(fn FilterNode, state TraversalState) {
+		if fn == nil {
+			return
+		}
 		switch n := fn.(type) {
 		case *AndOp:
 			if state == TraversalStateEnter {
@@ -277,7 +280,10 @@ func Clone(filter FilterNode) FilterNode {
 				currentOps.Top().Add(&ContradictionOp{})
 			}
 		case *EqualOp:
-			var field Field = *n.Left.Field
+			var field Field
+			if n.Left.Field != nil {
+				field = *n.Left.Field
+			}
 			sm := &EqualOp{
 				Left: Identifier{
 					Field: &field,
@@ -293,7 +299,10 @@ func Clone(filter FilterNode) FilterNode {
 			}
 
 		case *ContainsOp:
-			var field Field = *n.Left.Field
+			var field Field
+			if n.Left.Field != nil {
+				field = *n.Left.Field
+			}
 			sm := &ContainsOp{
 				Left: Identifier{
 					Field: &field,
@@ -309,7 +318,10 @@ func Clone(filter FilterNode) FilterNode {
 			}
 
 		case *ContainsPrefixOp:
-			var field Field = *n.Left.Field
+			var field Field
+			if n.Left.Field != nil {
+				field = *n.Left.Field
+			}
 			sm := &ContainsPrefixOp{
 				Left: Identifier{
 					Field: &field,
@@ -325,7 +337,10 @@ func Clone(filter FilterNode) FilterNode {
 			}
 
 		case *ContainsSuffixOp:
-			var field Field = *n.Left.Field
+			var field Field
+			if n.Left.Field != nil {
+				field = *n.Left.Field
+			}
 			sm := &ContainsSuffixOp{
 				Left: Identifier{
 					Field: &field,

pkg/filter21/matcher/compiler.go

@@ -48,7 +48,7 @@ func NewMatchCompiler[T any](
 	}
 }
 
-// Compile accepts anb `ast.FilterNode` tree and compiles it into a `Matcher[T]` implementation
+// Compile accepts an `ast.FilterNode` tree and compiles it into a `Matcher[T]` implementation
 // which can be used to match T instances dynamically.
 func (mc *MatchCompiler[T]) Compile(filter ast.FilterNode) (Matcher[T], error) {
 	// apply compiler passes on parsed ast

pkg/filter21/ops/ops.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/opencost/opencost/pkg/filter21/allocation"
+	"github.com/opencost/opencost/pkg/filter21/asset"
 	"github.com/opencost/opencost/pkg/filter21/ast"
 	"github.com/opencost/opencost/pkg/util/typeutil"
 )
@@ -26,7 +27,7 @@ type keyFieldType interface {
 var defaultFieldByType = map[string]any{
 	// typeutil.TypeOf[cloud.CloudAggregationField]():        cloud.DefaultFieldByName,
 	typeutil.TypeOf[allocation.AllocationField](): allocation.DefaultFieldByName,
-	// typeutil.TypeOf[asset.AssetField]():                   asset.DefaultFieldByName,
+	typeutil.TypeOf[asset.AssetField]():           asset.DefaultFieldByName,
 	// typeutil.TypeOf[containerstats.ContainerStatsField](): containerstats.DefaultFieldByName,
 }
 

pkg/kubecost/allocation.go

@@ -2,6 +2,7 @@ package kubecost
 
 import (
 	"fmt"
+	"math"
 	"sort"
 	"strings"
 	"time"
@@ -92,6 +93,45 @@ type Allocation struct {
 	// and appended to an Allocation, and so by default is is nil.
 	ProportionalAssetResourceCosts ProportionalAssetResourceCosts `json:"proportionalAssetResourceCosts"` //@bingen:field[ignore]
 	SharedCostBreakdown            SharedCostBreakdowns           `json:"sharedCostBreakdown"`            //@bingen:field[ignore]
+	LoadBalancers                  LbAllocations                  `json:"LoadBalancers"`                  // @bingen:field[version=18]
+
+}
+
+type LbAllocations map[string]*LbAllocation
+
+func (orig LbAllocations) Clone() LbAllocations {
+	if orig == nil {
+		return nil
+	}
+
+	newAllocs := LbAllocations{}
+
+	for key, lbAlloc := range orig {
+		newAllocs[key] = &LbAllocation{
+			Service: lbAlloc.Service,
+			Cost:    lbAlloc.Cost,
+			Private: lbAlloc.Private,
+			Ip:      lbAlloc.Ip,
+		}
+	}
+	return newAllocs
+}
+
+type LbAllocation struct {
+	Service string  `json:"service"`
+	Cost    float64 `json:"cost"`
+	Private bool    `json:"private"`
+	Ip      string  `json:"ip"` //@bingen:field[version=19]
+}
+
+func (lba *LbAllocation) SanitizeNaN() {
+	if lba == nil {
+		return
+	}
+	if math.IsNaN(lba.Cost) {
+		log.DedupedWarningf(5, "LBAllocation: Unexpected NaN found for Cost service:%s", lba.Service)
+		lba.Cost = 0
+	}
 }
 
 // RawAllocationOnlyData is information that only belong in "raw" Allocations,
@@ -147,6 +187,20 @@ func (r *RawAllocationOnlyData) Equal(that *RawAllocationOnlyData) bool {
 		util.IsApproximately(r.RAMBytesUsageMax, that.RAMBytesUsageMax)
 }
 
+func (r *RawAllocationOnlyData) SanitizeNaN() {
+	if r == nil {
+		return
+	}
+	if math.IsNaN(r.CPUCoreUsageMax) {
+		log.DedupedWarningf(5, "RawAllocationOnlyData: Unexpected NaN found for CPUCoreUsageMax")
+		r.CPUCoreUsageMax = 0
+	}
+	if math.IsNaN(r.RAMBytesUsageMax) {
+		log.DedupedWarningf(5, "RawAllocationOnlyData: Unexpected NaN found for RAMBytesUsageMax")
+		r.RAMBytesUsageMax = 0
+	}
+}
+
 // PVAllocations is a map of Disk Asset Identifiers to the
 // usage of them by an Allocation as recorded in a PVAllocation
 type PVAllocations map[PVKey]*PVAllocation
@@ -210,6 +264,12 @@ func (this PVAllocations) Equal(that PVAllocations) bool {
 	return true
 }
 
+func (pvs PVAllocations) SanitizeNaN() {
+	for _, pv := range pvs {
+		pv.SanitizeNaN()
+	}
+}
+
 // PVKey for identifying Disk type assets
 type PVKey struct {
 	Cluster string `json:"cluster"`
@@ -251,25 +311,46 @@ func (pva *PVAllocation) Equal(that *PVAllocation) bool {
 		util.IsApproximately(pva.Cost, that.Cost)
 }
 
+func (pva *PVAllocation) SanitizeNaN() {
+	if pva == nil {
+		return
+	}
+	if math.IsNaN(pva.ByteHours) {
+		log.DedupedWarningf(5, "PVAllocation: Unexpected NaN found for ByteHours")
+		pva.ByteHours = 0
+	}
+	if math.IsNaN(pva.Cost) {
+		log.DedupedWarningf(5, "PVAllocation: Unexpected NaN found for Cost")
+		pva.Cost = 0
+	}
+}
+
 type ProportionalAssetResourceCost struct {
-	Cluster                    string  `json:"cluster"`
-	Node                       string  `json:"node,omitempty"`
-	ProviderID                 string  `json:"providerID,omitempty"`
-	CPUPercentage              float64 `json:"cpuPercentage"`
-	GPUPercentage              float64 `json:"gpuPercentage"`
-	RAMPercentage              float64 `json:"ramPercentage"`
-	NodeResourceCostPercentage float64 `json:"nodeResourceCostPercentage"`
-	GPUTotalCost               float64 `json:"-"`
-	GPUProportionalCost        float64 `json:"-"`
-	CPUTotalCost               float64 `json:"-"`
-	CPUProportionalCost        float64 `json:"-"`
-	RAMTotalCost               float64 `json:"-"`
-	RAMProportionalCost        float64 `json:"-"`
-}
-
-func (parc ProportionalAssetResourceCost) Key(insertByNode bool) string {
-	if insertByNode {
-		return parc.Cluster + "," + parc.Node
+	Cluster                      string  `json:"cluster"`
+	Name                         string  `json:"name,omitempty"`
+	Type                         string  `json:"type,omitempty"`
+	ProviderID                   string  `json:"providerID,omitempty"`
+	CPUPercentage                float64 `json:"cpuPercentage"`
+	GPUPercentage                float64 `json:"gpuPercentage"`
+	RAMPercentage                float64 `json:"ramPercentage"`
+	LoadBalancerPercentage       float64 `json:"loadBalancerPercentage"`
+	PVPercentage                 float64 `json:"pvPercentage"`
+	NodeResourceCostPercentage   float64 `json:"nodeResourceCostPercentage"`
+	GPUTotalCost                 float64 `json:"-"`
+	GPUProportionalCost          float64 `json:"-"`
+	CPUTotalCost                 float64 `json:"-"`
+	CPUProportionalCost          float64 `json:"-"`
+	RAMTotalCost                 float64 `json:"-"`
+	RAMProportionalCost          float64 `json:"-"`
+	LoadBalancerProportionalCost float64 `json:"-"`
+	LoadBalancerTotalCost        float64 `json:"-"`
+	PVProportionalCost           float64 `json:"-"`
+	PVTotalCost                  float64 `json:"-"`
+}
+
+func (parc ProportionalAssetResourceCost) Key(insertByName bool) string {
+	if insertByName {
+		return parc.Cluster + "," + parc.Name
 	} else {
 		return parc.Cluster
 	}
@@ -287,36 +368,36 @@ func (parcs ProportionalAssetResourceCosts) Clone() ProportionalAssetResourceCos
 	return cloned
 }
 
-func (parcs ProportionalAssetResourceCosts) Insert(parc ProportionalAssetResourceCost, insertByNode bool) {
-	if !insertByNode {
-		parc.Node = ""
+func (parcs ProportionalAssetResourceCosts) Insert(parc ProportionalAssetResourceCost, insertByName bool) {
+	if !insertByName {
+		parc.Name = ""
+		parc.Type = ""
 		parc.ProviderID = ""
 	}
-	if curr, ok := parcs[parc.Key(insertByNode)]; ok {
 
+	if curr, ok := parcs[parc.Key(insertByName)]; ok {
 		toInsert := ProportionalAssetResourceCost{
-			Node:                curr.Node,
-			Cluster:             curr.Cluster,
-			ProviderID:          curr.ProviderID,
-			CPUTotalCost:        curr.CPUTotalCost + parc.CPUTotalCost,
-			CPUProportionalCost: curr.CPUProportionalCost + parc.CPUProportionalCost,
-			RAMTotalCost:        curr.RAMTotalCost + parc.RAMTotalCost,
-			RAMProportionalCost: curr.RAMProportionalCost + parc.RAMProportionalCost,
-			GPUProportionalCost: curr.GPUProportionalCost + parc.GPUProportionalCost,
-			GPUTotalCost:        curr.GPUTotalCost + parc.GPUTotalCost,
-		}
-
-		computePercentages(&toInsert)
-		parcs[parc.Key(insertByNode)] = toInsert
+			Name:                         curr.Name,
+			Type:                         curr.Type,
+			Cluster:                      curr.Cluster,
+			ProviderID:                   curr.ProviderID,
+			CPUProportionalCost:          curr.CPUProportionalCost + parc.CPUProportionalCost,
+			RAMProportionalCost:          curr.RAMProportionalCost + parc.RAMProportionalCost,
+			GPUProportionalCost:          curr.GPUProportionalCost + parc.GPUProportionalCost,
+			PVProportionalCost:           curr.PVProportionalCost + parc.PVProportionalCost,
+			LoadBalancerProportionalCost: curr.LoadBalancerProportionalCost + parc.LoadBalancerProportionalCost,
+		}
+
+		ComputePercentages(&toInsert)
+		parcs[parc.Key(insertByName)] = toInsert
 	} else {
-		computePercentages(&parc)
-		parcs[parc.Key(insertByNode)] = parc
+		ComputePercentages(&parc)
+		parcs[parc.Key(insertByName)] = parc
 	}
 }
 
-func computePercentages(toInsert *ProportionalAssetResourceCost) {
-	// compute percentages
-	totalCost := toInsert.RAMTotalCost + toInsert.CPUTotalCost + toInsert.GPUTotalCost
+func ComputePercentages(toInsert *ProportionalAssetResourceCost) {
+	totalNodeCost := toInsert.RAMTotalCost + toInsert.CPUTotalCost + toInsert.GPUTotalCost
 
 	if toInsert.CPUTotalCost > 0 {
 		toInsert.CPUPercentage = toInsert.CPUProportionalCost / toInsert.CPUTotalCost
@@ -326,21 +407,29 @@ func computePercentages(toInsert *ProportionalAssetResourceCost) {
 		toInsert.GPUPercentage = toInsert.GPUProportionalCost / toInsert.GPUTotalCost
 	}
 
+	if toInsert.LoadBalancerTotalCost > 0 {
+		toInsert.LoadBalancerPercentage = toInsert.LoadBalancerProportionalCost / toInsert.LoadBalancerTotalCost
+	}
+
 	if toInsert.RAMTotalCost > 0 {
 		toInsert.RAMPercentage = toInsert.RAMProportionalCost / toInsert.RAMTotalCost
 	}
 
-	ramFraction := toInsert.RAMTotalCost / totalCost
+	if toInsert.PVTotalCost > 0 {
+		toInsert.PVPercentage = toInsert.PVProportionalCost / toInsert.PVTotalCost
+	}
+
+	ramFraction := toInsert.RAMTotalCost / totalNodeCost
 	if ramFraction != ramFraction || ramFraction < 0 {
 		ramFraction = 0
 	}
 
-	cpuFraction := toInsert.CPUTotalCost / totalCost
+	cpuFraction := toInsert.CPUTotalCost / totalNodeCost
 	if cpuFraction != cpuFraction || cpuFraction < 0 {
 		cpuFraction = 0
 	}
 
-	gpuFraction := toInsert.GPUTotalCost / totalCost
+	gpuFraction := toInsert.GPUTotalCost / totalNodeCost
 	if gpuFraction != gpuFraction || gpuFraction < 0 {
 		gpuFraction = 0
 	}
@@ -350,14 +439,84 @@ func computePercentages(toInsert *ProportionalAssetResourceCost) {
 }
 
 func (parcs ProportionalAssetResourceCosts) Add(that ProportionalAssetResourceCosts) {
-
 	for _, parc := range that {
-		// if node field is empty, we know this is a cluster level PARC aggregation
-		insertByNode := true
-		if parc.Node == "" {
-			insertByNode = false
+		// if name field is empty, we know this is a cluster level PARC aggregation
+		insertByName := true
+		if parc.Name == "" {
+			insertByName = false
 		}
-		parcs.Insert(parc, insertByNode)
+		parcs.Insert(parc, insertByName)
+	}
+}
+
+func (parcs ProportionalAssetResourceCosts) SanitizeNaN() {
+	for key, parc := range parcs {
+		if math.IsNaN(parc.CPUPercentage) {
+			log.DedupedWarningf(5, "ProportionalAssetResourceCosts: Unexpected NaN found for CPUPercentage name:%s", parc.Name)
+			parc.CPUPercentage = 0
+		}
+		if math.IsNaN(parc.GPUPercentage) {
+			log.DedupedWarningf(5, "ProportionalAssetResourceCosts: Unexpected NaN found for GPUPercentage name:%s", parc.Name)
+			parc.GPUPercentage = 0
+		}
+		if math.IsNaN(parc.RAMPercentage) {
+			log.DedupedWarningf(5, "ProportionalAssetResourceCosts: Unexpected NaN found for RAMPercentage name:%s", parc.Name)
+			parc.RAMPercentage = 0
+		}
+		if math.IsNaN(parc.LoadBalancerPercentage) {
+			log.DedupedWarningf(5, "ProportionalAssetResourceCosts: Unexpected NaN found for LoadBalancerPercentage name:%s", parc.Name)
+			parc.LoadBalancerPercentage = 0
+		}
+		if math.IsNaN(parc.PVPercentage) {
+			log.DedupedWarningf(5, "ProportionalAssetResourceCosts: Unexpected NaN found for PVPercentage name:%s", parc.Name)
+			parc.PVPercentage = 0
+		}
+		if math.IsNaN(parc.NodeResourceCostPercentage) {
+			log.DedupedWarningf(5, "ProportionalAssetResourceCosts: Unexpected NaN found for NodeResourceCostPercentage name:%s", parc.Name)
+			parc.NodeResourceCostPercentage = 0
+		}
+		if math.IsNaN(parc.GPUTotalCost) {
+			log.DedupedWarningf(5, "ProportionalAssetResourceCosts: Unexpected NaN found for GPUTotalCost name:%s", parc.Name)
+			parc.GPUTotalCost = 0
+		}
+		if math.IsNaN(parc.GPUProportionalCost) {
+			log.DedupedWarningf(5, "ProportionalAssetResourceCosts: Unexpected NaN found for GPUProportionalCost name:%s", parc.Name)
+			parc.GPUProportionalCost = 0
+		}
+		if math.IsNaN(parc.CPUTotalCost) {
+			log.DedupedWarningf(5, "ProportionalAssetResourceCosts: Unexpected NaN found for CPUTotalCost name:%s", parc.Name)
+			parc.CPUTotalCost = 0
+		}
+		if math.IsNaN(parc.CPUProportionalCost) {
+			log.DedupedWarningf(5, "ProportionalAssetResourceCosts: Unexpected NaN found for CPUProportionalCost name:%s", parc.Name)
+			parc.CPUProportionalCost = 0
+		}
+		if math.IsNaN(parc.RAMTotalCost) {
+			log.DedupedWarningf(5, "ProportionalAssetResourceCosts: Unexpected NaN found for RAMTotalCost name:%s", parc.Name)
+			parc.RAMTotalCost = 0
+		}
+		if math.IsNaN(parc.RAMProportionalCost) {
+			log.DedupedWarningf(5, "ProportionalAssetResourceCosts: Unexpected NaN found for RAMProportionalCost name:%s", parc.Name)
+			parc.RAMProportionalCost = 0
+		}
+		if math.IsNaN(parc.LoadBalancerProportionalCost) {
+			log.DedupedWarningf(5, "ProportionalAssetResourceCosts: Unexpected NaN found for LoadBalancerProportionalCost name:%s", parc.Name)
+			parc.LoadBalancerProportionalCost = 0
+		}
+		if math.IsNaN(parc.LoadBalancerTotalCost) {
+			log.DedupedWarningf(5, "ProportionalAssetResourceCosts: Unexpected NaN found for LoadBalancerTotalCost name:%s", parc.Name)
+			parc.LoadBalancerTotalCost = 0
+		}
+		if math.IsNaN(parc.PVProportionalCost) {
+			log.DedupedWarningf(5, "ProportionalAssetResourceCosts: Unexpected NaN found for PVProportionalCost name:%s", parc.Name)
+			parc.PVProportionalCost = 0
+		}
+		if math.IsNaN(parc.PVTotalCost) {
+			log.DedupedWarningf(5, "ProportionalAssetResourceCosts: Unexpected NaN found for PVTotalCost name:%s", parc.Name)
+			parc.PVTotalCost = 0
+		}
+
+		parcs[key] = parc
 	}
 }
 
@@ -408,6 +567,44 @@ func (scbs SharedCostBreakdowns) Add(that SharedCostBreakdowns) {
 	}
 }
 
+func (scbs SharedCostBreakdowns) SanitizeNaN() {
+	for key, scb := range scbs {
+		if math.IsNaN(scb.CPUCost) {
+			log.DedupedWarningf(5, "SharedCostBreakdown: Unexpected NaN found for CPUCost name:%s", scb.Name)
+			scb.CPUCost = 0
+		}
+		if math.IsNaN(scb.GPUCost) {
+			log.DedupedWarningf(5, "SharedCostBreakdown: Unexpected NaN found for GPUCost name:%s", scb.Name)
+			scb.GPUCost = 0
+		}
+		if math.IsNaN(scb.RAMCost) {
+			log.DedupedWarningf(5, "SharedCostBreakdown: Unexpected NaN found for RAMCost name:%s", scb.Name)
+			scb.RAMCost = 0
+		}
+		if math.IsNaN(scb.PVCost) {
+			log.DedupedWarningf(5, "SharedCostBreakdown: Unexpected NaN found for PVCost name:%s", scb.Name)
+			scb.PVCost = 0
+		}
+		if math.IsNaN(scb.NetworkCost) {
+			log.DedupedWarningf(5, "SharedCostBreakdown: Unexpected NaN found for NetworkCost name:%s", scb.Name)
+			scb.NetworkCost = 0
+		}
+		if math.IsNaN(scb.LBCost) {
+			log.DedupedWarningf(5, "SharedCostBreakdown: Unexpected NaN found for LBCost name:%s", scb.Name)
+			scb.LBCost = 0
+		}
+		if math.IsNaN(scb.ExternalCost) {
+			log.DedupedWarningf(5, "SharedCostBreakdown: Unexpected NaN found for ExternalCost name:%s", scb.Name)
+			scb.ExternalCost = 0
+		}
+		if math.IsNaN(scb.TotalCost) {
+			log.DedupedWarningf(5, "SharedCostBreakdown: Unexpected NaN found for TotalCost name:%s", scb.Name)
+			scb.TotalCost = 0
+		}
+		scbs[key] = scb
+	}
+}
+
 // GetWindow returns the window of the struct
 func (a *Allocation) GetWindow() Window {
 	return a.Window
@@ -477,6 +674,7 @@ func (a *Allocation) Clone() *Allocation {
 		RawAllocationOnly:              a.RawAllocationOnly.Clone(),
 		ProportionalAssetResourceCosts: a.ProportionalAssetResourceCosts.Clone(),
 		SharedCostBreakdown:            a.SharedCostBreakdown.Clone(),
+		LoadBalancers:                  a.LoadBalancers.Clone(),
 	}
 }
 
@@ -822,12 +1020,21 @@ func (a *Allocation) IsUnallocated() bool {
 }
 
 // IsUnmounted is true if the given Allocation represents unmounted volume costs.
+// Note: Due to change in https://github.com/opencost/opencost/pull/1477 made to include Unmounted
+// PVC cost inside namespace we need to check unmounted suffix across all the three major properties
+// to actually classify it as unmounted.
 func (a *Allocation) IsUnmounted() bool {
 	if a == nil {
 		return false
 	}
 
-	return strings.Contains(a.Name, UnmountedSuffix)
+	props := a.Properties
+	if props != nil {
+		if props.Container == UnmountedSuffix && props.Namespace == UnmountedSuffix && props.Pod == UnmountedSuffix {
+			return true
+		}
+	}
+	return false
 }
 
 // Minutes returns the number of minutes the Allocation represents, as defined
@@ -999,11 +1206,44 @@ func (a *Allocation) add(that *Allocation) {
 	a.NetworkCostAdjustment += that.NetworkCostAdjustment
 	a.LoadBalancerCostAdjustment += that.LoadBalancerCostAdjustment
 
+	// Sum LoadBalancer Allocations
+	a.LoadBalancers = a.LoadBalancers.Add(that.LoadBalancers)
+
 	// Any data that is in a "raw allocation only" is not valid in any
 	// sort of cumulative Allocation (like one that is added).
 	a.RawAllocationOnly = nil
 }
 
+func (thisLbAllocs LbAllocations) Add(thatLbAllocs LbAllocations) LbAllocations {
+	// loop through both sets of LB allocations, building a new LBAllocations that has the summed set
+	mergedLbAllocs := thisLbAllocs.Clone()
+	if thatLbAllocs != nil {
+		if mergedLbAllocs == nil {
+			mergedLbAllocs = LbAllocations{}
+		}
+		for lbKey, thatlbAlloc := range thatLbAllocs {
+			thisLbAlloc, ok := mergedLbAllocs[lbKey]
+			if !ok {
+				thisLbAlloc = &LbAllocation{
+					Service: thatlbAlloc.Service,
+					Cost:    thatlbAlloc.Cost,
+				}
+				mergedLbAllocs[lbKey] = thisLbAlloc
+			} else {
+				thisLbAlloc.Cost += thatlbAlloc.Cost
+			}
+
+		}
+	}
+	return mergedLbAllocs
+}
+
+func (thisLbAllocs LbAllocations) SanitizeNaN() {
+	for _, lba := range thisLbAllocs {
+		lba.SanitizeNaN()
+	}
+}
+
 // AllocationSet stores a set of Allocations, each with a unique name, that share
 // a window. An AllocationSet is mutable, so treat it like a threadsafe map.
 type AllocationSet struct {
@@ -1158,6 +1398,12 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 	shouldShare := len(options.SharedHourlyCosts) > 0 || len(options.ShareFuncs) > 0
 	if !shouldAggregate && !shouldFilter && !shouldShare && options.ShareIdle == ShareNone && !options.IncludeProportionalAssetResourceCosts {
 		// There is nothing for AggregateBy to do, so simply return nil
+		// before returning, set aggregated metadata inclusion in properties
+		if options.IncludeAggregatedMetadata {
+			for index := range as.Allocations {
+				as.Allocations[index].Properties.AggregatedMetadata = true
+			}
+		}
 		return nil
 	}
 
@@ -1195,6 +1441,12 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 	for _, alloc := range as.Allocations {
 
 		alloc.Properties.AggregatedMetadata = options.IncludeAggregatedMetadata
+		// build a parallel set of allocations to only be used
+		// for computing PARCs
+		if options.IncludeProportionalAssetResourceCosts {
+			parcSet.Insert(alloc.Clone())
+		}
+
 		// External allocations get aggregated post-hoc (see step 6) and do
 		// not necessarily contain complete sets of properties, so they are
 		// moved to a separate AllocationSet.
@@ -1218,12 +1470,6 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 				aggSet.Insert(alloc)
 			}
 
-			// build a parallel set of allocations to only be used
-			// for computing PARCs
-			if options.IncludeProportionalAssetResourceCosts {
-				parcSet.Insert(alloc.Clone())
-			}
-
 			continue
 		}
 
@@ -1294,7 +1540,7 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 	// (2b) If proportional asset resource costs are to be included, compute them
 	// and add them to the allocations.
 	if options.IncludeProportionalAssetResourceCosts {
-		err := deriveProportionalAssetResourceCosts(options, as, shareSet)
+		err := deriveProportionalAssetResourceCosts(options, as, shareSet, parcSet)
 		if err != nil {
 			log.Debugf("AggregateBy: failed to derive proportional asset resource costs from idle coefficients: %s", err)
 			return fmt.Errorf("AggregateBy: failed to derive proportional asset resource costs from idle coefficients: %s", err)
@@ -1895,7 +2141,7 @@ func computeIdleCoeffs(options *AllocationAggregationOptions, as *AllocationSet,
 	return coeffs, totals, nil
 }
 
-func deriveProportionalAssetResourceCosts(options *AllocationAggregationOptions, as *AllocationSet, shareSet *AllocationSet) error {
+func deriveProportionalAssetResourceCosts(options *AllocationAggregationOptions, as *AllocationSet, shareSet, parcsSet *AllocationSet) error {
 
 	// Compute idle coefficients, then save them in AllocationAggregationOptions
 	// [idle_id][allocation name][resource] = [coeff]
@@ -1906,11 +2152,7 @@ func deriveProportionalAssetResourceCosts(options *AllocationAggregationOptions,
 	totals := map[string]map[string]float64{}
 
 	// Record allocation values first, then normalize by totals to get percentages
-	for _, alloc := range as.Allocations {
-		if alloc.IsIdle() {
-			// Skip idle allocations in coefficient calculation
-			continue
-		}
+	for _, alloc := range parcsSet.Allocations {
 
 		idleId, err := alloc.getIdleId(options)
 		if err != nil {
@@ -1931,6 +2173,22 @@ func deriveProportionalAssetResourceCosts(options *AllocationAggregationOptions,
 		if _, ok := coeffs[idleId][name]; !ok {
 			coeffs[idleId][name] = map[string]float64{}
 		}
+		// idle IDs for load balancers are their services
+		for key := range alloc.LoadBalancers {
+			if _, ok := totals[key]; !ok {
+				totals[key] = map[string]float64{}
+			}
+
+			if _, ok := coeffs[key]; !ok {
+				coeffs[key] = map[string]map[string]float64{}
+			}
+			if _, ok := coeffs[key][name]; !ok {
+				coeffs[key][name] = map[string]float64{}
+			}
+
+			coeffs[key][name]["loadbalancer"] += alloc.LoadBalancerTotalCost()
+			totals[key]["loadbalancer"] += alloc.LoadBalancerTotalCost()
+		}
 
 		coeffs[idleId][name]["cpu"] += alloc.CPUTotalCost()
 		coeffs[idleId][name]["gpu"] += alloc.GPUTotalCost()
@@ -1976,6 +2234,23 @@ func deriveProportionalAssetResourceCosts(options *AllocationAggregationOptions,
 		totals[idleId]["cpu"] += alloc.CPUTotalCost()
 		totals[idleId]["gpu"] += alloc.GPUTotalCost()
 		totals[idleId]["ram"] += alloc.RAMTotalCost()
+
+		// idle IDs for load balancers are their services
+		for key := range alloc.LoadBalancers {
+			if _, ok := totals[key]; !ok {
+				totals[key] = map[string]float64{}
+			}
+
+			if _, ok := coeffs[key]; !ok {
+				coeffs[key] = map[string]map[string]float64{}
+			}
+			if _, ok := coeffs[key][name]; !ok {
+				coeffs[key][name] = map[string]float64{}
+			}
+			coeffs[key][name]["loadbalancer"] += alloc.LoadBalancerTotalCost()
+			totals[key]["loadbalancer"] += alloc.LoadBalancerTotalCost()
+		}
+
 	}
 
 	// after totals are computed, loop through and set parcs on allocations
@@ -1988,15 +2263,35 @@ func deriveProportionalAssetResourceCosts(options *AllocationAggregationOptions,
 		alloc.ProportionalAssetResourceCosts = ProportionalAssetResourceCosts{}
 		alloc.ProportionalAssetResourceCosts.Insert(ProportionalAssetResourceCost{
 			Cluster:             alloc.Properties.Cluster,
-			Node:                alloc.Properties.Node,
+			Name:                alloc.Properties.Node,
+			Type:                "Node",
 			ProviderID:          alloc.Properties.ProviderID,
-			GPUTotalCost:        totals[idleId]["gpu"],
-			CPUTotalCost:        totals[idleId]["cpu"],
-			RAMTotalCost:        totals[idleId]["ram"],
 			GPUProportionalCost: coeffs[idleId][alloc.Name]["gpu"],
 			CPUProportionalCost: coeffs[idleId][alloc.Name]["cpu"],
 			RAMProportionalCost: coeffs[idleId][alloc.Name]["ram"],
 		}, options.IdleByNode)
+		// insert a separate PARC for the load balancer
+		if alloc.LoadBalancerCost != 0 {
+			for key, svc := range alloc.LoadBalancers {
+
+				alloc.ProportionalAssetResourceCosts.Insert(ProportionalAssetResourceCost{
+					Cluster:                      alloc.Properties.Cluster,
+					Name:                         svc.Service,
+					Type:                         "LoadBalancer",
+					LoadBalancerProportionalCost: coeffs[key][alloc.Name]["loadbalancer"],
+				}, options.IdleByNode)
+			}
+		}
+
+		for name, pvAlloc := range alloc.PVs {
+			// insert a separate PARC for each PV attached
+			alloc.ProportionalAssetResourceCosts.Insert(ProportionalAssetResourceCost{
+				Cluster:            name.Cluster,
+				Name:               name.Name,
+				Type:               "PV",
+				PVProportionalCost: pvAlloc.Cost,
+			}, options.IdleByNode)
+		}
 	}
 
 	return nil
@@ -2169,6 +2464,118 @@ func (a *Allocation) StringMapProperty(property string) (map[string]string, erro
 	}
 }
 
+func (a *Allocation) SanitizeNaN() {
+	if a == nil {
+		return
+	}
+	if math.IsNaN(a.CPUCost) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for CPUCost: name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.CPUCost = 0
+	}
+	if math.IsNaN(a.CPUCoreRequestAverage) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for CPUCoreRequestAverage: name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.CPUCoreRequestAverage = 0
+	}
+	if math.IsNaN(a.CPUCoreHours) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for CPUCoreHours: name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.CPUCoreHours = 0
+	}
+	if math.IsNaN(a.CPUCoreUsageAverage) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for CPUCoreUsageAverage name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.CPUCoreUsageAverage = 0
+	}
+	if math.IsNaN(a.CPUCostAdjustment) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for CPUCostAdjustment name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.CPUCostAdjustment = 0
+	}
+	if math.IsNaN(a.GPUHours) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for GPUHours name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.GPUHours = 0
+	}
+	if math.IsNaN(a.GPUCost) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for GPUCost name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.GPUCost = 0
+	}
+	if math.IsNaN(a.GPUCostAdjustment) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for GPUCostAdjustment name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.GPUCostAdjustment = 0
+	}
+	if math.IsNaN(a.NetworkTransferBytes) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for NetworkTransferBytes name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.NetworkTransferBytes = 0
+	}
+	if math.IsNaN(a.NetworkReceiveBytes) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for NetworkReceiveBytes name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.NetworkReceiveBytes = 0
+	}
+	if math.IsNaN(a.NetworkCost) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for NetworkCost name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.NetworkCost = 0
+	}
+	if math.IsNaN(a.NetworkCrossZoneCost) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for NetworkCrossZoneCost name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.NetworkCrossZoneCost = 0
+	}
+	if math.IsNaN(a.NetworkCrossRegionCost) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for NetworkCrossRegionCost name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.NetworkCrossRegionCost = 0
+	}
+	if math.IsNaN(a.NetworkInternetCost) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for NetworkInternetCost name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.NetworkInternetCost = 0
+	}
+	if math.IsNaN(a.NetworkCostAdjustment) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for NetworkCostAdjustment name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.NetworkCostAdjustment = 0
+	}
+	if math.IsNaN(a.LoadBalancerCost) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for LoadBalancerCost name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.LoadBalancerCost = 0
+	}
+	if math.IsNaN(a.LoadBalancerCostAdjustment) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for LoadBalancerCostAdjustment name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.LoadBalancerCostAdjustment = 0
+	}
+	if math.IsNaN(a.PVCostAdjustment) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for PVCostAdjustment name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.PVCostAdjustment = 0
+	}
+	if math.IsNaN(a.RAMByteHours) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for RAMByteHours name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.RAMByteHours = 0
+	}
+	if math.IsNaN(a.RAMBytesRequestAverage) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for RAMBytesRequestAverage name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.RAMBytesRequestAverage = 0
+	}
+	if math.IsNaN(a.RAMBytesUsageAverage) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for RAMBytesUsageAverage name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.RAMBytesUsageAverage = 0
+	}
+	if math.IsNaN(a.RAMCost) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for RAMCost name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.RAMCost = 0
+	}
+	if math.IsNaN(a.RAMCostAdjustment) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for RAMCostAdjustment name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.RAMCostAdjustment = 0
+	}
+	if math.IsNaN(a.SharedCost) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for SharedCost name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.SharedCost = 0
+	}
+	if math.IsNaN(a.ExternalCost) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for ExternalCost name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.ExternalCost = 0
+	}
+
+	a.PVs.SanitizeNaN()
+	a.RawAllocationOnly.SanitizeNaN()
+	a.ProportionalAssetResourceCosts.SanitizeNaN()
+	a.SharedCostBreakdown.SanitizeNaN()
+	a.LoadBalancers.SanitizeNaN()
+}
+
 // Clone returns a new AllocationSet with a deep copy of the given
 // AllocationSet's allocations.
 func (as *AllocationSet) Clone() *AllocationSet {
@@ -2493,6 +2900,15 @@ func (as *AllocationSet) Accumulate(that *AllocationSet) (*AllocationSet, error)
 	return acc, nil
 }
 
+func (as *AllocationSet) SanitizeNaN() {
+	if as == nil {
+		return
+	}
+	for _, a := range as.Allocations {
+		a.SanitizeNaN()
+	}
+}
+
 // AllocationSetRange is a thread-safe slice of AllocationSets. It is meant to
 // be used such that the AllocationSets held are consecutive and coherent with
 // respect to using the same aggregation properties, UTC offset, and

pkg/kubecost/allocation_json.go

@@ -55,6 +55,7 @@ type AllocationJSON struct {
 	TotalEfficiency                *float64                        `json:"totalEfficiency"`
 	RawAllocationOnly              *RawAllocationOnlyData          `json:"rawAllocationOnly,omitempty"`
 	ProportionalAssetResourceCosts *ProportionalAssetResourceCosts `json:"proportionalAssetResourceCosts,omitempty"`
+	LoadBalancers                  LbAllocations                   `json:"lbAllocations"`
 	SharedCostBreakdown            *SharedCostBreakdowns           `json:"sharedCostBreakdown,omitempty"`
 }
 
@@ -106,8 +107,8 @@ func (aj *AllocationJSON) BuildFromAllocation(a *Allocation) {
 	aj.TotalEfficiency = formatFloat64ForResponse(a.TotalEfficiency())
 	aj.RawAllocationOnly = a.RawAllocationOnly
 	aj.ProportionalAssetResourceCosts = &a.ProportionalAssetResourceCosts
+	aj.LoadBalancers = a.LoadBalancers
 	aj.SharedCostBreakdown = &a.SharedCostBreakdown
-
 }
 
 // formatFloat64ForResponse - take an existing float64, round it to 6 decimal places and return is possible, or return nil if invalid

pkg/kubecost/allocation_json_test.go

@@ -2,10 +2,11 @@ package kubecost
 
 import (
 	"encoding/json"
-	"github.com/opencost/opencost/pkg/util/mathutil"
 	"math"
 	"testing"
 	"time"
+
+	"github.com/opencost/opencost/pkg/util/mathutil"
 )
 
 func TestAllocation_MarshalJSON(t *testing.T) {
@@ -155,6 +156,72 @@ func TestPVAllocations_MarshalJSON(t *testing.T) {
 
 }
 
+func TestLbAllocation_MarshalJSON(t *testing.T) {
+	testCases := map[string]LbAllocations{
+		"empty": {},
+		"single": {
+			"cluster1/namespace1/ingress": {
+				Service: "namespace1/ingress",
+				Cost:    1,
+				Private: false,
+				Ip:      "127.0.0.1",
+			},
+		},
+		"multi": {
+			"cluster1/namespace1/ingress": {
+				Service: "namespace1/ingress",
+				Cost:    1,
+				Private: false,
+				Ip:      "127.0.0.1",
+			},
+			"cluster1/namespace1/frontend": {
+				Service: "namespace1/frontend",
+				Cost:    1,
+				Private: false,
+				Ip:      "127.0.0.2",
+			},
+		},
+		"emptyLB": {
+			"cluster1/namespace1/pod": {},
+		},
+	}
+
+	for name, before := range testCases {
+		t.Run(name, func(t *testing.T) {
+			data, err := json.Marshal(before)
+			if err != nil {
+				t.Fatalf("LbAllocations.MarshalJSON: unexpected error: %s", err)
+			}
+
+			after := LbAllocations{}
+			err = json.Unmarshal(data, &after)
+			if err != nil {
+				t.Fatalf("LbAllocations.UnmarshalJSON: unexpected error: %s", err)
+			}
+
+			if len(before) != len(after) {
+				t.Fatalf("LbAllocations.MarshalJSON: before and after are not equal")
+			}
+
+			for serviceKey, beforeLB := range before {
+				afterLB, ok := after[serviceKey]
+				if !ok {
+					t.Fatalf("LbAllocations.MarshalJSON: after missing serviceKey %s", serviceKey)
+				}
+				if beforeLB.Cost != afterLB.Cost {
+					t.Fatalf("LbAllocations.MarshalJSON: LbAllocation Cost not equal for serviceKey %s", serviceKey)
+				}
+
+				if beforeLB.Ip != afterLB.Ip {
+					t.Fatalf("LbAllocations.MarshalJSON: LbAllocation Ip not equal for serviceKey %s", serviceKey)
+				}
+			}
+
+		})
+	}
+
+}
+
 func TestFormatFloat64ForResponse(t *testing.T) {
 	type formatTestCase struct {
 		name          string

pkg/kubecost/allocation_test.go

@@ -551,8 +551,9 @@ func assertParcResults(t *testing.T, as *AllocationSet, msg string, exps map[str
 			actualParc.CPUPercentage = roundFloat(actualParc.CPUPercentage)
 			actualParc.RAMPercentage = roundFloat(actualParc.RAMPercentage)
 			actualParc.GPUPercentage = roundFloat(actualParc.GPUPercentage)
+			actualParc.PVPercentage = roundFloat(actualParc.PVPercentage)
 			if !reflect.DeepEqual(expectedParcs[key], actualParc) {
-				t.Fatalf("actual PARC %v did not match expected PARC %v", actualParc, expectedParcs[key])
+				t.Fatalf("actual PARC %+v did not match expected PARC %+v", actualParc, expectedParcs[key])
 			}
 		}
 
@@ -759,9 +760,11 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 
 		// 1a AggregationProperties=(Cluster)
 		"1a": {
-			start:      start,
-			aggBy:      []string{AllocationClusterProp},
-			aggOpts:    nil,
+			start: start,
+			aggBy: []string{AllocationClusterProp},
+			aggOpts: &AllocationAggregationOptions{
+				IncludeProportionalAssetResourceCosts: true,
+			},
 			numResults: numClusters + numIdle,
 			totalCost:  activeTotalCost + idleTotalCost,
 			results: map[string]float64{
@@ -772,6 +775,32 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			windowStart: startYesterday,
 			windowEnd:   endYesterday,
 			expMinutes:  1440.0,
+			expectedParcResults: map[string]ProportionalAssetResourceCosts{
+				"cluster1": {
+					"cluster1": ProportionalAssetResourceCost{
+						Cluster:             "cluster1",
+						Name:                "",
+						Type:                "",
+						ProviderID:          "",
+						GPUProportionalCost: 6.0,
+						CPUProportionalCost: 6.0,
+						RAMProportionalCost: 16.0,
+						PVProportionalCost:  6.0,
+					},
+				},
+				"cluster2": {
+					"cluster2": ProportionalAssetResourceCost{
+						Cluster:             "cluster2",
+						Name:                "",
+						Type:                "",
+						ProviderID:          "",
+						GPUProportionalCost: 6,
+						CPUProportionalCost: 6,
+						RAMProportionalCost: 6,
+						PVProportionalCost:  6,
+					},
+				},
+			},
 		},
 		// 1b AggregationProperties=(Namespace)
 		"1b": {
@@ -792,9 +821,11 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 		},
 		// 1c AggregationProperties=(Pod)
 		"1c": {
-			start:      start,
-			aggBy:      []string{AllocationPodProp},
-			aggOpts:    nil,
+			start: start,
+			aggBy: []string{AllocationPodProp},
+			aggOpts: &AllocationAggregationOptions{
+				IncludeProportionalAssetResourceCosts: true,
+			},
 			numResults: numPods + numIdle,
 			totalCost:  activeTotalCost + idleTotalCost,
 			results: map[string]float64{
@@ -812,6 +843,116 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			windowStart: startYesterday,
 			windowEnd:   endYesterday,
 			expMinutes:  1440.0,
+			expectedParcResults: map[string]ProportionalAssetResourceCosts{
+				"pod1": {
+					"cluster1": ProportionalAssetResourceCost{
+						Cluster:             "cluster1",
+						Name:                "",
+						Type:                "",
+						ProviderID:          "",
+						GPUProportionalCost: 1.0,
+						CPUProportionalCost: 1.0,
+						RAMProportionalCost: 11.0,
+						PVProportionalCost:  1.0,
+					},
+				},
+				"pod-abc": {
+					"cluster1": ProportionalAssetResourceCost{
+						Cluster:             "cluster1",
+						Name:                "",
+						Type:                "",
+						ProviderID:          "",
+						GPUProportionalCost: 1.0,
+						CPUProportionalCost: 1.0,
+						RAMProportionalCost: 1.0,
+						PVProportionalCost:  1.0,
+					},
+				},
+				"pod-def": {
+					"cluster1": ProportionalAssetResourceCost{
+						Cluster:             "cluster1",
+						Name:                "",
+						Type:                "",
+						ProviderID:          "",
+						GPUProportionalCost: 1.0,
+						CPUProportionalCost: 1.0,
+						RAMProportionalCost: 1.0,
+						PVProportionalCost:  1.0,
+					},
+				},
+				"pod-ghi": {
+					"cluster1": ProportionalAssetResourceCost{
+						Cluster:             "cluster1",
+						Name:                "",
+						Type:                "",
+						ProviderID:          "",
+						GPUProportionalCost: 2.0,
+						CPUProportionalCost: 2.0,
+						RAMProportionalCost: 2.0,
+						PVProportionalCost:  2.0,
+					},
+				},
+				"pod-jkl": {
+					"cluster1": ProportionalAssetResourceCost{
+						Cluster:             "cluster1",
+						Name:                "",
+						Type:                "",
+						ProviderID:          "",
+						GPUProportionalCost: 1.0,
+						CPUProportionalCost: 1.0,
+						RAMProportionalCost: 1.0,
+						PVProportionalCost:  1.0,
+					},
+				},
+				"pod-mno": {
+					"cluster2": ProportionalAssetResourceCost{
+						Cluster:             "cluster2",
+						Name:                "",
+						Type:                "",
+						ProviderID:          "",
+						GPUProportionalCost: 2.0,
+						CPUProportionalCost: 2.0,
+						RAMProportionalCost: 2.0,
+						PVProportionalCost:  2.0,
+					},
+				},
+				"pod-pqr": {
+					"cluster2": ProportionalAssetResourceCost{
+						Cluster:             "cluster2",
+						Name:                "",
+						Type:                "",
+						ProviderID:          "",
+						GPUProportionalCost: 1.0,
+						CPUProportionalCost: 1.0,
+						RAMProportionalCost: 1.0,
+						PVProportionalCost:  1.0,
+					},
+				},
+				"pod-stu": {
+					"cluster2": ProportionalAssetResourceCost{
+						Cluster:             "cluster2",
+						Name:                "",
+						Type:                "",
+						ProviderID:          "",
+						GPUProportionalCost: 1.0,
+						CPUProportionalCost: 1.0,
+						RAMProportionalCost: 1.0,
+						PVProportionalCost:  1.0,
+					},
+				},
+				"pod-vwx": {
+					"cluster2": ProportionalAssetResourceCost{
+						Cluster:             "cluster2",
+						Name:                "",
+						Type:                "",
+						ProviderID:          "",
+						GPUProportionalCost: 2.0,
+						CPUProportionalCost: 2.0,
+						RAMProportionalCost: 2.0,
+						PVProportionalCost:  2.0,
+					},
+				},
+			},
 		},
 		// 1d AggregationProperties=(Container)
 		"1d": {
@@ -1114,51 +1255,36 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			expectedParcResults: map[string]ProportionalAssetResourceCosts{
 				"namespace1": {
 					"cluster1": ProportionalAssetResourceCost{
-						Cluster:                    "cluster1",
-						Node:                       "",
-						ProviderID:                 "",
-						CPUPercentage:              0.16667,
-						GPUPercentage:              0.16667,
-						RAMPercentage:              0.27083,
-						NodeResourceCostPercentage: 0.22619,
-						GPUTotalCost:               18,
-						GPUProportionalCost:        3,
-						CPUTotalCost:               18,
-						CPUProportionalCost:        3,
-						RAMTotalCost:               48,
-						RAMProportionalCost:        13,
+						Cluster:             "cluster1",
+						Name:                "",
+						Type:                "",
+						ProviderID:          "",
+						GPUProportionalCost: 3,
+						CPUProportionalCost: 3,
+						RAMProportionalCost: 13,
+						PVProportionalCost:  3,
 					},
 				},
 				"namespace2": {
 					"cluster1": ProportionalAssetResourceCost{
-						Cluster:                    "cluster1",
-						Node:                       "",
-						ProviderID:                 "",
-						CPUPercentage:              0.16667,
-						GPUPercentage:              0.16667,
-						RAMPercentage:              0.0625,
-						NodeResourceCostPercentage: 0.10714,
-						GPUTotalCost:               18,
-						GPUProportionalCost:        3,
-						CPUTotalCost:               18,
-						CPUProportionalCost:        3,
-						RAMTotalCost:               48,
-						RAMProportionalCost:        3,
+						Cluster:             "cluster1",
+						Name:                "",
+						Type:                "",
+						ProviderID:          "",
+						GPUProportionalCost: 3,
+						CPUProportionalCost: 3,
+						RAMProportionalCost: 3,
+						PVProportionalCost:  3,
 					},
 					"cluster2": ProportionalAssetResourceCost{
-						Cluster:                    "cluster2",
-						Node:                       "",
-						ProviderID:                 "",
-						CPUPercentage:              0.16667,
-						GPUPercentage:              0.16667,
-						RAMPercentage:              0.16667,
-						NodeResourceCostPercentage: 0.16667,
-						GPUTotalCost:               18,
-						GPUProportionalCost:        3,
-						CPUTotalCost:               18,
-						CPUProportionalCost:        3,
-						RAMTotalCost:               18,
-						RAMProportionalCost:        3,
+						Cluster:             "cluster2",
+						Name:                "",
+						Type:                "",
+						ProviderID:          "",
+						GPUProportionalCost: 3,
+						CPUProportionalCost: 3,
+						RAMProportionalCost: 3,
+						PVProportionalCost:  3,
 					},
 				},
 			},
@@ -1566,113 +1692,143 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			expectedParcResults: map[string]ProportionalAssetResourceCosts{
 				"namespace1": {
 					"cluster1,c1nodes": ProportionalAssetResourceCost{
-						Cluster:                    "cluster1",
-						Node:                       "c1nodes",
-						ProviderID:                 "c1nodes",
-						CPUPercentage:              0.16667,
-						GPUPercentage:              0.16667,
-						RAMPercentage:              0.27083,
-						NodeResourceCostPercentage: 0.22619,
-						GPUTotalCost:               18,
-						GPUProportionalCost:        3,
-						CPUTotalCost:               18,
-						CPUProportionalCost:        3,
-						RAMTotalCost:               48,
-						RAMProportionalCost:        13,
+						Cluster:             "cluster1",
+						Name:                "c1nodes",
+						Type:                "Node",
+						ProviderID:          "c1nodes",
+						GPUProportionalCost: 3,
+						CPUProportionalCost: 3,
+						RAMProportionalCost: 13,
 					},
 					"cluster2,node2": ProportionalAssetResourceCost{
-						Cluster:                    "cluster2",
-						Node:                       "node2",
-						ProviderID:                 "node2",
-						CPUPercentage:              0.16667,
-						GPUPercentage:              0.16667,
-						RAMPercentage:              0.0625,
-						NodeResourceCostPercentage: 0.10714,
-						GPUTotalCost:               18,
-						GPUProportionalCost:        3,
-						CPUTotalCost:               18,
-						CPUProportionalCost:        3,
-						RAMTotalCost:               48,
-						RAMProportionalCost:        3,
+						Cluster:             "cluster2",
+						Name:                "node2",
+						Type:                "Node",
+						ProviderID:          "node2",
+						GPUProportionalCost: 3,
+						CPUProportionalCost: 3,
+						RAMProportionalCost: 3,
+					},
+					"cluster1,pv-a1111": {
+						Cluster:            "cluster1",
+						Name:               "pv-a1111",
+						Type:               "PV",
+						PVProportionalCost: 1,
+					},
+					"cluster1,pv-a11abc2": {
+						Cluster:            "cluster1",
+						Name:               "pv-a11abc2",
+						Type:               "PV",
+						PVProportionalCost: 1,
+					},
+					"cluster1,pv-a11def3": {
+						Cluster:            "cluster1",
+						Name:               "pv-a11def3",
+						Type:               "PV",
+						PVProportionalCost: 1,
 					},
 				},
 				"namespace2": {
 					"cluster1,c1nodes": ProportionalAssetResourceCost{
-						Cluster:                    "cluster1",
-						Node:                       "c1nodes",
-						ProviderID:                 "c1nodes",
-						CPUPercentage:              0.16667,
-						GPUPercentage:              0.16667,
-						RAMPercentage:              0.0625,
-						NodeResourceCostPercentage: 0.10714,
-						GPUTotalCost:               18,
-						GPUProportionalCost:        3,
-						CPUTotalCost:               18,
-						CPUProportionalCost:        3,
-						RAMTotalCost:               48,
-						RAMProportionalCost:        3,
+						Cluster:             "cluster1",
+						Name:                "c1nodes",
+						Type:                "Node",
+						ProviderID:          "c1nodes",
+						GPUProportionalCost: 3,
+						CPUProportionalCost: 3,
+						RAMProportionalCost: 3,
 					},
 					"cluster2,node1": ProportionalAssetResourceCost{
-						Cluster:                    "cluster2",
-						Node:                       "node1",
-						ProviderID:                 "node1",
-						CPUPercentage:              0.5,
-						GPUPercentage:              0.5,
-						RAMPercentage:              0.5,
-						NodeResourceCostPercentage: 0.5,
-						GPUTotalCost:               4,
-						GPUProportionalCost:        2,
-						CPUTotalCost:               4,
-						CPUProportionalCost:        2,
-						RAMTotalCost:               4,
-						RAMProportionalCost:        2,
+						Cluster:             "cluster2",
+						Name:                "node1",
+						Type:                "Node",
+						ProviderID:          "node1",
+						GPUProportionalCost: 2,
+						CPUProportionalCost: 2,
+						RAMProportionalCost: 2,
 					},
 					"cluster2,node2": ProportionalAssetResourceCost{
-						Cluster:                    "cluster2",
-						Node:                       "node2",
-						ProviderID:                 "node2",
-						CPUPercentage:              0.5,
-						GPUPercentage:              0.5,
-						RAMPercentage:              0.5,
-						NodeResourceCostPercentage: 0.5,
-						GPUTotalCost:               2,
-						GPUProportionalCost:        1,
-						CPUTotalCost:               2,
-						CPUProportionalCost:        1,
-						RAMTotalCost:               2,
-						RAMProportionalCost:        1,
+						Cluster:             "cluster2",
+						Name:                "node2",
+						Type:                "Node",
+						ProviderID:          "node2",
+						GPUProportionalCost: 1,
+						CPUProportionalCost: 1,
+						RAMProportionalCost: 1,
+					},
+					"cluster1,pv-a12ghi4": {
+						Cluster:            "cluster1",
+						Name:               "pv-a12ghi4",
+						Type:               "PV",
+						PVProportionalCost: 1,
+					},
+					"cluster1,pv-a12ghi5": {
+						Cluster:            "cluster1",
+						Name:               "pv-a12ghi5",
+						Type:               "PV",
+						PVProportionalCost: 1,
+					},
+					"cluster1,pv-a12jkl6": {
+						Cluster:            "cluster1",
+						Name:               "pv-a12jkl6",
+						Type:               "PV",
+						PVProportionalCost: 1,
+					},
+					"cluster2,pv-a22mno4": {
+						Cluster:            "cluster2",
+						Name:               "pv-a22mno4",
+						Type:               "PV",
+						PVProportionalCost: 1,
+					},
+					"cluster2,pv-a22mno5": {
+						Cluster:            "cluster2",
+						Name:               "pv-a22mno5",
+						Type:               "PV",
+						PVProportionalCost: 1,
+					},
+					"cluster2,pv-a22pqr6": {
+						Cluster:            "cluster2",
+						Name:               "pv-a22pqr6",
+						Type:               "PV",
+						PVProportionalCost: 1,
 					},
 				},
 				"namespace3": {
 					"cluster2,node3": ProportionalAssetResourceCost{
-						Cluster:                    "cluster2",
-						Node:                       "node3",
-						ProviderID:                 "node3",
-						CPUPercentage:              0.5,
-						GPUPercentage:              0.5,
-						RAMPercentage:              0.5,
-						NodeResourceCostPercentage: 0.5,
-						GPUTotalCost:               4,
-						GPUProportionalCost:        2,
-						CPUTotalCost:               4,
-						CPUProportionalCost:        2,
-						RAMTotalCost:               4,
-						RAMProportionalCost:        2,
+						Cluster:             "cluster2",
+						Name:                "node3",
+						Type:                "Node",
+						ProviderID:          "node3",
+						GPUProportionalCost: 2,
+						CPUProportionalCost: 2,
+						RAMProportionalCost: 2,
 					},
 					"cluster2,node2": ProportionalAssetResourceCost{
-						Cluster:                    "cluster2",
-						Node:                       "node2",
-						ProviderID:                 "node2",
-						CPUPercentage:              0.5,
-						GPUPercentage:              0.5,
-						RAMPercentage:              0.5,
-						NodeResourceCostPercentage: 0.5,
-						GPUTotalCost:               2,
-						GPUProportionalCost:        1,
-						CPUTotalCost:               2,
-						CPUProportionalCost:        1,
-						RAMTotalCost:               2,
-						RAMProportionalCost:        1,
+						Cluster:             "cluster2",
+						Name:                "node2",
+						Type:                "Node",
+						ProviderID:          "node2",
+						GPUProportionalCost: 1,
+						CPUProportionalCost: 1,
+						RAMProportionalCost: 1,
+					},
+					"cluster2,pv-a23stu7": {
+						Cluster:            "cluster2",
+						Name:               "pv-a23stu7",
+						Type:               "PV",
+						PVProportionalCost: 1,
+					},
+					"cluster2,pv-a23vwx8": {
+						Cluster:            "cluster2",
+						Name:               "pv-a23vwx8",
+						Type:               "PV",
+						PVProportionalCost: 1,
+					},
+					"cluster2,pv-a23vwx9": {
+						Cluster:            "cluster2",
+						Name:               "pv-a23vwx9",
+						Type:               "PV",
+						PVProportionalCost: 1,
 					},
 				},
 			},
@@ -1951,7 +2107,8 @@ func TestAllocationSet_insertMatchingWindow(t *testing.T) {
 func TestParcInsert(t *testing.T) {
 	pod1_hour1 := ProportionalAssetResourceCost{
 		Cluster:                    "cluster1",
-		Node:                       "node1",
+		Name:                       "node1",
+		Type:                       "Node",
 		ProviderID:                 "i-1234",
 		CPUPercentage:              0.125,
 		GPUPercentage:              0,
@@ -1963,7 +2120,8 @@ func TestParcInsert(t *testing.T) {
 
 	pod1_hour2 := ProportionalAssetResourceCost{
 		Cluster:                    "cluster1",
-		Node:                       "node1",
+		Name:                       "node1",
+		Type:                       "Node",
 		ProviderID:                 "i-1234",
 		CPUPercentage:              0.0,
 		GPUPercentage:              0,
@@ -1974,7 +2132,8 @@ func TestParcInsert(t *testing.T) {
 
 	pod1_hour3 := ProportionalAssetResourceCost{
 		Cluster:                    "cluster1",
-		Node:                       "node1",
+		Name:                       "node1",
+		Type:                       "Node",
 		ProviderID:                 "i-1234",
 		CPUPercentage:              0.0,
 		GPUPercentage:              0,
@@ -1985,7 +2144,8 @@ func TestParcInsert(t *testing.T) {
 
 	pod2_hour1 := ProportionalAssetResourceCost{
 		Cluster:                    "cluster1",
-		Node:                       "node2",
+		Name:                       "node2",
+		Type:                       "Node",
 		ProviderID:                 "i-1234",
 		CPUPercentage:              0.0,
 		GPUPercentage:              0,
@@ -1997,7 +2157,8 @@ func TestParcInsert(t *testing.T) {
 
 	pod2_hour2 := ProportionalAssetResourceCost{
 		Cluster:                    "cluster1",
-		Node:                       "node2",
+		Name:                       "node2",
+		Type:                       "Node",
 		ProviderID:                 "i-1234",
 		CPUPercentage:              0.0,
 		GPUPercentage:              0,
@@ -2009,7 +2170,8 @@ func TestParcInsert(t *testing.T) {
 
 	pod2_hour3 := ProportionalAssetResourceCost{
 		Cluster:                    "cluster1",
-		Node:                       "node2",
+		Name:                       "node2",
+		Type:                       "Node",
 		ProviderID:                 "i-1234",
 		CPUPercentage:              0.0,
 		GPUPercentage:              0,
@@ -2021,7 +2183,8 @@ func TestParcInsert(t *testing.T) {
 
 	pod3_hour1 := ProportionalAssetResourceCost{
 		Cluster:                    "cluster1",
-		Node:                       "node3",
+		Name:                       "node3",
+		Type:                       "Node",
 		ProviderID:                 "i-1234",
 		CPUPercentage:              0.0,
 		GPUPercentage:              0,
@@ -2033,7 +2196,8 @@ func TestParcInsert(t *testing.T) {
 
 	pod3_hour2 := ProportionalAssetResourceCost{
 		Cluster:                    "cluster1",
-		Node:                       "node3",
+		Name:                       "node3",
+		Type:                       "Node",
 		ProviderID:                 "i-1234",
 		CPUPercentage:              0.0,
 		GPUPercentage:              0,
@@ -2045,7 +2209,8 @@ func TestParcInsert(t *testing.T) {
 
 	pod3_hour3 := ProportionalAssetResourceCost{
 		Cluster:                    "cluster1",
-		Node:                       "node3",
+		Name:                       "node3",
+		Type:                       "Node",
 		ProviderID:                 "i-1234",
 		CPUPercentage:              0.0,
 		GPUPercentage:              0,
@@ -2067,6 +2232,32 @@ func TestParcInsert(t *testing.T) {
 	parcs.Insert(pod3_hour3, true)
 	log.Debug("added all parcs")
 
+	// set totals, compute percentaves
+	parc1, ok := parcs["cluster1,node1"]
+	if !ok {
+		t.Fatalf("parc1 not found")
+	}
+	parc1.CPUTotalCost = 12
+
+	parc2, ok := parcs["cluster1,node2"]
+	if !ok {
+		t.Fatalf("parc2 not found")
+	}
+	parc2.CPUTotalCost = 12
+
+	parc3, ok := parcs["cluster1,node3"]
+	if !ok {
+		t.Fatalf("parc1 not found")
+	}
+	parc3.CPUTotalCost = 12
+
+	ComputePercentages(&parc1)
+	ComputePercentages(&parc2)
+	ComputePercentages(&parc3)
+	parcs["cluster1,node1"] = parc1
+	parcs["cluster1,node2"] = parc2
+	parcs["cluster1,node3"] = parc3
+
 	expectedParcs := ProportionalAssetResourceCosts{
 		"cluster1,node1": ProportionalAssetResourceCost{
 			CPUPercentage:              0.041666666666666664,
@@ -3461,3 +3652,240 @@ func TestIsFilterEmptyFalse(t *testing.T) {
 		t.Errorf("matcher '%+v' should be not be reported empty but was", matcher)
 	}
 }
+
+func TestAllocation_SanitizeNaN(t *testing.T) {
+	tcName := "TestAllocation_SanitizeNaN"
+	alloc := getMockAllocation(math.NaN())
+	alloc.SanitizeNaN()
+	checkAllocation(t, tcName, alloc)
+}
+
+func checkAllocation(t *testing.T, tcName string, alloc Allocation) {
+	v := reflect.ValueOf(alloc)
+	checkAllFloat64sForNaN(t, v, tcName)
+
+	vRaw := reflect.ValueOf(*alloc.RawAllocationOnly)
+	checkAllFloat64sForNaN(t, vRaw, tcName)
+
+	for _, pv := range alloc.PVs {
+		vPV := reflect.ValueOf(*pv)
+		checkAllFloat64sForNaN(t, vPV, tcName)
+	}
+
+	for _, parc := range alloc.ProportionalAssetResourceCosts {
+		vParc := reflect.ValueOf(parc)
+		checkAllFloat64sForNaN(t, vParc, tcName)
+	}
+
+	for _, scb := range alloc.SharedCostBreakdown {
+		vScb := reflect.ValueOf(scb)
+		checkAllFloat64sForNaN(t, vScb, tcName)
+	}
+
+	for _, lb := range alloc.LoadBalancers {
+		vLb := reflect.ValueOf(*lb)
+		checkAllFloat64sForNaN(t, vLb, tcName)
+	}
+}
+
+func TestAllocationSet_SanitizeNaN(t *testing.T) {
+	allocNaN := getMockAllocation(math.NaN())
+	allocNotNaN := getMockAllocation(1.2)
+	allocSet := AllocationSet{
+		Allocations: map[string]*Allocation{"NaN": &allocNaN, "notNaN": &allocNotNaN},
+	}
+
+	allocSet.SanitizeNaN()
+
+	for _, a := range allocSet.Allocations {
+		checkAllocation(t, "TestAllocationSet_SanitizeNaN", *a)
+	}
+
+}
+
+func getMockAllocation(f float64) Allocation {
+	alloc := Allocation{
+		Name:                           "mockAllocation",
+		Properties:                     nil,
+		Window:                         Window{},
+		Start:                          time.Time{},
+		End:                            time.Time{},
+		CPUCoreHours:                   f,
+		CPUCoreRequestAverage:          f,
+		CPUCoreUsageAverage:            f,
+		CPUCost:                        f,
+		CPUCostAdjustment:              f,
+		GPUHours:                       f,
+		GPUCost:                        f,
+		GPUCostAdjustment:              f,
+		NetworkTransferBytes:           f,
+		NetworkReceiveBytes:            f,
+		NetworkCost:                    f,
+		NetworkCrossZoneCost:           f,
+		NetworkCrossRegionCost:         f,
+		NetworkInternetCost:            f,
+		NetworkCostAdjustment:          f,
+		LoadBalancerCost:               f,
+		LoadBalancerCostAdjustment:     f,
+		PVs:                            PVAllocations{{Cluster: "testPV", Name: "PVName"}: getMockPVAllocation(math.NaN())},
+		PVCostAdjustment:               f,
+		RAMByteHours:                   f,
+		RAMBytesRequestAverage:         f,
+		RAMBytesUsageAverage:           f,
+		RAMCost:                        f,
+		RAMCostAdjustment:              f,
+		SharedCost:                     f,
+		ExternalCost:                   f,
+		RawAllocationOnly:              getMockRawAllocationOnlyData(f),
+		ProportionalAssetResourceCosts: ProportionalAssetResourceCosts{"NaN": *getMockPARC(f)},
+		SharedCostBreakdown:            SharedCostBreakdowns{"NaN": *getMockSharedCostBreakdown(f)},
+		LoadBalancers:                  LbAllocations{"NaN": getMockLbAllocation(f)},
+	}
+	return alloc
+}
+
+func TestPVAllocation_SanitizeNaN(t *testing.T) {
+	pva := getMockPVAllocation(math.NaN())
+	pva.SanitizeNaN()
+	v := reflect.ValueOf(*pva)
+	checkAllFloat64sForNaN(t, v, "TestPVAllocation_SanitizeNaN")
+}
+
+func TestPVAllocations_SanitizeNaN(t *testing.T) {
+	pvaNaN := getMockPVAllocation(math.NaN())
+	pvaNotNaN := getMockPVAllocation(1.2)
+	pvs := PVAllocations{{Cluster: "testPV", Name: "PVName1"}: pvaNaN, {Cluster: "testPV", Name: "PVName2"}: pvaNotNaN}
+	pvs.SanitizeNaN()
+	for _, pv := range pvs {
+		v := reflect.ValueOf(*pv)
+		checkAllFloat64sForNaN(t, v, "TestPVAllocations_SanitizeNaN")
+	}
+
+}
+
+func getMockPVAllocation(f float64) *PVAllocation {
+	return &PVAllocation{
+		ByteHours: f,
+		Cost:      f,
+	}
+}
+
+func TestRawAllocationOnlyData_SanitizeNaN(t *testing.T) {
+	raw := getMockRawAllocationOnlyData(math.NaN())
+	raw.SanitizeNaN()
+	v := reflect.ValueOf(*raw)
+	checkAllFloat64sForNaN(t, v, "TestRawAllocationOnlyData_SanitizeNaN")
+}
+
+func getMockRawAllocationOnlyData(f float64) *RawAllocationOnlyData {
+	return &RawAllocationOnlyData{
+		CPUCoreUsageMax:  f,
+		RAMBytesUsageMax: f,
+	}
+}
+
+func TestLbAllocation_SanitizeNaN(t *testing.T) {
+	lbaNaN := getMockLbAllocation(math.NaN())
+	lbaNaN.SanitizeNaN()
+	v := reflect.ValueOf(*lbaNaN)
+	checkAllFloat64sForNaN(t, v, "TestLbAllocation_SanitizeNaN")
+}
+
+func TestLbAllocations_SanitizeNaN(t *testing.T) {
+	lbaNaN := getMockLbAllocation(math.NaN())
+	lbaValid := getMockLbAllocation(1.2)
+
+	lbas := LbAllocations{"NaN": lbaNaN, "notNaN": lbaValid}
+	lbas.SanitizeNaN()
+	for _, lba := range lbas {
+		v := reflect.ValueOf(*lba)
+		checkAllFloat64sForNaN(t, v, "TestLbAllocations_SanitizeNaN")
+	}
+}
+
+func getMockLbAllocation(f float64) *LbAllocation {
+	return &LbAllocation{
+		Service: "testLoadBalancer",
+		Cost:    f,
+		Private: false,
+	}
+}
+
+func TestProportionalAssetResourceCosts_SanitizeNaN(t *testing.T) {
+	parcAllNaN := getMockPARC(math.NaN())
+	parcNotNaN := getMockPARC(1.2)
+
+	parcs := ProportionalAssetResourceCosts{"NaN": *parcAllNaN, "notNaN": *parcNotNaN}
+	parcs.SanitizeNaN()
+
+	for _, parc := range parcs {
+		v := reflect.ValueOf(parc)
+		checkAllFloat64sForNaN(t, v, "TestProportionalAssetResourceCosts_SanitizeNaN")
+	}
+}
+
+func getMockPARC(f float64) *ProportionalAssetResourceCost {
+	return &ProportionalAssetResourceCost{
+		Cluster:                      "testCluster",
+		Name:                         "testName",
+		Type:                         "testType",
+		ProviderID:                   "testProvider",
+		CPUPercentage:                f,
+		GPUPercentage:                f,
+		RAMPercentage:                f,
+		LoadBalancerPercentage:       f,
+		PVPercentage:                 f,
+		NodeResourceCostPercentage:   f,
+		GPUTotalCost:                 f,
+		GPUProportionalCost:          f,
+		CPUTotalCost:                 f,
+		CPUProportionalCost:          f,
+		RAMTotalCost:                 f,
+		RAMProportionalCost:          f,
+		LoadBalancerProportionalCost: f,
+		LoadBalancerTotalCost:        f,
+		PVProportionalCost:           f,
+		PVTotalCost:                  f,
+	}
+}
+
+func TestSharedCostBreakdowns_SanitizeNaN(t *testing.T) {
+	scbNaN := getMockSharedCostBreakdown(math.NaN())
+	scbNotNaN := getMockSharedCostBreakdown(1.2)
+
+	scbs := SharedCostBreakdowns{"NaN": *scbNaN, "notNaN": *scbNotNaN}
+	scbs.SanitizeNaN()
+	for _, scb := range scbs {
+		v := reflect.ValueOf(scb)
+		checkAllFloat64sForNaN(t, v, "TestSharedCostBreakdowns_SanitizeNaN")
+	}
+}
+
+func getMockSharedCostBreakdown(f float64) *SharedCostBreakdown {
+	return &SharedCostBreakdown{
+		Name:         "testBreakdown",
+		TotalCost:    f,
+		CPUCost:      f,
+		GPUCost:      f,
+		RAMCost:      f,
+		PVCost:       f,
+		NetworkCost:  f,
+		LBCost:       f,
+		ExternalCost: f,
+	}
+}
+
+func checkAllFloat64sForNaN(t *testing.T, v reflect.Value, testCaseName string) {
+	vType := v.Type()
+
+	// go through each field on the struct
+	for i := 0; i < v.NumField(); i++ {
+		// Check if field is public and can be converted to a float
+		if v.Field(i).CanInterface() && v.Field(i).CanFloat() {
+			f := v.Field(i).Float()
+			if math.IsNaN(f) {
+				t.Fatalf("%s: expected not NaN for field: %s, got:NaN", testCaseName, vType.Field(i).Name)
+			}
+		}
+	}
+}

pkg/kubecost/allocationfilter_test.go

@@ -440,11 +440,11 @@ func Test_AllocationFilterCondition_Matches(t *testing.T) {
 			expected: true,
 		},
 		{
-			name: `product != unallocated -> true`,
+			name: `department != unallocated -> true`,
 			a: &Allocation{
 				Properties: &AllocationProperties{
 					Annotations: AllocationAnnotations{
-						"keyproduct": "foo",
+						"keydepartment": "foo",
 					},
 				},
 			},
@@ -460,6 +460,43 @@ func Test_AllocationFilterCondition_Matches(t *testing.T) {
 			},
 			expected: true,
 		},
+		{
+			name: `product == unallocated -> true`,
+			a: &Allocation{
+				Properties: &AllocationProperties{
+					Annotations: AllocationAnnotations{
+						"keydepartment": "foo",
+					},
+				},
+			},
+			filter: &ast.EqualOp{
+				Left: ast.Identifier{
+					Field: ast.NewAliasField(afilter.AliasProduct),
+				},
+				Right: UnallocatedSuffix,
+			},
+			expected: true,
+		},
+		{
+			name: `product == "" -> true`,
+			a: &Allocation{
+				Properties: &AllocationProperties{
+					Labels: AllocationLabels{
+						"keydepartment": "foo",
+					},
+					Annotations: AllocationAnnotations{
+						"keyowner": "bar",
+					},
+				},
+			},
+			filter: &ast.EqualOp{
+				Left: ast.Identifier{
+					Field: ast.NewAliasField(afilter.AliasProduct),
+				},
+				Right: "",
+			},
+			expected: true,
+		},
 	}
 
 	for _, c := range cases {

pkg/kubecost/allocationmatcher.go

@@ -29,7 +29,7 @@ func NewAllocationMatchCompiler(labelConfig *LabelConfig) *matcher.MatchCompiler
 
 	// The label config pass should be the first pass
 	if labelConfig != nil {
-		passes = append(passes, NewAliasPass(*labelConfig))
+		passes = append(passes, NewAllocationAliasPass(*labelConfig))
 	}
 
 	passes = append(passes,
@@ -46,6 +46,12 @@ func NewAllocationMatchCompiler(labelConfig *LabelConfig) *matcher.MatchCompiler
 
 // Maps fields from an allocation to a string value based on an identifier
 func allocationFieldMap(a *Allocation, identifier ast.Identifier) (string, error) {
+	if a == nil {
+		return "", fmt.Errorf("cannot map to nil allocation")
+	}
+	if a.Properties == nil {
+		return "", fmt.Errorf("cannot map to nil properties")
+	}
 	if identifier.Field == nil {
 		return "", fmt.Errorf("cannot map field from identifier with nil field")
 	}
@@ -96,10 +102,10 @@ func allocationMapFieldMap(a *Allocation, identifier ast.Identifier) (map[string
 	return nil, fmt.Errorf("Failed to find map[string]string identifier on Allocation: %s", identifier.Field.Name)
 }
 
-// aliasPass implements the transform.CompilerPass interface, providing a pass
-// which converts alias nodes to logically-equivalent label/annotation filter
-// nodes based on the label config.
-type aliasPass struct {
+// allocatioAliasPass implements the transform.CompilerPass interface, providing
+// a pass which converts alias nodes to logically-equivalent label/annotation
+// filter nodes based on the label config.
+type allocationAliasPass struct {
 	Config              LabelConfig
 	AliasNameToAliasKey map[afilter.AllocationAlias]string
 }
@@ -118,7 +124,7 @@ type aliasPass struct {
 //	(and (not (contains labels <parseraliaskey>))
 //	     (and (contains annotations departmentkey)
 //	          (<op> annotations[<parseraliaskey>] <filtervalue>))))
-func NewAliasPass(config LabelConfig) transform.CompilerPass {
+func NewAllocationAliasPass(config LabelConfig) transform.CompilerPass {
 	aliasNameToAliasKey := map[afilter.AllocationAlias]string{
 		afilter.AliasDepartment:  config.DepartmentLabel,
 		afilter.AliasEnvironment: config.EnvironmentLabel,
@@ -127,7 +133,7 @@ func NewAliasPass(config LabelConfig) transform.CompilerPass {
 		afilter.AliasTeam:        config.TeamLabel,
 	}
 
-	return &aliasPass{
+	return &allocationAliasPass{
 		Config:              config,
 		AliasNameToAliasKey: aliasNameToAliasKey,
 	}
@@ -135,7 +141,7 @@ func NewAliasPass(config LabelConfig) transform.CompilerPass {
 
 // Exec implements the transform.CompilerPass interface for an alias pass.
 // See aliasPass struct documentation for an explanation.
-func (p *aliasPass) Exec(filter ast.FilterNode) (ast.FilterNode, error) {
+func (p *allocationAliasPass) Exec(filter ast.FilterNode) (ast.FilterNode, error) {
 	if p.AliasNameToAliasKey == nil {
 		return nil, fmt.Errorf("cannot perform alias conversion with nil mapping of alias name -> key")
 	}
@@ -238,7 +244,9 @@ func convertAliasFilterToLabelAnnotationFilter(aliasKey string, filterValue stri
 		return nil, fmt.Errorf("unsupported op type '%s' for alias conversion", op)
 	}
 
-	return ops.Or(
+	// This handles the case where a label EXISTS/IS PRESENT for (is extant)
+	// for an aliased field. That's the primary case.
+	extantCaseNode := ops.Or(
 		ops.And(
 			ops.Contains(afilter.FieldLabel, aliasKey),
 			labelOp,
@@ -250,5 +258,27 @@ func convertAliasFilterToLabelAnnotationFilter(aliasKey string, filterValue stri
 				annotationOp,
 			),
 		),
-	), nil
+	)
+	var node ast.FilterNode
+	// This handles the special case of unallocated aliased value. There's
+	// two forms of this; first is where the label/annotation exists, but
+	// has an empty string value. That's actually handled by the extant case,
+	// because the API passes through that empty string. The other is when
+	// the aliased label/annotation doesn't exist for an allocation. That's
+	// what this modification to the tree handles. This matters when you're
+	// trying to drill into/identify workloads "not allocated" within that
+	// specific aliased field.
+	if filterValue == "" || filterValue == UnallocatedSuffix {
+		node = ops.Or(
+			extantCaseNode,
+			ops.And(
+				ops.Not(ops.Contains(afilter.FieldLabel, aliasKey)),
+				ops.Not(ops.Contains(afilter.FieldAnnotation, aliasKey)),
+			),
+		)
+	} else {
+		node = extantCaseNode
+	}
+
+	return node, nil
 }

pkg/kubecost/allocationmatcher_test.go

@@ -48,7 +48,7 @@ func TestAliasPass(t *testing.T) {
 	}
 
 	for _, c := range cases {
-		pass := NewAliasPass(*labelConfig)
+		pass := NewAllocationAliasPass(*labelConfig)
 
 		t.Run(c.name, func(t *testing.T) {
 			result, err := pass.Exec(c.input)

pkg/kubecost/allocationprops.go

@@ -161,6 +161,7 @@ func (p *AllocationProperties) Clone() *AllocationProperties {
 	}
 	clone.NamespaceAnnotations = nsAnnotations
 
+	clone.AggregatedMetadata = p.AggregatedMetadata
 	return clone
 }
 

pkg/kubecost/asset.go

@@ -8,6 +8,9 @@ import (
 	"strings"
 	"time"
 
+	filter21 "github.com/opencost/opencost/pkg/filter21"
+	"github.com/opencost/opencost/pkg/filter21/ast"
+	"github.com/opencost/opencost/pkg/filter21/matcher"
 	"github.com/opencost/opencost/pkg/log"
 	"github.com/opencost/opencost/pkg/util/json"
 	"github.com/opencost/opencost/pkg/util/timeutil"
@@ -57,6 +60,7 @@ type Asset interface {
 	Add(Asset) Asset
 	Clone() Asset
 	Equal(Asset) bool
+	SanitizeNaN()
 
 	// Representations
 	encoding.BinaryMarshaler
@@ -422,10 +426,6 @@ func (al AssetLabels) Append(newLabels map[string]string, overwrite bool) {
 	}
 }
 
-// AssetMatchFunc is a function that can be used to match Assets by
-// returning true for any given Asset if a condition is met.
-type AssetMatchFunc func(Asset) bool
-
 // AssetType identifies a type of Asset
 type AssetType int
 
@@ -675,6 +675,20 @@ func (a *Any) String() string {
 	return toString(a)
 }
 
+func (a *Any) SanitizeNaN() {
+	if a == nil {
+		return
+	}
+	if math.IsNaN(a.Adjustment) {
+		log.DedupedWarningf(5, "Any: Unexpected NaN found for Adjustment: labels:%v, window:%s, properties:%s", a.Labels, a.Window.String(), a.Properties.String())
+		a.Adjustment = 0
+	}
+	if math.IsNaN(a.Cost) {
+		log.DedupedWarningf(5, "Any: Unexpected NaN found for Cost: name:%v, window:%s, properties:%s", a.Labels, a.Window.String(), a.Properties.String())
+		a.Cost = 0
+	}
+}
+
 // Cloud describes a cloud asset
 type Cloud struct {
 	Labels     AssetLabels
@@ -903,6 +917,24 @@ func (ca *Cloud) String() string {
 	return toString(ca)
 }
 
+func (ca *Cloud) SanitizeNaN() {
+	if ca == nil {
+		return
+	}
+	if math.IsNaN(ca.Adjustment) {
+		log.DedupedWarningf(5, "Cloud: Unexpected NaN found for Adjustment: labels:%v, window:%s, properties:%s", ca.Labels, ca.Window.String(), ca.Properties.String())
+		ca.Adjustment = 0
+	}
+	if math.IsNaN(ca.Cost) {
+		log.DedupedWarningf(5, "Cloud: Unexpected NaN found for Cost: name:%v, window:%s, properties:%s", ca.Labels, ca.Window.String(), ca.Properties.String())
+		ca.Cost = 0
+	}
+	if math.IsNaN(ca.Credit) {
+		log.DedupedWarningf(5, "Cloud: Unexpected NaN found for Credit: name:%v, window:%s, properties:%s", ca.Labels, ca.Window.String(), ca.Properties.String())
+		ca.Credit = 0
+	}
+}
+
 // ClusterManagement describes a provider's cluster management fee
 type ClusterManagement struct {
 	Labels     AssetLabels
@@ -1097,6 +1129,20 @@ func (cm *ClusterManagement) String() string {
 	return toString(cm)
 }
 
+func (cm *ClusterManagement) SanitizeNaN() {
+	if cm == nil {
+		return
+	}
+	if math.IsNaN(cm.Adjustment) {
+		log.DedupedWarningf(5, "ClusterManagement: Unexpected NaN found for Adjustment: labels:%v, window:%s, properties:%s", cm.Labels, cm.Window.String(), cm.Properties.String())
+		cm.Adjustment = 0
+	}
+	if math.IsNaN(cm.Cost) {
+		log.DedupedWarningf(5, "ClusterManagement: Unexpected NaN found for Cost: name:%v, window:%s, properties:%s", cm.Labels, cm.Window.String(), cm.Properties.String())
+		cm.Cost = 0
+	}
+}
+
 // Disk represents an in-cluster disk Asset
 type Disk struct {
 	Labels         AssetLabels
@@ -1466,6 +1512,40 @@ func (d *Disk) Bytes() float64 {
 	return d.ByteHours * (60.0 / d.Minutes())
 }
 
+func (d *Disk) SanitizeNaN() {
+	if d == nil {
+		return
+	}
+	if math.IsNaN(d.Adjustment) {
+		log.DedupedWarningf(5, "Disk: Unexpected NaN found for Adjustment: labels:%v, window:%s, properties:%s", d.Labels, d.Window.String(), d.Properties.String())
+		d.Adjustment = 0
+	}
+	if math.IsNaN(d.Cost) {
+		log.DedupedWarningf(5, "Disk: Unexpected NaN found for Cost: labels:%v, window:%s, properties:%s", d.Labels, d.Window.String(), d.Properties.String())
+		d.Cost = 0
+	}
+	if math.IsNaN(d.ByteHours) {
+		log.DedupedWarningf(5, "Disk: Unexpected NaN found for ByteHours: labels:%v, window:%s, properties:%s", d.Labels, d.Window.String(), d.Properties.String())
+		d.ByteHours = 0
+	}
+	if math.IsNaN(d.Local) {
+		log.DedupedWarningf(5, "Disk: Unexpected NaN found for Local: labels:%v, window:%s, properties:%s", d.Labels, d.Window.String(), d.Properties.String())
+		d.Local = 0
+	}
+	if d.ByteHoursUsed != nil && math.IsNaN(*d.ByteHoursUsed) {
+		log.DedupedWarningf(5, "Disk: Unexpected NaN found for ByteHoursUsed: labels:%v, window:%s, properties:%s", d.Labels, d.Window.String(), d.Properties.String())
+		f := 0.0
+		d.ByteHoursUsed = &f
+	}
+	if d.ByteUsageMax != nil && math.IsNaN(*d.ByteUsageMax) {
+		log.DedupedWarningf(5, "Disk: Unexpected NaN found for ByteUsageMax: labels:%v, window:%s, properties:%s", d.Labels, d.Window.String(), d.Properties.String())
+		f := 0.0
+		d.ByteUsageMax = &f
+	}
+
+	d.Breakdown.SanitizeNaN()
+}
+
 // Breakdown describes a resource's use as a percentage of various usage types
 type Breakdown struct {
 	Idle   float64 `json:"idle"`
@@ -1474,6 +1554,25 @@ type Breakdown struct {
 	User   float64 `json:"user"`
 }
 
+func (b *Breakdown) SanitizeNaN() {
+	if math.IsNaN(b.Idle) {
+		log.DedupedWarningf(5, "Breakdown: Unexpected NaN found for Idle")
+		b.Idle = 0
+	}
+	if math.IsNaN(b.Other) {
+		log.DedupedWarningf(5, "Breakdown: Unexpected NaN found for Other")
+		b.Other = 0
+	}
+	if math.IsNaN(b.System) {
+		log.DedupedWarningf(5, "Breakdown: Unexpected NaN found for System")
+		b.System = 0
+	}
+	if math.IsNaN(b.User) {
+		log.DedupedWarningf(5, "Breakdown: Unexpected NaN found for User")
+		b.User = 0
+	}
+}
+
 // Clone returns a cloned instance of the Breakdown
 func (b *Breakdown) Clone() *Breakdown {
 	if b == nil {
@@ -1753,6 +1852,20 @@ func (n *Network) String() string {
 	return toString(n)
 }
 
+func (n *Network) SanitizeNaN() {
+	if n == nil {
+		return
+	}
+	if math.IsNaN(n.Adjustment) {
+		log.DedupedWarningf(5, "Network: Unexpected NaN found for Adjustment: labels:%v, window:%s, properties:%s", n.Labels, n.Window.String(), n.Properties.String())
+		n.Adjustment = 0
+	}
+	if math.IsNaN(n.Cost) {
+		log.DedupedWarningf(5, "Network: Unexpected NaN found for Cost: labels:%v, window:%s, properties:%s", n.Labels, n.Window.String(), n.Properties.String())
+		n.Cost = 0
+	}
+}
+
 // NodeOverhead represents the delta between the allocatable resources
 // of the node and the node nameplate capacity
 type NodeOverhead struct {
@@ -1761,6 +1874,21 @@ type NodeOverhead struct {
 	OverheadCostFraction float64
 }
 
+func (n *NodeOverhead) SanitizeNaN() {
+	if math.IsNaN(n.CpuOverheadFraction) {
+		log.DedupedWarningf(5, "NodeOverhead: Unexpected NaN found for CpuOverheadFraction")
+		n.CpuOverheadFraction = 0
+	}
+	if math.IsNaN(n.RamOverheadFraction) {
+		log.DedupedWarningf(5, "NodeOverhead: Unexpected NaN found for RamOverheadFraction")
+		n.RamOverheadFraction = 0
+	}
+	if math.IsNaN(n.OverheadCostFraction) {
+		log.DedupedWarningf(5, "NodeOverhead: Unexpected NaN found for OverheadCostFraction")
+		n.OverheadCostFraction = 0
+	}
+}
+
 // Node is an Asset representing a single node in a cluster
 type Node struct {
 	Properties   *AssetProperties
@@ -2175,6 +2303,53 @@ func (n *Node) GPUs() float64 {
 	return n.GPUHours * (60.0 / n.Minutes())
 }
 
+func (n *Node) SanitizeNaN() {
+	if math.IsNaN(n.Adjustment) {
+		log.DedupedWarningf(5, "Node: Unexpected NaN found for Adjustment: labels:%v, window:%s, properties:%s", n.Labels, n.Window.String(), n.Properties.String())
+		n.Adjustment = 0
+	}
+	if math.IsNaN(n.CPUCoreHours) {
+		log.DedupedWarningf(5, "Node: Unexpected NaN found for CPUCoreHours: labels:%v, window:%s, properties:%s", n.Labels, n.Window.String(), n.Properties.String())
+		n.CPUCoreHours = 0
+	}
+	if math.IsNaN(n.RAMByteHours) {
+		log.DedupedWarningf(5, "Node: Unexpected NaN found for RAMByteHours: labels:%v, window:%s, properties:%s", n.Labels, n.Window.String(), n.Properties.String())
+		n.RAMByteHours = 0
+	}
+	if math.IsNaN(n.GPUHours) {
+		log.DedupedWarningf(5, "Node: Unexpected NaN found for GPUHours: labels:%v, window:%s, properties:%s", n.Labels, n.Window.String(), n.Properties.String())
+		n.GPUHours = 0
+	}
+	if math.IsNaN(n.CPUCost) {
+		log.DedupedWarningf(5, "Node: Unexpected NaN found for CPUCost: labels:%v, window:%s, properties:%s", n.Labels, n.Window.String(), n.Properties.String())
+		n.CPUCost = 0
+	}
+	if math.IsNaN(n.GPUCost) {
+		log.DedupedWarningf(5, "Node: Unexpected NaN found for GPUCost: labels:%v, window:%s, properties:%s", n.Labels, n.Window.String(), n.Properties.String())
+		n.GPUCost = 0
+	}
+	if math.IsNaN(n.GPUCount) {
+		log.DedupedWarningf(5, "Node: Unexpected NaN found for GPUCount: labels:%v, window:%s, properties:%s", n.Labels, n.Window.String(), n.Properties.String())
+		n.GPUCount = 0
+	}
+	if math.IsNaN(n.RAMCost) {
+		log.DedupedWarningf(5, "Node: Unexpected NaN found for RAMCost: labels:%v, window:%s, properties:%s", n.Labels, n.Window.String(), n.Properties.String())
+		n.RAMCost = 0
+	}
+	if math.IsNaN(n.Discount) {
+		log.DedupedWarningf(5, "Node: Unexpected NaN found for Discount: labels:%v, window:%s, properties:%s", n.Labels, n.Window.String(), n.Properties.String())
+		n.Discount = 0
+	}
+	if math.IsNaN(n.Preemptible) {
+		log.DedupedWarningf(5, "Node: Unexpected NaN found for Preemptible: labels:%v, window:%s, properties:%s", n.Labels, n.Window.String(), n.Properties.String())
+		n.Preemptible = 0
+	}
+
+	n.CPUBreakdown.SanitizeNaN()
+	n.RAMBreakdown.SanitizeNaN()
+	n.Overhead.SanitizeNaN()
+}
+
 // LoadBalancer is an Asset representing a single load balancer in a cluster
 // TODO: add GB of ingress processed, numForwardingRules once we start recording those to prometheus metric
 type LoadBalancer struct {
@@ -2185,10 +2360,12 @@ type LoadBalancer struct {
 	Window     Window
 	Adjustment float64
 	Cost       float64
+	Private    bool   // @bingen:field[version=20]
+	Ip         string // @bingen:field[version=21]
 }
 
 // NewLoadBalancer instantiates and returns a new LoadBalancer
-func NewLoadBalancer(name, cluster, providerID string, start, end time.Time, window Window) *LoadBalancer {
+func NewLoadBalancer(name, cluster, providerID string, start, end time.Time, window Window, private bool, ip string) *LoadBalancer {
 	properties := &AssetProperties{
 		Category:   NetworkCategory,
 		Name:       name,
@@ -2203,6 +2380,8 @@ func NewLoadBalancer(name, cluster, providerID string, start, end time.Time, win
 		Start:      start,
 		End:        end,
 		Window:     window,
+		Private:    private,
+		Ip:         ip,
 	}
 }
 
@@ -2349,6 +2528,11 @@ func (lb *LoadBalancer) add(that *LoadBalancer) {
 
 	lb.Cost += that.Cost
 	lb.Adjustment += that.Adjustment
+
+	if lb.Ip != that.Ip {
+		//TODO: should we add to an array here or just ignore?
+		log.DedupedWarningf(5, "LoadBalancer add: load balancer ip fields (%s and %s) do not match. ignoring...", lb.Ip, that.Ip)
+	}
 }
 
 // Clone returns a cloned instance of the given Asset
@@ -2361,10 +2545,12 @@ func (lb *LoadBalancer) Clone() Asset {
 		Window:     lb.Window.Clone(),
 		Adjustment: lb.Adjustment,
 		Cost:       lb.Cost,
+		Private:    lb.Private,
+		Ip:         lb.Ip,
 	}
 }
 
-// Equal returns true if the tow Assets match precisely
+// Equal returns true if the two Assets match precisely
 func (lb *LoadBalancer) Equal(a Asset) bool {
 	that, ok := a.(*LoadBalancer)
 	if !ok {
@@ -2392,6 +2578,12 @@ func (lb *LoadBalancer) Equal(a Asset) bool {
 	if lb.Cost != that.Cost {
 		return false
 	}
+	if lb.Private != that.Private {
+		return false
+	}
+	if lb.Ip != that.Ip {
+		return false
+	}
 
 	return true
 }
@@ -2401,6 +2593,20 @@ func (lb *LoadBalancer) String() string {
 	return toString(lb)
 }
 
+func (lb *LoadBalancer) SanitizeNaN() {
+	if lb == nil {
+		return
+	}
+	if math.IsNaN(lb.Adjustment) {
+		log.DedupedWarningf(5, "LoadBalancer: Unexpected NaN found for Adjustment: labels:%v, window:%s, properties:%s", lb.Labels, lb.Window.String(), lb.Properties.String())
+		lb.Adjustment = 0
+	}
+	if math.IsNaN(lb.Cost) {
+		log.DedupedWarningf(5, "LoadBalancer: Unexpected NaN found for Cost: labels:%v, window:%s, properties:%s", lb.Labels, lb.Window.String(), lb.Properties.String())
+		lb.Cost = 0
+	}
+}
+
 // SharedAsset is an Asset representing a shared cost
 type SharedAsset struct {
 	Properties *AssetProperties
@@ -2590,6 +2796,16 @@ func (sa *SharedAsset) String() string {
 	return toString(sa)
 }
 
+func (sa *SharedAsset) SanitizeNaN() {
+	if sa == nil {
+		return
+	}
+	if math.IsNaN(sa.Cost) {
+		log.DedupedWarningf(5, "SharedAsset: Unexpected NaN found for Cost: labels:%v, window:%s, properties:%s", sa.Labels, sa.Window.String(), sa.Properties.String())
+		sa.Cost = 0
+	}
+}
+
 // This type exists because only the assets map of AssetSet is marshaled to
 // json, which makes it impossible to recreate an AssetSet struct. Thus,
 // the type when unmarshaling a marshaled AssetSet,is AssetSetResponse
@@ -2743,6 +2959,21 @@ func (as *AssetSet) AggregateBy(aggregateBy []string, opts *AssetAggregationOpti
 		return nil
 	}
 
+	var filter AssetMatcher
+	if opts.Filter == nil {
+		filter = &matcher.AllPass[Asset]{}
+	} else {
+		compiler := NewAssetMatchCompiler()
+		var err error
+		filter, err = compiler.Compile(opts.Filter)
+		if err != nil {
+			return fmt.Errorf("compiling filter '%s': %w", ast.ToPreOrderShortString(opts.Filter), err)
+		}
+	}
+	if filter == nil {
+		return fmt.Errorf("unexpected nil filter")
+	}
+
 	aggSet := NewAssetSet(as.Start(), as.End())
 	aggSet.AggregationKeys = aggregateBy
 
@@ -2759,15 +2990,8 @@ func (as *AssetSet) AggregateBy(aggregateBy []string, opts *AssetAggregationOpti
 		sa := NewSharedAsset(name, as.Window.Clone())
 		sa.Cost = hourlyCost * hours
 
-		// Insert shared asset if it passes all filters
-		insert := true
-		for _, ff := range opts.FilterFuncs {
-			if !ff(sa) {
-				insert = false
-				break
-			}
-		}
-		if insert {
+		// Insert shared asset if it passes filter
+		if filter.Matches(sa) {
 			err := aggSet.Insert(sa, opts.LabelConfig)
 			if err != nil {
 				return err
@@ -2776,11 +3000,9 @@ func (as *AssetSet) AggregateBy(aggregateBy []string, opts *AssetAggregationOpti
 	}
 
 	// Delete the Assets that don't pass each filter
-	for _, ff := range opts.FilterFuncs {
-		for key, asset := range as.Assets {
-			if !ff(asset) {
-				delete(as.Assets, key)
-			}
+	for key, asset := range as.Assets {
+		if !filter.Matches(asset) {
+			delete(as.Assets, key)
 		}
 	}
 
@@ -3181,6 +3403,44 @@ func (as *AssetSet) accumulate(that *AssetSet) (*AssetSet, error) {
 	return acc, nil
 }
 
+func (as *AssetSet) SanitizeNaN() {
+	for _, a := range as.Assets {
+		a.SanitizeNaN()
+	}
+
+	for _, a := range as.Any {
+		a.SanitizeNaN()
+	}
+
+	for _, c := range as.Cloud {
+		c.SanitizeNaN()
+	}
+
+	for _, cm := range as.ClusterManagement {
+		cm.SanitizeNaN()
+	}
+
+	for _, d := range as.Disks {
+		d.SanitizeNaN()
+	}
+
+	for _, n := range as.Network {
+		n.SanitizeNaN()
+	}
+
+	for _, n := range as.Nodes {
+		n.SanitizeNaN()
+	}
+
+	for _, lb := range as.LoadBalancers {
+		lb.SanitizeNaN()
+	}
+
+	for _, sa := range as.SharedAssets {
+		sa.SanitizeNaN()
+	}
+}
+
 type DiffKind string
 
 const (
@@ -3460,7 +3720,7 @@ func (asr *AssetSetRange) newAccumulation() (*AssetSet, error) {
 
 type AssetAggregationOptions struct {
 	SharedHourlyCosts map[string]float64
-	FilterFuncs       []AssetMatchFunc
+	Filter            filter21.Filter
 	LabelConfig       *LabelConfig
 }
 

pkg/kubecost/asset_json.go

@@ -613,7 +613,9 @@ func (lb *LoadBalancer) MarshalJSON() ([]byte, error) {
 	jsonEncodeString(buffer, "end", lb.End.Format(time.RFC3339), ",")
 	jsonEncodeFloat64(buffer, "minutes", lb.Minutes(), ",")
 	jsonEncodeFloat64(buffer, "adjustment", lb.Adjustment, ",")
-	jsonEncodeFloat64(buffer, "totalCost", lb.TotalCost(), "")
+	jsonEncodeFloat64(buffer, "totalCost", lb.TotalCost(), ",")
+	jsonEncode(buffer, "private", lb.Private, ",")
+	jsonEncodeString(buffer, "ip", lb.Ip, "")
 	buffer.WriteString("}")
 	return buffer.Bytes(), nil
 }
@@ -675,6 +677,12 @@ func (lb *LoadBalancer) InterfaceToLoadBalancer(itf interface{}) error {
 	if Cost, err := getTypedVal(fmap["totalCost"]); err == nil {
 		lb.Cost = Cost.(float64) - lb.Adjustment
 	}
+	if private, err := getTypedVal(fmap["private"]); err == nil {
+		lb.Private = private.(bool)
+	}
+	if ip, err := getTypedVal(fmap["ip"]); err == nil {
+		lb.Ip = ip.(string)
+	}
 
 	return nil
 

pkg/kubecost/asset_json_test.go

@@ -419,7 +419,7 @@ func TestNode_Unmarshal(t *testing.T) {
 
 func TestLoadBalancer_Unmarshal(t *testing.T) {
 
-	lb1 := NewLoadBalancer("loadbalancer1", "cluster1", "provider1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
+	lb1 := NewLoadBalancer("loadbalancer1", "cluster1", "provider1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow, false, "127.0.0.1")
 	lb1.Cost = 12.0
 	lb1.SetAdjustment(4.0)
 
@@ -457,6 +457,12 @@ func TestLoadBalancer_Unmarshal(t *testing.T) {
 	if lb1.Cost != lb2.Cost {
 		t.Fatalf("LoadBalancer Unmarshal: cost mutated in unmarshal")
 	}
+	if lb1.Private != lb2.Private {
+		t.Fatalf("LoadBalancer Unmarshal: private mutated in unmarshal")
+	}
+	if lb1.Ip != lb2.Ip {
+		t.Fatalf("LoadBalancer Unmarshal: ip mutated in unmarshal")
+	}
 
 	// As a final check, make sure the above checks out
 	if !lb1.Equal(lb2) {
@@ -515,7 +521,7 @@ func TestAssetset_Unmarshal(t *testing.T) {
 	disk := NewDisk("disk1", "cluster1", "disk1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
 	network := NewNetwork("network1", "cluster1", "provider1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
 	node := NewNode("node1", "cluster1", "provider1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
-	lb := NewLoadBalancer("loadbalancer1", "cluster1", "provider1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow)
+	lb := NewLoadBalancer("loadbalancer1", "cluster1", "provider1", *unmarshalWindow.start, *unmarshalWindow.end, unmarshalWindow, false, "127.0.0.1")
 	sa := NewSharedAsset("sharedasset1", unmarshalWindow)
 
 	assetList := []Asset{any, cloud, cm, disk, network, node, lb, sa}

pkg/kubecost/asset_test.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"math"
+	"reflect"
 	"testing"
 	"time"
 
@@ -1685,3 +1686,260 @@ func TestAssetSetRange_AccumulateBy_Month(t *testing.T) {
 		}
 	}
 }
+
+func TestAny_SanitizeNaN(t *testing.T) {
+	any := getMockAny(math.NaN())
+	any.SanitizeNaN()
+	v := reflect.ValueOf(any)
+	checkAllFloat64sForNaN(t, v, "TestAny_SanitizeNaN")
+}
+
+func getMockAny(f float64) Any {
+	return Any{
+		Adjustment: f,
+		Cost:       f,
+	}
+}
+
+func TestCloud_SanitizeNaN(t *testing.T) {
+	cloud := getMockCloud(math.NaN())
+	cloud.SanitizeNaN()
+	v := reflect.ValueOf(cloud)
+	checkAllFloat64sForNaN(t, v, "TestCloud_SanitizeNaN")
+}
+
+func getMockCloud(f float64) Cloud {
+	return Cloud{
+		Adjustment: f,
+		Cost:       f,
+		Credit:     f,
+	}
+}
+
+func TestClusterManagement_SanitizeNaN(t *testing.T) {
+	cm := getMockClusterManagement(math.NaN())
+	cm.SanitizeNaN()
+	v := reflect.ValueOf(cm)
+	checkAllFloat64sForNaN(t, v, "TestClusterManagement_SanitizeNaN")
+}
+
+func getMockClusterManagement(f float64) ClusterManagement {
+	return ClusterManagement{
+		Cost:       f,
+		Adjustment: f,
+	}
+}
+
+func TestDisk_SanitizeNaN(t *testing.T) {
+	disk := getMockDisk(math.NaN())
+	disk.SanitizeNaN()
+	v := reflect.ValueOf(disk)
+	checkAllFloat64sForNaN(t, v, "TestDisk_SanitizeNaN")
+
+	vBreakdown := reflect.ValueOf(*disk.Breakdown)
+	checkAllFloat64sForNaN(t, vBreakdown, "TestDisk_SanitizeNaN")
+}
+
+func getMockDisk(f float64) Disk {
+	bhu := f
+	bum := f
+	breakdown := getMockBreakdown(f)
+	return Disk{
+		Adjustment:    f,
+		Cost:          f,
+		ByteHours:     f,
+		Local:         f,
+		Breakdown:     &breakdown,
+		ByteHoursUsed: &bhu,
+		ByteUsageMax:  &bum,
+	}
+}
+
+func TestBreakdown_SanitizeNaN(t *testing.T) {
+	b := getMockBreakdown(math.NaN())
+	b.SanitizeNaN()
+	v := reflect.ValueOf(b)
+	checkAllFloat64sForNaN(t, v, "TestBreakdown_SanitizeNaN")
+}
+
+func getMockBreakdown(f float64) Breakdown {
+	return Breakdown{
+		Idle:   f,
+		Other:  f,
+		System: f,
+		User:   f,
+	}
+}
+
+func TestNetwork_SanitizeNaN(t *testing.T) {
+	n := getMockNetwork(math.NaN())
+	n.SanitizeNaN()
+	v := reflect.ValueOf(n)
+	checkAllFloat64sForNaN(t, v, "TestNetwork_SanitizeNaN")
+}
+
+func getMockNetwork(f float64) Network {
+	return Network{
+		Adjustment: f,
+		Cost:       f,
+	}
+}
+
+func TestNodeOverhead_SanitizeNaN(t *testing.T) {
+	n := getMockNodeOverhead(math.NaN())
+	n.SanitizeNaN()
+	v := reflect.ValueOf(n)
+	checkAllFloat64sForNaN(t, v, "TestNodeOverhead_SanitizeNaN")
+}
+
+func getMockNodeOverhead(f float64) NodeOverhead {
+	return NodeOverhead{
+		CpuOverheadFraction:  f,
+		RamOverheadFraction:  f,
+		OverheadCostFraction: f,
+	}
+}
+
+func TestNode_SanitizeNaN(t *testing.T) {
+	n := getMockNode(math.NaN())
+	n.SanitizeNaN()
+	v := reflect.ValueOf(n)
+	checkAllFloat64sForNaN(t, v, "TestNode_SanitizeNaN")
+
+	vCpu := reflect.ValueOf(*n.CPUBreakdown)
+	checkAllFloat64sForNaN(t, vCpu, "TestNode_SanitizeNaN")
+
+	vRam := reflect.ValueOf(*n.RAMBreakdown)
+	checkAllFloat64sForNaN(t, vRam, "TestNode_SanitizeNaN")
+
+	vOverhead := reflect.ValueOf(*n.Overhead)
+	checkAllFloat64sForNaN(t, vOverhead, "TestNode_SanitizeNaN")
+}
+
+func getMockNode(f float64) Node {
+	cpuBreakdown := getMockBreakdown(f)
+	ramBreakdown := getMockBreakdown(f)
+	overhead := getMockNodeOverhead(f)
+	return Node{
+		Adjustment:   f,
+		CPUCoreHours: f,
+		RAMByteHours: f,
+		GPUHours:     f,
+		CPUBreakdown: &cpuBreakdown,
+		RAMBreakdown: &ramBreakdown,
+		CPUCost:      f,
+		GPUCost:      f,
+		GPUCount:     f,
+		RAMCost:      f,
+		Discount:     f,
+		Preemptible:  f,
+		Overhead:     &overhead,
+	}
+}
+
+func TestLoadBalancer_SanitizeNaN(t *testing.T) {
+	lb := getMockLoadBalancer(math.NaN())
+	lb.SanitizeNaN()
+	v := reflect.ValueOf(lb)
+	checkAllFloat64sForNaN(t, v, "TestLoadBalancer_SanitizeNaN")
+}
+
+func getMockLoadBalancer(f float64) LoadBalancer {
+	return LoadBalancer{
+		Adjustment: f,
+		Cost:       f,
+	}
+}
+
+func TestSharedAsset_SanitizeNaN(t *testing.T) {
+	sa := getMockSharedAsset(math.NaN())
+	sa.SanitizeNaN()
+	v := reflect.ValueOf(sa)
+	checkAllFloat64sForNaN(t, v, "TestSharedAsset_SanitizeNaN")
+}
+
+func getMockSharedAsset(f float64) SharedAsset {
+	return SharedAsset{
+		Cost: f,
+	}
+}
+
+func TestAssetSet_SanitizeNaN(t *testing.T) {
+	testCaseName := "TestAssetSet_SanitizeNaN"
+	as := getMockAssetSet(math.NaN())
+	as.SanitizeNaN()
+	v := reflect.ValueOf(as)
+	checkAllFloat64sForNaN(t, v, testCaseName)
+
+	for _, a := range as.Assets {
+		if math.IsNaN(a.TotalCost()) {
+			t.Fatalf("TestAssetSet_SanitizeNaN: Asset: expected not NaN for TotalCost(): expected NaN, got:%f", a.TotalCost())
+
+		}
+		if math.IsNaN(a.GetAdjustment()) {
+			t.Fatalf("TestAssetSet_SanitizeNaN: Asset: expected not NaN for GetAdjustment(): expected NaN, got:%f", a.GetAdjustment())
+		}
+	}
+
+	for _, any := range as.Any {
+		vAny := reflect.ValueOf(*any)
+		checkAllFloat64sForNaN(t, vAny, testCaseName)
+	}
+
+	for _, cloud := range as.Cloud {
+		vCloud := reflect.ValueOf(*cloud)
+		checkAllFloat64sForNaN(t, vCloud, testCaseName)
+	}
+
+	for _, cm := range as.ClusterManagement {
+		vCM := reflect.ValueOf(*cm)
+		checkAllFloat64sForNaN(t, vCM, testCaseName)
+	}
+
+	for _, disk := range as.Disks {
+		vDisk := reflect.ValueOf(*disk)
+		checkAllFloat64sForNaN(t, vDisk, testCaseName)
+	}
+
+	for _, network := range as.Network {
+		vNetwork := reflect.ValueOf(*network)
+		checkAllFloat64sForNaN(t, vNetwork, testCaseName)
+	}
+
+	for _, node := range as.Nodes {
+		vNode := reflect.ValueOf(*node)
+		checkAllFloat64sForNaN(t, vNode, testCaseName)
+	}
+
+	for _, sa := range as.SharedAssets {
+		vSA := reflect.ValueOf(*sa)
+		checkAllFloat64sForNaN(t, vSA, testCaseName)
+	}
+
+}
+
+func getMockAssetSet(f float64) AssetSet {
+	any := getMockAny(f)
+	cloud := getMockCloud(f)
+	cm := getMockClusterManagement(f)
+	disk := getMockDisk(f)
+	network := getMockNetwork(f)
+	node := getMockNode(f)
+	lb := getMockLoadBalancer(f)
+	sa := getMockSharedAsset(f)
+
+	assets := map[string]Asset{"any": &any, "cloud": &cloud}
+	as := AssetSet{
+		Assets:            assets,
+		Any:               map[string]*Any{"NaN": &any},
+		Cloud:             map[string]*Cloud{"NaN": &cloud},
+		ClusterManagement: map[string]*ClusterManagement{"NaN": &cm},
+		Disks:             map[string]*Disk{"NaN": &disk},
+		Network:           map[string]*Network{"NaN": &network},
+		Nodes:             map[string]*Node{"NaN": &node},
+		LoadBalancers:     map[string]*LoadBalancer{"NaN": &lb},
+		SharedAssets:      map[string]*SharedAsset{"NaN": &sa},
+	}
+
+	return as
+}

pkg/kubecost/assetmatcher.go

@@ -0,0 +1,105 @@
+package kubecost
+
+import (
+	"fmt"
+	"strings"
+
+	afilter "github.com/opencost/opencost/pkg/filter21/asset"
+	"github.com/opencost/opencost/pkg/filter21/ast"
+	"github.com/opencost/opencost/pkg/filter21/matcher"
+	"github.com/opencost/opencost/pkg/filter21/transform"
+)
+
+// AssetMatcher is a matcher implementation for Asset instances,
+// compiled using the matcher.MatchCompiler.
+type AssetMatcher matcher.Matcher[Asset]
+
+// NewAssetMatchCompiler creates a new instance of a
+// matcher.MatchCompiler[Asset] which can be used to compile filter.Filter
+// ASTs into matcher.Matcher[Asset] implementations.
+//
+// If the label config is nil, the compiler will fail to compile alias filters
+// if any are present in the AST.
+//
+// If storage interfaces every support querying natively by alias (e.g. if a
+// data store contained a "product" attribute on an Asset row), that should
+// be handled by a purpose-built AST compiler.
+func NewAssetMatchCompiler() *matcher.MatchCompiler[Asset] {
+	passes := []transform.CompilerPass{}
+
+	passes = append(passes,
+		transform.PrometheusKeySanitizePass(),
+		transform.UnallocatedReplacementPass(),
+	)
+	return matcher.NewMatchCompiler(
+		assetFieldMap,
+		assetSliceFieldMap,
+		assetMapFieldMap,
+		passes...,
+	)
+}
+
+// Maps fields from an asset to a string value based on an identifier
+func assetFieldMap(a Asset, identifier ast.Identifier) (string, error) {
+	if identifier.Field == nil {
+		return "", fmt.Errorf("cannot map field from identifier with nil field")
+	}
+	if a == nil {
+		return "", fmt.Errorf("cannot map field for nil Asset")
+	}
+
+	// Check special fields before defaulting to properties-based fields
+	switch afilter.AssetField(identifier.Field.Name) {
+	case afilter.FieldType:
+		return strings.ToLower(a.Type().String()), nil
+	case afilter.FieldLabel:
+		labels := a.GetLabels()
+		if labels == nil {
+			return "", nil
+		}
+		return labels[identifier.Key], nil
+	}
+
+	props := a.GetProperties()
+	if props == nil {
+		return "", fmt.Errorf("cannot map field for Asset with nil props")
+	}
+
+	switch afilter.AssetField(identifier.Field.Name) {
+	case afilter.FieldName:
+		return props.Name, nil
+	case afilter.FieldCategory:
+		return props.Category, nil
+	case afilter.FieldClusterID:
+		return props.Cluster, nil
+	case afilter.FieldProject:
+		return props.Project, nil
+	case afilter.FieldProvider:
+		return props.Provider, nil
+	case afilter.FieldProviderID:
+		return props.ProviderID, nil
+	case afilter.FieldAccount:
+		return props.Account, nil
+	case afilter.FieldService:
+		return props.Service, nil
+	}
+
+	return "", fmt.Errorf("Failed to find string identifier on Asset: %s", identifier.Field.Name)
+}
+
+// Maps slice fields from an asset to a []string value based on an identifier
+func assetSliceFieldMap(a Asset, identifier ast.Identifier) ([]string, error) {
+	return nil, fmt.Errorf("Assets have no slice fields")
+}
+
+// Maps map fields from an Asset to a map[string]string value based on an identifier
+func assetMapFieldMap(a Asset, identifier ast.Identifier) (map[string]string, error) {
+	if a == nil {
+		return nil, fmt.Errorf("cannot get map field for nil Asset")
+	}
+	switch afilter.AssetField(identifier.Field.Name) {
+	case afilter.FieldLabel:
+		return a.GetLabels(), nil
+	}
+	return nil, fmt.Errorf("Failed to find map[string]string identifier on Asset: %s", identifier.Field.Name)
+}

pkg/kubecost/audit.go

@@ -1,363 +0,0 @@
-package kubecost
-
-import (
-	"sync"
-	"time"
-
-	"golang.org/x/exp/slices"
-)
-
-// AuditType the types of Audits, each of which should be contained in an AuditSet
-type AuditType string
-
-const (
-	AuditAllocationReconciliation AuditType = "AuditAllocationReconciliation"
-	AuditAllocationTotalStore     AuditType = "AuditAllocationTotalStore"
-	AuditAllocationAggStore       AuditType = "AuditAllocationAggStore"
-	AuditAssetReconciliation      AuditType = "AuditAssetReconciliation"
-	AuditAssetTotalStore          AuditType = "AuditAssetTotalStore"
-	AuditAssetAggStore            AuditType = "AuditAssetAggStore"
-	AuditClusterEquality          AuditType = "AuditClusterEquality"
-
-	AuditAll         AuditType = ""
-	AuditInvalidType AuditType = "InvalidType"
-)
-
-// ToAuditType converts a string to an Audit type
-func ToAuditType(check string) AuditType {
-	switch check {
-	case string(AuditAllocationReconciliation):
-		return AuditAllocationReconciliation
-	case string(AuditAllocationTotalStore):
-		return AuditAllocationTotalStore
-	case string(AuditAllocationAggStore):
-		return AuditAllocationAggStore
-	case string(AuditAssetReconciliation):
-		return AuditAssetReconciliation
-	case string(AuditAssetTotalStore):
-		return AuditAssetTotalStore
-	case string(AuditAssetAggStore):
-		return AuditAssetAggStore
-	case string(AuditClusterEquality):
-		return AuditClusterEquality
-	case string(AuditAll):
-		return AuditAll
-	default:
-		return AuditInvalidType
-	}
-}
-
-// AuditStatus are possible outcomes of an audit
-type AuditStatus string
-
-const (
-	FailedStatus  AuditStatus = "Failed"
-	WarningStatus             = "Warning"
-	PassedStatus              = "Passed"
-)
-
-// AuditMissingValue records when a value that should be present in a store or in the audit generated results are missing
-type AuditMissingValue struct {
-	Description string
-	Key         string
-}
-
-// AuditFloatResult structure for holding the results of a failed audit on a float value, Expected should be the value
-// calculated by the Audit func while Actual is what is contained in the relevant store.
-type AuditFloatResult struct {
-	Expected float64
-	Actual   float64
-}
-
-// Clone returns a deep copy of the caller
-func (afr *AuditFloatResult) Clone() *AuditFloatResult {
-	return &AuditFloatResult{
-		Expected: afr.Expected,
-		Actual:   afr.Actual,
-	}
-}
-
-// AllocationReconciliationAudit records the differences of between compute resources (cpu, ram, gpu) costs between
-// allocations by nodes and node assets keyed on node name and compute resource
-type AllocationReconciliationAudit struct {
-	Status        AuditStatus
-	Description   string
-	LastRun       time.Time
-	Resources     map[string]map[string]*AuditFloatResult
-	MissingValues []*AuditMissingValue
-}
-
-// Clone returns a deep copy of the caller
-func (ara *AllocationReconciliationAudit) Clone() *AllocationReconciliationAudit {
-	if ara == nil {
-		return nil
-	}
-
-	resources := make(map[string]map[string]*AuditFloatResult, len(ara.Resources))
-	for node, resourceMap := range ara.Resources {
-		copyResourceMap := make(map[string]*AuditFloatResult, len(resourceMap))
-		for resourceName, val := range resourceMap {
-			copyResourceMap[resourceName] = val.Clone()
-		}
-		resources[node] = copyResourceMap
-	}
-	return &AllocationReconciliationAudit{
-		Status:        ara.Status,
-		Description:   ara.Description,
-		LastRun:       ara.LastRun,
-		Resources:     resources,
-		MissingValues: slices.Clone(ara.MissingValues),
-	}
-}
-
-// TotalAudit records the differences between a total store and the totaled results of the store that it is based on
-// keyed by cluster and node names
-type TotalAudit struct {
-	Status         AuditStatus
-	Description    string
-	LastRun        time.Time
-	TotalByNode    map[string]*AuditFloatResult
-	TotalByCluster map[string]*AuditFloatResult
-	MissingValues  []*AuditMissingValue
-}
-
-// Clone returns a deep copy of the caller
-func (ta *TotalAudit) Clone() *TotalAudit {
-	if ta == nil {
-		return nil
-	}
-
-	tbn := make(map[string]*AuditFloatResult, len(ta.TotalByNode))
-	for k, v := range ta.TotalByNode {
-		tbn[k] = v
-	}
-	tbc := make(map[string]*AuditFloatResult, len(ta.TotalByNode))
-	for k, v := range ta.TotalByCluster {
-		tbc[k] = v
-	}
-
-	return &TotalAudit{
-		Status:         ta.Status,
-		Description:    ta.Description,
-		LastRun:        ta.LastRun,
-		TotalByNode:    tbn,
-		TotalByCluster: tbc,
-		MissingValues:  slices.Clone(ta.MissingValues),
-	}
-}
-
-// AggAudit contains the results of an Audit on an AggStore keyed on aggregation prop and Allocation key
-type AggAudit struct {
-	Status        AuditStatus
-	Description   string
-	LastRun       time.Time
-	Results       map[string]map[string]*AuditFloatResult
-	MissingValues []*AuditMissingValue
-}
-
-// Clone returns a deep copy of the caller
-func (aa *AggAudit) Clone() *AggAudit {
-	if aa == nil {
-		return nil
-	}
-	res := make(map[string]map[string]*AuditFloatResult, len(aa.Results))
-	for aggType, aggResults := range aa.Results {
-		copyAggResult := make(map[string]*AuditFloatResult, len(aggResults))
-		for aggName, auditFloatResult := range aggResults {
-			copyAggResult[aggName] = auditFloatResult
-		}
-		res[aggType] = copyAggResult
-	}
-
-	return &AggAudit{
-		Status:        aa.Status,
-		Description:   aa.Description,
-		LastRun:       aa.LastRun,
-		Results:       res,
-		MissingValues: slices.Clone(aa.MissingValues),
-	}
-}
-
-// AssetReconciliationAudit records differences in assets and the Cloud
-type AssetReconciliationAudit struct {
-	Status        AuditStatus
-	Description   string
-	LastRun       time.Time
-	Results       map[string]map[string]*AuditFloatResult
-	MissingValues []*AuditMissingValue
-}
-
-// Clone returns a deep copy of the caller
-func (ara *AssetReconciliationAudit) Clone() *AssetReconciliationAudit {
-	res := make(map[string]map[string]*AuditFloatResult, len(ara.Results))
-	for aggType, aggResults := range ara.Results {
-		copyAggResult := make(map[string]*AuditFloatResult, len(aggResults))
-		for aggName, auditFloatResult := range aggResults {
-			copyAggResult[aggName] = auditFloatResult
-		}
-		res[aggType] = copyAggResult
-	}
-
-	return &AssetReconciliationAudit{
-		Status:        ara.Status,
-		Description:   ara.Description,
-		LastRun:       ara.LastRun,
-		Results:       res,
-		MissingValues: slices.Clone(ara.MissingValues),
-	}
-}
-
-// EqualityAudit records the difference in cost between Allocations and Assets aggregated by cluster and keyed on cluster
-type EqualityAudit struct {
-	Status        AuditStatus
-	Description   string
-	LastRun       time.Time
-	Clusters      map[string]*AuditFloatResult
-	MissingValues []*AuditMissingValue
-}
-
-// Clone returns a deep copy of the caller
-func (ea *EqualityAudit) Clone() *EqualityAudit {
-	if ea == nil {
-		return nil
-	}
-	clusters := make(map[string]*AuditFloatResult, len(ea.Clusters))
-	for k, v := range ea.Clusters {
-		clusters[k] = v
-	}
-	return &EqualityAudit{
-		Status:        ea.Status,
-		Description:   ea.Description,
-		LastRun:       ea.LastRun,
-		Clusters:      clusters,
-		MissingValues: slices.Clone(ea.MissingValues),
-	}
-}
-
-// AuditCoverage tracks coverage of each audit type
-type AuditCoverage struct {
-	AllocationReconciliation Window `json:"allocationReconciliation"`
-	AllocationAgg            Window `json:"allocationAgg"`
-	AllocationTotal          Window `json:"allocationTotal"`
-	AssetTotal               Window `json:"assetTotal"`
-	AssetReconciliation      Window `json:"assetReconciliation"`
-	ClusterEquality          Window `json:"clusterEquality"`
-}
-
-// NewAuditCoverage create default AuditCoverage
-func NewAuditCoverage() *AuditCoverage {
-	return &AuditCoverage{}
-}
-
-// Update expands the coverage of each Window in the coverage that the given AuditSet's Window if the corresponding Audit is not nil
-// Note: This means of determining coverage can lead to holes in the given window
-func (ac *AuditCoverage) Update(as *AuditSet) {
-	if as != nil && as.AllocationReconciliation != nil {
-		ac.AllocationReconciliation.Expand(as.Window)
-		ac.AllocationAgg.Expand(as.Window)
-		ac.AllocationTotal.Expand(as.Window)
-		ac.AssetTotal.Expand(as.Window)
-		ac.AssetReconciliation.Expand(as.Window)
-		ac.ClusterEquality.Expand(as.Window)
-	}
-
-}
-
-// AuditSet is a ETLSet which contains all kind of Audits for a given Window
-type AuditSet struct {
-	sync.RWMutex
-	AllocationReconciliation *AllocationReconciliationAudit `json:"allocationReconciliation"`
-	AllocationAgg            *AggAudit                      `json:"allocationAgg"`
-	AllocationTotal          *TotalAudit                    `json:"allocationTotal"`
-	AssetTotal               *TotalAudit                    `json:"assetTotal"`
-	AssetReconciliation      *AssetReconciliationAudit      `json:"assetReconciliation"`
-	ClusterEquality          *EqualityAudit                 `json:"clusterEquality"`
-	Window                   Window                         `json:"window"`
-}
-
-// NewAuditSet creates an empty AuditSet with the given window
-func NewAuditSet(start, end time.Time) *AuditSet {
-	return &AuditSet{
-		Window: NewWindow(&start, &end),
-	}
-}
-
-// UpdateAuditSet overwrites any audit fields in the caller with those in the given AuditSet which are not nil
-func (as *AuditSet) UpdateAuditSet(that *AuditSet) *AuditSet {
-	if as == nil {
-		return that
-	}
-
-	if that.AllocationReconciliation != nil {
-		as.AllocationReconciliation = that.AllocationReconciliation
-	}
-	if that.AllocationAgg != nil {
-		as.AllocationAgg = that.AllocationAgg
-	}
-	if that.AllocationTotal != nil {
-		as.AllocationTotal = that.AllocationTotal
-	}
-	if that.AssetTotal != nil {
-		as.AssetTotal = that.AssetTotal
-	}
-	if that.AssetReconciliation != nil {
-		as.AssetReconciliation = that.AssetReconciliation
-	}
-
-	if that.ClusterEquality != nil {
-		as.ClusterEquality = that.ClusterEquality
-	}
-
-	return as
-}
-
-// ConstructSet fulfills the ETLSet interface to provide an empty version of itself so that it can be initialized in its
-// generic form.
-func (as *AuditSet) ConstructSet() ETLSet {
-	return &AuditSet{}
-}
-
-// IsEmpty returns true if any of the audits are non-nil
-func (as *AuditSet) IsEmpty() bool {
-	return as == nil || (as.AllocationReconciliation == nil &&
-		as.AllocationAgg == nil &&
-		as.AllocationTotal == nil &&
-		as.AssetTotal == nil &&
-		as.AssetReconciliation == nil &&
-		as.ClusterEquality == nil)
-}
-
-// GetWindow returns AuditSet Window
-func (as *AuditSet) GetWindow() Window {
-	return as.Window
-}
-
-// Clone returns a deep copy of the caller
-func (as *AuditSet) Clone() *AuditSet {
-	if as == nil {
-		return nil
-	}
-
-	as.RLock()
-	defer as.RUnlock()
-
-	return &AuditSet{
-		AllocationReconciliation: as.AllocationReconciliation.Clone(),
-		AllocationAgg:            as.AllocationAgg.Clone(),
-		AllocationTotal:          as.AllocationTotal.Clone(),
-		AssetTotal:               as.AssetTotal.Clone(),
-		AssetReconciliation:      as.AssetReconciliation.Clone(),
-		ClusterEquality:          as.ClusterEquality.Clone(),
-		Window:                   as.Window.Clone(),
-	}
-}
-
-// CloneSet returns a deep copy of the caller and returns set
-func (as *AuditSet) CloneSet() ETLSet {
-	return as.Clone()
-}
-
-// AuditSetRange SetRange of AuditSets
-type AuditSetRange struct {
-	SetRange[*AuditSet]
-}

pkg/kubecost/bingen.go

@@ -26,7 +26,7 @@ package kubecost
 // @bingen:generate:CoverageSet
 
 // Asset Version Set: Includes Asset pipeline specific resources
-// @bingen:set[name=Assets,version=19]
+// @bingen:set[name=Assets,version=21]
 // @bingen:generate:Any
 // @bingen:generate:Asset
 // @bingen:generate:AssetLabels
@@ -46,7 +46,7 @@ package kubecost
 // @bingen:end
 
 // Allocation Version Set: Includes Allocation pipeline specific resources
-// @bingen:set[name=Allocation,version=17]
+// @bingen:set[name=Allocation,version=19]
 // @bingen:generate:Allocation
 // @bingen:generate[stringtable]:AllocationSet
 // @bingen:generate:AllocationSetRange
@@ -58,20 +58,8 @@ package kubecost
 // @bingen:generate:PVAllocations
 // @bingen:generate:PVKey
 // @bingen:generate:PVAllocation
-// @bingen:end
-
-// @bingen:set[name=Audit,version=1]
-// @bingen:generate:AllocationReconciliationAudit
-// @bingen:generate:TotalAudit
-// @bingen:generate:AggAudit
-// @bingen:generate:AuditFloatResult
-// @bingen:generate:AuditMissingValue
-// @bingen:generate:AssetReconciliationAudit
-// @bingen:generate:EqualityAudit
-// @bingen:generate:AuditType
-// @bingen:generate:AuditStatus
-// @bingen:generate[stringtable]:AuditSet
-// @bingen:generate:AuditSetRange
+// @bingen:generate:LbAllocations
+// @bingen:generate:LbAllocation
 // @bingen:end
 
 // @bingen:set[name=CloudCost,version=2]

pkg/kubecost/cloudusage.go

@@ -11,6 +11,3 @@ type CloudUsageSetRange = AssetSetRange
 
 // CloudUsageAggregationOptions is temporarily aliased as the AssetAggregationOptions until further infrastructure and pages can be built to support its usage
 type CloudUsageAggregationOptions = AssetAggregationOptions
-
-// CloudUsageMatchFunc is temporarily aliased as the AssetMatchFunc until further infrastructure and pages can be built to support its usage
-type CloudUsageMatchFunc = AssetMatchFunc

pkg/kubecost/coverage.go

@@ -26,10 +26,18 @@ func (c *Coverage) Key() string {
 }
 
 func (c *Coverage) IsEmpty() bool {
+	if c == nil {
+		log.Warnf("calling IsEmpty() on a nil Coverage")
+		return true
+	}
 	return c.Type == "" && c.Count == 0 && len(c.Errors) == 0 && len(c.Warnings) == 0 && c.Updated == time.Time{}
 }
 
 func (c *Coverage) Clone() *Coverage {
+	if c == nil {
+		log.Warnf("calling Clone() on a nil Coverage")
+		return nil
+	}
 	var errors []string
 	if len(c.Errors) > 0 {
 		errors = make([]string, len(c.Errors))

pkg/kubecost/etlrange.go

@@ -1,90 +0,0 @@
-package kubecost
-
-import (
-	"fmt"
-	"sync"
-
-	"github.com/opencost/opencost/pkg/util/json"
-)
-
-// SetRange is a generic implementation of the SetRanges that act as containers. It covers the basic functionality that
-// is shared by the basic types but is meant to be extended by each implementation.
-type SetRange[T ETLSet] struct {
-	lock sync.RWMutex
-	sets []T
-}
-
-// Append attaches the given ETLSet to the end of the sets slice.
-// currently does not check that the window is correct.
-func (r *SetRange[T]) Append(that T) {
-	if r == nil {
-		return
-	}
-	r.lock.Lock()
-	defer r.lock.Unlock()
-	r.sets = append(r.sets, that)
-}
-
-// Each invokes the given function for each ETLSet in the SetRange
-func (r *SetRange[T]) Each(f func(int, T)) {
-	if r == nil {
-		return
-	}
-
-	for i, set := range r.sets {
-		f(i, set)
-	}
-}
-
-// Get retrieves the given index from the sets slice
-func (r *SetRange[T]) Get(i int) (T, error) {
-	var set T
-	if r == nil {
-		return set, fmt.Errorf("SetRange: Get: is nil")
-	}
-	if i < 0 || i >= len(r.sets) {
-
-		return set, fmt.Errorf("SetRange: Get: index out of range: %d", i)
-	}
-
-	r.lock.RLock()
-	defer r.lock.RUnlock()
-	return r.sets[i], nil
-}
-
-// Length returns the length of the sets slice
-func (r *SetRange[T]) Length() int {
-	if r == nil || r.sets == nil {
-		return 0
-	}
-
-	r.lock.RLock()
-	defer r.lock.RUnlock()
-	return len(r.sets)
-}
-
-// IsEmpty returns false if SetRange contains a single ETLSet that is not empty
-func (r *SetRange[T]) IsEmpty() bool {
-	if r == nil || r.Length() == 0 {
-		return true
-	}
-	r.lock.RLock()
-	defer r.lock.RUnlock()
-
-	for _, set := range r.sets {
-		if !set.IsEmpty() {
-			return false
-		}
-	}
-	return true
-}
-
-// MarshalJSON converts SetRange to JSON
-func (r *SetRange[T]) MarshalJSON() ([]byte, error) {
-	if r == nil {
-		return json.Marshal([]T{})
-	}
-	r.lock.RLock()
-	defer r.lock.RUnlock()
-	return json.Marshal(r.sets)
-}

pkg/kubecost/etlset.go

@@ -1,15 +0,0 @@
-package kubecost
-
-import "encoding"
-
-// ETLSet is an interface which represents the basic data block of an ETL. It is keyed by its Window
-type ETLSet interface {
-	ConstructSet() ETLSet
-	CloneSet() ETLSet
-	IsEmpty() bool
-	GetWindow() Window
-
-	// Representations
-	encoding.BinaryMarshaler
-	encoding.BinaryUnmarshaler
-}

pkg/kubecost/mock.go

@@ -163,6 +163,12 @@ func GenerateMockAllocationSet(start time.Time) *AllocationSet {
 		Node:       "c1nodes",
 	})
 	a1111.RAMCost = 11.00
+	a1111.PVs = PVAllocations{
+		PVKey{Cluster: "cluster1", Name: "pv-a1111"}: {
+			ByteHours: 1,
+			Cost:      1,
+		},
+	}
 
 	a11abc2 := NewMockUnitAllocation("cluster1/namespace1/pod-abc/container2", start, day, &AllocationProperties{
 		Cluster:    "cluster1",
@@ -172,6 +178,12 @@ func GenerateMockAllocationSet(start time.Time) *AllocationSet {
 		ProviderID: "c1nodes",
 		Node:       "c1nodes",
 	})
+	a11abc2.PVs = PVAllocations{
+		PVKey{Cluster: "cluster1", Name: "pv-a11abc2"}: {
+			ByteHours: 1,
+			Cost:      1,
+		},
+	}
 
 	a11def3 := NewMockUnitAllocation("cluster1/namespace1/pod-def/container3", start, day, &AllocationProperties{
 		Cluster:    "cluster1",
@@ -181,6 +193,12 @@ func GenerateMockAllocationSet(start time.Time) *AllocationSet {
 		ProviderID: "c1nodes",
 		Node:       "c1nodes",
 	})
+	a11def3.PVs = PVAllocations{
+		PVKey{Cluster: "cluster1", Name: "pv-a11def3"}: {
+			ByteHours: 1,
+			Cost:      1,
+		},
+	}
 
 	a12ghi4 := NewMockUnitAllocation("cluster1/namespace2/pod-ghi/container4", start, day, &AllocationProperties{
 		Cluster:    "cluster1",
@@ -190,6 +208,12 @@ func GenerateMockAllocationSet(start time.Time) *AllocationSet {
 		ProviderID: "c1nodes",
 		Node:       "c1nodes",
 	})
+	a12ghi4.PVs = PVAllocations{
+		PVKey{Cluster: "cluster1", Name: "pv-a12ghi4"}: {
+			ByteHours: 1,
+			Cost:      1,
+		},
+	}
 
 	a12ghi5 := NewMockUnitAllocation("cluster1/namespace2/pod-ghi/container5", start, day, &AllocationProperties{
 		Cluster:    "cluster1",
@@ -199,6 +223,12 @@ func GenerateMockAllocationSet(start time.Time) *AllocationSet {
 		ProviderID: "c1nodes",
 		Node:       "c1nodes",
 	})
+	a12ghi5.PVs = PVAllocations{
+		PVKey{Cluster: "cluster1", Name: "pv-a12ghi5"}: {
+			ByteHours: 1,
+			Cost:      1,
+		},
+	}
 
 	a12jkl6 := NewMockUnitAllocation("cluster1/namespace2/pod-jkl/container6", start, day, &AllocationProperties{
 		Cluster:    "cluster1",
@@ -208,6 +238,12 @@ func GenerateMockAllocationSet(start time.Time) *AllocationSet {
 		ProviderID: "c1nodes",
 		Node:       "c1nodes",
 	})
+	a12jkl6.PVs = PVAllocations{
+		PVKey{Cluster: "cluster1", Name: "pv-a12jkl6"}: {
+			ByteHours: 1,
+			Cost:      1,
+		},
+	}
 
 	a22mno4 := NewMockUnitAllocation("cluster2/namespace2/pod-mno/container4", start, day, &AllocationProperties{
 		Cluster:    "cluster2",
@@ -217,6 +253,12 @@ func GenerateMockAllocationSet(start time.Time) *AllocationSet {
 		ProviderID: "node1",
 		Node:       "node1",
 	})
+	a22mno4.PVs = PVAllocations{
+		PVKey{Cluster: "cluster2", Name: "pv-a22mno4"}: {
+			ByteHours: 1,
+			Cost:      1,
+		},
+	}
 
 	a22mno5 := NewMockUnitAllocation("cluster2/namespace2/pod-mno/container5", start, day, &AllocationProperties{
 		Cluster:    "cluster2",
@@ -226,6 +268,12 @@ func GenerateMockAllocationSet(start time.Time) *AllocationSet {
 		ProviderID: "node1",
 		Node:       "node1",
 	})
+	a22mno5.PVs = PVAllocations{
+		PVKey{Cluster: "cluster2", Name: "pv-a22mno5"}: {
+			ByteHours: 1,
+			Cost:      1,
+		},
+	}
 
 	a22pqr6 := NewMockUnitAllocation("cluster2/namespace2/pod-pqr/container6", start, day, &AllocationProperties{
 		Cluster:    "cluster2",
@@ -235,6 +283,12 @@ func GenerateMockAllocationSet(start time.Time) *AllocationSet {
 		ProviderID: "node2",
 		Node:       "node2",
 	})
+	a22pqr6.PVs = PVAllocations{
+		PVKey{Cluster: "cluster2", Name: "pv-a22pqr6"}: {
+			ByteHours: 1,
+			Cost:      1,
+		},
+	}
 
 	a23stu7 := NewMockUnitAllocation("cluster2/namespace3/pod-stu/container7", start, day, &AllocationProperties{
 		Cluster:    "cluster2",
@@ -244,6 +298,12 @@ func GenerateMockAllocationSet(start time.Time) *AllocationSet {
 		ProviderID: "node2",
 		Node:       "node2",
 	})
+	a23stu7.PVs = PVAllocations{
+		PVKey{Cluster: "cluster2", Name: "pv-a23stu7"}: {
+			ByteHours: 1,
+			Cost:      1,
+		},
+	}
 
 	a23vwx8 := NewMockUnitAllocation("cluster2/namespace3/pod-vwx/container8", start, day, &AllocationProperties{
 		Cluster:    "cluster2",
@@ -253,6 +313,12 @@ func GenerateMockAllocationSet(start time.Time) *AllocationSet {
 		ProviderID: "node3",
 		Node:       "node3",
 	})
+	a23vwx8.PVs = PVAllocations{
+		PVKey{Cluster: "cluster2", Name: "pv-a23vwx8"}: {
+			ByteHours: 1,
+			Cost:      1,
+		},
+	}
 
 	a23vwx9 := NewMockUnitAllocation("cluster2/namespace3/pod-vwx/container9", start, day, &AllocationProperties{
 		Cluster:    "cluster2",
@@ -262,6 +328,12 @@ func GenerateMockAllocationSet(start time.Time) *AllocationSet {
 		ProviderID: "node3",
 		Node:       "node3",
 	})
+	a23vwx9.PVs = PVAllocations{
+		PVKey{Cluster: "cluster2", Name: "pv-a23vwx9"}: {
+			ByteHours: 1,
+			Cost:      1,
+		},
+	}
 
 	// Controllers
 
@@ -476,10 +548,10 @@ func GenerateMockAssetSets(start, end time.Time) []*AssetSet {
 	node3Network.Cost = 2.0
 
 	// Add LoadBalancers
-	cluster2LoadBalancer1 := NewLoadBalancer("namespace2/loadBalancer1", "cluster2", "lb1", start, end, NewWindow(&start, &end))
+	cluster2LoadBalancer1 := NewLoadBalancer("namespace2/loadBalancer1", "cluster2", "lb1", start, end, NewWindow(&start, &end), false, "127.0.0.1")
 	cluster2LoadBalancer1.Cost = 10.0
 
-	cluster2LoadBalancer2 := NewLoadBalancer("namespace2/loadBalancer2", "cluster2", "lb2", start, end, NewWindow(&start, &end))
+	cluster2LoadBalancer2 := NewLoadBalancer("namespace2/loadBalancer2", "cluster2", "lb2", start, end, NewWindow(&start, &end), false, "127.0.0.1")
 	cluster2LoadBalancer2.Cost = 15.0
 
 	assetSet1 := NewAssetSet(start, end, cluster1Nodes, cluster2Node1, cluster2Node2, cluster2Node3, cluster2Disk1,

pkg/kubecost/query.go

@@ -60,12 +60,13 @@ type AllocationQueryOptions struct {
 type AccumulateOption string
 
 const (
-	AccumulateOptionNone  AccumulateOption = ""
-	AccumulateOptionAll   AccumulateOption = "all"
-	AccumulateOptionHour  AccumulateOption = "hour"
-	AccumulateOptionDay   AccumulateOption = "day"
-	AccumulateOptionWeek  AccumulateOption = "week"
-	AccumulateOptionMonth AccumulateOption = "month"
+	AccumulateOptionNone    AccumulateOption = ""
+	AccumulateOptionAll     AccumulateOption = "all"
+	AccumulateOptionHour    AccumulateOption = "hour"
+	AccumulateOptionDay     AccumulateOption = "day"
+	AccumulateOptionWeek    AccumulateOption = "week"
+	AccumulateOptionMonth   AccumulateOption = "month"
+	AccumulateOptionQuarter AccumulateOption = "quarter"
 )
 
 // AssetQueryOptions defines optional parameters for querying an Asset Store
@@ -75,7 +76,7 @@ type AssetQueryOptions struct {
 	Compute                 bool
 	DisableAdjustments      bool
 	DisableAggregatedStores bool
-	FilterFuncs             []AssetMatchFunc
+	Filter                  filter21.Filter
 	IncludeCloud            bool
 	SharedHourlyCosts       map[string]float64
 	Step                    time.Duration
@@ -87,7 +88,7 @@ type CloudUsageQueryOptions struct {
 	Accumulate   bool
 	AggregateBy  []string
 	Compute      bool
-	FilterFuncs  []CloudUsageMatchFunc
+	Filter       filter21.Filter
 	FilterValues CloudUsageFilter
 	LabelConfig  *LabelConfig
 }

pkg/kubecost/summaryallocation.go

@@ -294,12 +294,20 @@ func (sa *SummaryAllocation) IsUnallocated() bool {
 
 // IsUnmounted is true if the given SummaryAllocation represents unmounted
 // volume costs.
+// Note: Due to change in https://github.com/opencost/opencost/pull/1477 made to include Unmounted
+// PVC cost inside namespace we need to check unmounted suffix across all the three major properties
+// to actually classify it as unmounted.
 func (sa *SummaryAllocation) IsUnmounted() bool {
 	if sa == nil {
 		return false
 	}
-
-	return strings.Contains(sa.Name, UnmountedSuffix)
+	props := sa.Properties
+	if props != nil {
+		if props.Container == UnmountedSuffix && props.Namespace == UnmountedSuffix && props.Pod == UnmountedSuffix {
+			return true
+		}
+	}
+	return false
 }
 
 // Minutes returns the number of minutes the SummaryAllocation represents, as
@@ -982,6 +990,7 @@ func (sas *SummaryAllocationSet) AggregateBy(aggregateBy []string, options *Allo
 	// 11. Distribute shared resources according to sharing coefficients.
 	// NOTE: ShareEven is not supported
 	if len(shareSet.SummaryAllocations) > 0 {
+
 		sharingCoeffDenominator := 0.0
 		for _, rt := range allocTotals {
 			sharingCoeffDenominator += rt.TotalCost()
@@ -1000,9 +1009,14 @@ func (sas *SummaryAllocationSet) AggregateBy(aggregateBy []string, options *Allo
 		if sharingCoeffDenominator <= 0.0 {
 			log.Warnf("SummaryAllocation: sharing coefficient denominator is %f", sharingCoeffDenominator)
 		} else {
+
 			// Compute sharing coeffs by dividing the thus-far accumulated
 			// numerators by the now-finalized denominator.
 			for key := range sharingCoeffs {
+				// Do not share the value with unmounted suffix since it's not included in the computation.
+				if key == UnmountedSuffix {
+					continue
+				}
 				if sharingCoeffs[key] > 0.0 {
 					sharingCoeffs[key] /= sharingCoeffDenominator
 				} else {

pkg/kubecost/totals.go

@@ -210,6 +210,7 @@ type AssetTotals struct {
 	PersistentVolumeCostAdjustment  float64   `json:"persistentVolumeCostAdjustment"`
 	RAMCost                         float64   `json:"ramCost"`
 	RAMCostAdjustment               float64   `json:"ramCostAdjustment"`
+	PrivateLoadBalancer             bool      `json:"privateLoadBalancer"`
 }
 
 // ClearAdjustments sets all adjustment fields to 0.0
@@ -245,6 +246,7 @@ func (art *AssetTotals) Clone() *AssetTotals {
 		PersistentVolumeCostAdjustment:  art.PersistentVolumeCostAdjustment,
 		RAMCost:                         art.RAMCost,
 		RAMCostAdjustment:               art.RAMCostAdjustment,
+		PrivateLoadBalancer:             art.PrivateLoadBalancer,
 	}
 }
 
@@ -295,7 +297,7 @@ func (art *AssetTotals) TotalCost() float64 {
 // use the fully-qualified (cluster, node) tuple.
 // NOTE: we're not capturing LoadBalancers here yet, but only because we don't
 // yet need them. They could be added.
-func ComputeAssetTotals(as *AssetSet, prop AssetProperty) map[string]*AssetTotals {
+func ComputeAssetTotals(as *AssetSet, byAsset bool) map[string]*AssetTotals {
 	arts := map[string]*AssetTotals{}
 
 	// Attached disks are tracked by matching their name with the name of the
@@ -306,7 +308,7 @@ func ComputeAssetTotals(as *AssetSet, prop AssetProperty) map[string]*AssetTotal
 	for _, node := range as.Nodes {
 		// Default to computing totals by Cluster, but allow override to use Node.
 		key := node.Properties.Cluster
-		if prop == AssetNodeProp {
+		if byAsset {
 			key = fmt.Sprintf("%s/%s", node.Properties.Cluster, node.Properties.Name)
 		}
 
@@ -397,25 +399,30 @@ func ComputeAssetTotals(as *AssetSet, prop AssetProperty) map[string]*AssetTotal
 		arts[key].GPUCostAdjustment += gpuCostAdjustment
 	}
 
-	// Only record LoadBalancer and ClusterManagement when prop
-	// is cluster. We can't breakdown these types by Node.
-	if prop == AssetClusterProp {
-		for _, lb := range as.LoadBalancers {
-			key := lb.Properties.Cluster
+	for _, lb := range as.LoadBalancers {
+		// Default to computing totals by Cluster, but allow override to use LoadBalancer.
+		key := lb.Properties.Cluster
+		if byAsset {
+			key = fmt.Sprintf("%s/%s", lb.Properties.Cluster, lb.Properties.Name)
+		}
 
-			if _, ok := arts[key]; !ok {
-				arts[key] = &AssetTotals{
-					Start:   lb.Start,
-					End:     lb.End,
-					Cluster: lb.Properties.Cluster,
-				}
+		if _, ok := arts[key]; !ok {
+			arts[key] = &AssetTotals{
+				Start:               lb.Start,
+				End:                 lb.End,
+				Cluster:             lb.Properties.Cluster,
+				Node:                lb.Properties.Name,
+				PrivateLoadBalancer: lb.Private,
 			}
-
-			arts[key].Count++
-			arts[key].LoadBalancerCost += lb.Cost
-			arts[key].LoadBalancerCostAdjustment += lb.Adjustment
 		}
 
+		arts[key].LoadBalancerCost += lb.Cost
+		arts[key].LoadBalancerCostAdjustment += lb.Adjustment
+	}
+
+	// Only record ClusterManagement when prop
+	// is cluster. We can't breakdown these types by Node.
+	if !byAsset {
 		for _, cm := range as.ClusterManagement {
 			key := cm.Properties.Cluster
 
@@ -447,7 +454,7 @@ func ComputeAssetTotals(as *AssetSet, prop AssetProperty) map[string]*AssetTotal
 		// cluster/node. But if we're aggregating by cluster only, then
 		// reset the key to just the cluster.
 		key := name
-		if prop == AssetClusterProp {
+		if !byAsset {
 			key = disk.Properties.Cluster
 		}
 
@@ -458,7 +465,7 @@ func ComputeAssetTotals(as *AssetSet, prop AssetProperty) map[string]*AssetTotal
 				Cluster: disk.Properties.Cluster,
 			}
 
-			if prop == AssetNodeProp {
+			if byAsset {
 				arts[key].Node = disk.Properties.Name
 			}
 		}
@@ -471,10 +478,9 @@ func ComputeAssetTotals(as *AssetSet, prop AssetProperty) map[string]*AssetTotal
 			arts[key].Count++
 			arts[key].AttachedVolumeCost += disk.Cost
 			arts[key].AttachedVolumeCostAdjustment += disk.Adjustment
-		} else if prop == AssetClusterProp {
+		} else {
 			// Here, we're looking at a PersistentVolume because we're not
-			// looking at an AttachedVolume. Only record PersistentVolume data
-			// at the cluster level (i.e. prop == AssetClusterProp).
+			// looking at an AttachedVolume.
 			arts[key].Count++
 			arts[key].PersistentVolumeCost += disk.Cost
 			arts[key].PersistentVolumeCostAdjustment += disk.Adjustment
@@ -621,10 +627,10 @@ func UpdateAssetTotalsStore(arts AssetTotalsStore, as *AssetSet) (*AssetTotalsSe
 	start := *as.Window.Start()
 	end := *as.Window.End()
 
-	artsByCluster := ComputeAssetTotals(as, AssetClusterProp)
+	artsByCluster := ComputeAssetTotals(as, false)
 	arts.SetAssetTotalsByCluster(start, end, artsByCluster)
 
-	artsByNode := ComputeAssetTotals(as, AssetNodeProp)
+	artsByNode := ComputeAssetTotals(as, true)
 	arts.SetAssetTotalsByNode(start, end, artsByNode)
 
 	log.Debugf("ETL: Asset: updated resource totals for %s", as.Window)
@@ -730,6 +736,8 @@ func (mts *MemoryTotalsStore) GetAssetTotalsByCluster(start time.Time, end time.
 func (mts *MemoryTotalsStore) GetAssetTotalsByNode(start time.Time, end time.Time) (map[string]*AssetTotals, bool) {
 	k := storeKey(start, end)
 	if raw, ok := mts.assetTotalsByNode.Get(k); !ok {
+		// it's possible that after accumulation, the time chunks stored here
+		// are being queried combined
 		return map[string]*AssetTotals{}, false
 	} else {
 		original := raw.(map[string]*AssetTotals)

pkg/metrics/deploymentmetrics.go

@@ -42,7 +42,7 @@ func (kdc KubecostDeploymentCollector) Collect(ch chan<- prometheus.Metric) {
 		deploymentName := deployment.GetName()
 		deploymentNS := deployment.GetNamespace()
 
-		labels, values := prom.KubeLabelsToLabels(deployment.Spec.Selector.MatchLabels)
+		labels, values := prom.KubeLabelsToLabels(prom.SanitizeLabels(deployment.Spec.Selector.MatchLabels))
 		if len(labels) > 0 {
 			m := newDeploymentMatchLabelsMetric(deploymentName, deploymentNS, "deployment_match_labels", labels, values)
 			ch <- m

pkg/metrics/namespacemetrics.go

@@ -139,7 +139,7 @@ func (nsac KubeNamespaceCollector) Collect(ch chan<- prometheus.Metric) {
 	for _, namespace := range namespaces {
 		nsName := namespace.GetName()
 
-		labels, values := prom.KubeLabelsToLabels(namespace.Labels)
+		labels, values := prom.KubeLabelsToLabels(prom.SanitizeLabels(namespace.Labels))
 		if len(labels) > 0 {
 			m := newNamespaceAnnotationsMetric("kube_namespace_labels", nsName, labels, values)
 			ch <- m

pkg/metrics/nodemetrics.go

@@ -120,7 +120,7 @@ func (nsac KubeNodeCollector) Collect(ch chan<- prometheus.Metric) {
 
 		// node labels
 		if _, disabled := disabledMetrics["kube_node_labels"]; !disabled {
-			labelNames, labelValues := prom.KubePrependQualifierToLabels(node.GetLabels(), "label_")
+			labelNames, labelValues := prom.KubePrependQualifierToLabels(prom.SanitizeLabels(node.GetLabels()), "label_")
 			ch <- newKubeNodeLabelsMetric(nodeName, "kube_node_labels", labelNames, labelValues)
 		}
 

pkg/metrics/podlabelmetrics.go

@@ -6,36 +6,6 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-//--------------------------------------------------------------------------
-//  KubecostPodCollector
-//--------------------------------------------------------------------------
-
-// KubecostPodCollector is a prometheus collector that emits pod metrics
-type KubecostPodLabelsCollector struct {
-	KubeClusterCache clustercache.ClusterCache
-}
-
-// Describe sends the super-set of all possible descriptors of metrics
-// collected by this Collector.
-func (kpmc KubecostPodLabelsCollector) Describe(ch chan<- *prometheus.Desc) {
-	ch <- prometheus.NewDesc("kube_pod_annotations", "All annotations for each pod prefix with annotation_", []string{}, nil)
-}
-
-// Collect is called by the Prometheus registry when collecting metrics.
-func (kpmc KubecostPodLabelsCollector) Collect(ch chan<- prometheus.Metric) {
-	pods := kpmc.KubeClusterCache.GetAllPods()
-	for _, pod := range pods {
-		podName := pod.GetName()
-		podNS := pod.GetNamespace()
-
-		// Pod Annotations
-		labels, values := prom.KubeAnnotationsToLabels(pod.Annotations)
-		if len(labels) > 0 {
-			ch <- newPodAnnotationMetric("kube_pod_annotations", podNS, podName, labels, values)
-		}
-	}
-}
-
 //--------------------------------------------------------------------------
 //  KubePodLabelsCollector
 //--------------------------------------------------------------------------
@@ -71,7 +41,7 @@ func (kpmc KubePodLabelsCollector) Collect(ch chan<- prometheus.Metric) {
 
 		// Pod Labels
 		if _, disabled := disabledMetrics["kube_pod_labels"]; !disabled {
-			labelNames, labelValues := prom.KubePrependQualifierToLabels(pod.GetLabels(), "label_")
+			labelNames, labelValues := prom.KubePrependQualifierToLabels(prom.SanitizeLabels(pod.GetLabels()), "label_")
 			ch <- newKubePodLabelsMetric("kube_pod_labels", podNS, podName, podUID, labelNames, labelValues)
 		}
 

pkg/metrics/podmetrics.go

@@ -135,7 +135,7 @@ func (kpmc KubePodCollector) Collect(ch chan<- prometheus.Metric) {
 
 		// Pod Labels
 		if _, disabled := disabledMetrics["kube_pod_labels"]; !disabled {
-			labelNames, labelValues := prom.KubePrependQualifierToLabels(pod.GetLabels(), "label_")
+			labelNames, labelValues := prom.KubePrependQualifierToLabels(prom.SanitizeLabels(pod.GetLabels()), "label_")
 			ch <- newKubePodLabelsMetric("kube_pod_labels", podNS, podName, podUID, labelNames, labelValues)
 		}
 

pkg/metrics/servicemetrics.go

@@ -42,7 +42,7 @@ func (sc KubecostServiceCollector) Collect(ch chan<- prometheus.Metric) {
 		serviceName := svc.GetName()
 		serviceNS := svc.GetNamespace()
 
-		labels, values := prom.KubeLabelsToLabels(svc.Spec.Selector)
+		labels, values := prom.KubeLabelsToLabels(prom.SanitizeLabels(svc.Spec.Selector))
 		if len(labels) > 0 {
 			m := newServiceSelectorLabelsMetric(serviceName, serviceNS, "service_selector_labels", labels, values)
 			ch <- m

pkg/metrics/statefulsetmetrics.go

@@ -41,7 +41,7 @@ func (sc KubecostStatefulsetCollector) Collect(ch chan<- prometheus.Metric) {
 		statefulsetName := statefulset.GetName()
 		statefulsetNS := statefulset.GetNamespace()
 
-		labels, values := prom.KubeLabelsToLabels(statefulset.Spec.Selector.MatchLabels)
+		labels, values := prom.KubeLabelsToLabels(prom.SanitizeLabels(statefulset.Spec.Selector.MatchLabels))
 		if len(labels) > 0 {
 			m := newStatefulsetMatchLabelsMetric(statefulsetName, statefulsetNS, "statefulSet_match_labels", labels, values)
 			ch <- m

pkg/metrics/telemetry.go

@@ -2,9 +2,10 @@ package metrics
 
 import (
 	"fmt"
-	"github.com/opencost/opencost/pkg/version"
 	"sync"
 
+	"github.com/opencost/opencost/pkg/version"
+
 	"github.com/kubecost/events"
 	"github.com/prometheus/client_golang/prometheus"
 )

pkg/prom/metrics.go

@@ -104,3 +104,17 @@ func KubeAnnotationsToLabels(labels map[string]string) ([]string, []string) {
 func SanitizeLabelName(s string) string {
 	return invalidLabelCharRE.ReplaceAllString(s, "_")
 }
+
+// SanitizeLabels sanitizes all label names in the given map. This may cause
+// collisions, which is intentional as collisions that are not caught prior to
+// attempted emission will cause fatal errors. In the case of a collision, the
+// last value seen will be set, and all previous values will be overwritten.
+func SanitizeLabels(labels map[string]string) map[string]string {
+	response := make(map[string]string, len(labels))
+
+	for k, v := range labels {
+		response[SanitizeLabelName(k)] = v
+	}
+
+	return response
+}

pkg/prom/metrics_test.go

@@ -2,6 +2,7 @@ package prom
 
 import (
 	"fmt"
+	"reflect"
 	"testing"
 )
 
@@ -93,3 +94,67 @@ func TestKubeLabelsToPromLabels(t *testing.T) {
 		t.Errorf("%s", err)
 	}
 }
+
+func TestSanitizeLabels(t *testing.T) {
+	type testCase struct {
+		in  map[string]string
+		exp map[string]string
+	}
+
+	tcs := map[string]testCase{
+		"empty labels": {
+			in:  map[string]string{},
+			exp: map[string]string{},
+		},
+		"no op": {
+			in: map[string]string{
+				"foo": "bar",
+				"baz": "loo",
+			},
+			exp: map[string]string{
+				"foo": "bar",
+				"baz": "loo",
+			},
+		},
+		"modification, no collisions": {
+			in: map[string]string{
+				"foo-foo":   "bar",
+				"baz---baz": "loo",
+			},
+			exp: map[string]string{
+				"foo_foo":   "bar",
+				"baz___baz": "loo",
+			},
+		},
+		"modification, one collision": {
+			in: map[string]string{
+				"foo-foo":   "bar",
+				"foo+foo":   "bar",
+				"baz---baz": "loo",
+			},
+			exp: map[string]string{
+				"foo_foo":   "bar",
+				"baz___baz": "loo",
+			},
+		},
+		"modification, all collisions": {
+			in: map[string]string{
+				"foo-foo": "bar",
+				"foo+foo": "bar",
+				"foo_foo": "bar",
+			},
+			exp: map[string]string{
+				"foo_foo": "bar",
+			},
+		},
+	}
+
+	for name, tc := range tcs {
+		t.Run(name, func(t *testing.T) {
+			act := SanitizeLabels(tc.in)
+			if !reflect.DeepEqual(tc.exp, act) {
+				t.Errorf("sanitizing labels failed for case %s: %+v != %+v", name, tc.exp, act)
+			}
+		})
+	}
+}

pkg/prom/prom.go

@@ -68,6 +68,9 @@ func (auth *ClientAuth) Apply(req *http.Request) {
 // during a retry. This is to prevent starvation on the request threads
 const MaxRetryAfterDuration = 10 * time.Second
 
+// Default header key for Mimir/Cortex-Tenant API requests
+const HeaderXScopeOrgId = "X-Scope-OrgID"
+
 // RateLimitRetryOpts contains retry options
 type RateLimitRetryOpts struct {
 	MaxRetries       int
@@ -114,14 +117,15 @@ func (rlre *RateLimitedResponseError) Error() string {
 // RateLimitedPrometheusClient is a prometheus client which limits the total number of
 // concurrent outbound requests allowed at a given moment.
 type RateLimitedPrometheusClient struct {
-	id             string
-	client         prometheus.Client
-	auth           *ClientAuth
-	queue          collections.BlockingQueue[*workRequest]
-	decorator      QueryParamsDecorator
-	rateLimitRetry *RateLimitRetryOpts
-	outbound       atomic.Int32
-	fileLogger     *golog.Logger
+	id                string
+	client            prometheus.Client
+	auth              *ClientAuth
+	queue             collections.BlockingQueue[*workRequest]
+	decorator         QueryParamsDecorator
+	rateLimitRetry    *RateLimitRetryOpts
+	outbound          atomic.Int32
+	fileLogger        *golog.Logger
+	headerXScopeOrgId string
 }
 
 // requestCounter is used to determine if the prometheus client keeps track of
@@ -140,7 +144,8 @@ func NewRateLimitedClient(
 	auth *ClientAuth,
 	decorator QueryParamsDecorator,
 	rateLimitRetryOpts *RateLimitRetryOpts,
-	queryLogFile string) (prometheus.Client, error) {
+	queryLogFile string,
+	headerXScopeOrgId string) (prometheus.Client, error) {
 
 	queue := collections.NewBlockingQueue[*workRequest]()
 
@@ -169,13 +174,14 @@ func NewRateLimitedClient(
 	}
 
 	rlpc := &RateLimitedPrometheusClient{
-		id:             id,
-		client:         client,
-		queue:          queue,
-		decorator:      decorator,
-		rateLimitRetry: rateLimitRetryOpts,
-		auth:           auth,
-		fileLogger:     logger,
+		id:                id,
+		client:            client,
+		queue:             queue,
+		decorator:         decorator,
+		rateLimitRetry:    rateLimitRetryOpts,
+		auth:              auth,
+		fileLogger:        logger,
+		headerXScopeOrgId: headerXScopeOrgId,
 	}
 
 	// Start concurrent request processing
@@ -313,6 +319,10 @@ func (rlpc *RateLimitedPrometheusClient) worker() {
 
 // Rate limit and passthrough to prometheus client API
 func (rlpc *RateLimitedPrometheusClient) Do(ctx context.Context, req *http.Request) (*http.Response, []byte, error) {
+	if rlpc.headerXScopeOrgId != "" {
+		req.Header.Set(HeaderXScopeOrgId, rlpc.headerXScopeOrgId)
+	}
+
 	rlpc.auth.Apply(req)
 
 	respChan := make(chan *workResponse)
@@ -353,6 +363,7 @@ type PrometheusClientConfig struct {
 	Auth                  *ClientAuth
 	QueryConcurrency      int
 	QueryLogFile          string
+	HeaderXScopeOrgId     string
 }
 
 // NewPrometheusClient creates a new rate limited client which limits by outbound concurrent requests.
@@ -387,6 +398,7 @@ func NewPrometheusClient(address string, config *PrometheusClientConfig) (promet
 		nil,
 		config.RateLimitRetryOpts,
 		config.QueryLogFile,
+		config.HeaderXScopeOrgId,
 	)
 }
 

pkg/prom/ratelimitedclient_test.go

@@ -142,6 +142,7 @@ func TestRateLimitedOnceAndSuccess(t *testing.T) {
 		nil,
 		newTestRetryOpts(),
 		"",
+		"",
 	)
 
 	if err != nil {
@@ -183,6 +184,7 @@ func TestRateLimitedOnceAndFail(t *testing.T) {
 		nil,
 		newTestRetryOpts(),
 		"",
+		"",
 	)
 
 	if err != nil {
@@ -229,6 +231,7 @@ func TestRateLimitedResponses(t *testing.T) {
 		nil,
 		newTestRetryOpts(),
 		"",
+		"",
 	)
 
 	if err != nil {
@@ -342,6 +345,7 @@ func TestConcurrentRateLimiting(t *testing.T) {
 		nil,
 		newTestRetryOpts(),
 		"",
+		"",
 	)
 
 	if err != nil {

pkg/thanos/thanos.go

@@ -104,5 +104,6 @@ func NewThanosClient(address string, config *prom.PrometheusClientConfig) (prome
 		maxSourceDecorator,
 		config.RateLimitRetryOpts,
 		config.QueryLogFile,
+		"",
 	)
 }

pkg/util/filterutil/asset_test.go

@@ -0,0 +1,470 @@
+package filterutil
+
+import (
+	"testing"
+
+	"github.com/opencost/opencost/pkg/costmodel/clusters"
+	"github.com/opencost/opencost/pkg/kubecost"
+	"github.com/opencost/opencost/pkg/util/mapper"
+)
+
+var assetCompiler = kubecost.NewAssetMatchCompiler()
+
+func TestAssetFiltersFromParamsV1(t *testing.T) {
+	cases := []struct {
+		name           string
+		qp             map[string]string
+		shouldMatch    []kubecost.Asset
+		shouldNotMatch []kubecost.Asset
+	}{
+		{
+			name: "empty",
+			qp:   map[string]string{},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.Node{},
+				&kubecost.Any{},
+				&kubecost.Cloud{},
+				&kubecost.LoadBalancer{},
+				&kubecost.ClusterManagement{},
+				&kubecost.Disk{},
+				&kubecost.Network{},
+				&kubecost.SharedAsset{},
+			},
+			shouldNotMatch: []kubecost.Asset{},
+		},
+		{
+			name: "type: node",
+			qp: map[string]string{
+				ParamFilterTypes: "node",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.Node{},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.Any{},
+				&kubecost.Cloud{},
+				&kubecost.LoadBalancer{},
+				&kubecost.ClusterManagement{},
+				&kubecost.Disk{},
+				&kubecost.Network{},
+				&kubecost.SharedAsset{},
+			},
+		},
+		{
+			name: "type: node capitalized",
+			qp: map[string]string{
+				ParamFilterTypes: "Node",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.Node{},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.Any{},
+				&kubecost.Cloud{},
+				&kubecost.LoadBalancer{},
+				&kubecost.ClusterManagement{},
+				&kubecost.Disk{},
+				&kubecost.Network{},
+				&kubecost.SharedAsset{},
+			},
+		},
+		{
+			name: "type: disk",
+			qp: map[string]string{
+				ParamFilterTypes: "disk",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.Disk{},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.Any{},
+				&kubecost.Cloud{},
+				&kubecost.Network{},
+				&kubecost.Node{},
+				&kubecost.LoadBalancer{},
+				&kubecost.ClusterManagement{},
+				&kubecost.SharedAsset{},
+			},
+		},
+		{
+			name: "type: loadbalancer",
+			qp: map[string]string{
+				ParamFilterTypes: "loadbalancer",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.LoadBalancer{},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.Any{},
+				&kubecost.Cloud{},
+				&kubecost.Node{},
+				&kubecost.ClusterManagement{},
+				&kubecost.Disk{},
+				&kubecost.Network{},
+				&kubecost.SharedAsset{},
+			},
+		},
+		{
+			name: "type: clustermanagement",
+			qp: map[string]string{
+				ParamFilterTypes: "clustermanagement",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.ClusterManagement{},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.Any{},
+				&kubecost.Cloud{},
+				&kubecost.LoadBalancer{},
+				&kubecost.Node{},
+				&kubecost.Disk{},
+				&kubecost.Network{},
+				&kubecost.SharedAsset{},
+			},
+		},
+		{
+			name: "type: network",
+			qp: map[string]string{
+				ParamFilterTypes: "network",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.Network{},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.Any{},
+				&kubecost.Cloud{},
+				&kubecost.LoadBalancer{},
+				&kubecost.ClusterManagement{},
+				&kubecost.Node{},
+				&kubecost.Disk{},
+				&kubecost.SharedAsset{},
+			},
+		},
+		{
+			name: "account",
+			qp: map[string]string{
+				ParamFilterAccounts: "foo,bar",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.Node{
+					Properties: &kubecost.AssetProperties{
+						Account: "foo",
+					},
+				},
+				&kubecost.Network{
+					Properties: &kubecost.AssetProperties{
+						Account: "bar",
+					},
+				},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.Network{
+					Properties: &kubecost.AssetProperties{
+						Account: "baz",
+					},
+				},
+			},
+		},
+		{
+			name: "category",
+			qp: map[string]string{
+				ParamFilterCategories: "Network,Compute",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.Network{
+					Properties: &kubecost.AssetProperties{
+						Category: kubecost.NetworkCategory,
+					},
+				},
+				&kubecost.Node{
+					Properties: &kubecost.AssetProperties{
+						Category: kubecost.ComputeCategory,
+					},
+				},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.ClusterManagement{
+					Properties: &kubecost.AssetProperties{
+						Category: kubecost.ManagementCategory,
+					},
+				},
+			},
+		},
+		{
+			name: "cluster",
+			qp: map[string]string{
+				ParamFilterClusters: "cluster-one",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.LoadBalancer{
+					Properties: &kubecost.AssetProperties{
+						Cluster: "cluster-one",
+					},
+				},
+				&kubecost.Node{
+					Properties: &kubecost.AssetProperties{
+						Cluster: "cluster-one",
+					},
+				},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.ClusterManagement{
+					Properties: &kubecost.AssetProperties{
+						Cluster: "cluster-two",
+					},
+				},
+			},
+		},
+		{
+			name: "project",
+			qp: map[string]string{
+				ParamFilterProjects: "proj1,proj2",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.Disk{
+					Properties: &kubecost.AssetProperties{
+						Project: "proj1",
+					},
+				},
+				&kubecost.Node{
+					Properties: &kubecost.AssetProperties{
+						Project: "proj2",
+					},
+				},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.ClusterManagement{
+					Properties: &kubecost.AssetProperties{
+						Project: "proj3",
+					},
+				},
+			},
+		},
+		{
+			name: "provider",
+			qp: map[string]string{
+				ParamFilterProviders: "p1,p2",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.Disk{
+					Properties: &kubecost.AssetProperties{
+						Provider: "p1",
+					},
+				},
+				&kubecost.Network{
+					Properties: &kubecost.AssetProperties{
+						Provider: "p2",
+					},
+				},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.Node{
+					Properties: &kubecost.AssetProperties{
+						Provider: "p3",
+					},
+				},
+			},
+		},
+		{
+			name: "providerID v1",
+			qp: map[string]string{
+				ParamFilterProviderIDs: "p1,p2",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.Disk{
+					Properties: &kubecost.AssetProperties{
+						ProviderID: "p1",
+					},
+				},
+				&kubecost.Network{
+					Properties: &kubecost.AssetProperties{
+						ProviderID: "p2",
+					},
+				},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.Node{
+					Properties: &kubecost.AssetProperties{
+						ProviderID: "p3",
+					},
+				},
+			},
+		},
+		{
+			name: "providerID v2",
+			qp: map[string]string{
+				ParamFilterProviderIDsV2: "p1,p2",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.Disk{
+					Properties: &kubecost.AssetProperties{
+						ProviderID: "p1",
+					},
+				},
+				&kubecost.Network{
+					Properties: &kubecost.AssetProperties{
+						ProviderID: "p2",
+					},
+				},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.Node{
+					Properties: &kubecost.AssetProperties{
+						ProviderID: "p3",
+					},
+				},
+			},
+		},
+		{
+			name: "service",
+			qp: map[string]string{
+				ParamFilterServices: "p1,p2",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.Disk{
+					Properties: &kubecost.AssetProperties{
+						Service: "p1",
+					},
+				},
+				&kubecost.Network{
+					Properties: &kubecost.AssetProperties{
+						Service: "p2",
+					},
+				},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.Node{
+					Properties: &kubecost.AssetProperties{
+						Service: "p3",
+					},
+				},
+			},
+		},
+		{
+			name: "label",
+			qp: map[string]string{
+				ParamFilterLabels: "foo:bar,baz:qux",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.Disk{
+					Labels: kubecost.AssetLabels{
+						"foo": "bar",
+						"baz": "other",
+					},
+				},
+				&kubecost.Node{
+					Labels: kubecost.AssetLabels{
+						"baz": "qux",
+					},
+				},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.ClusterManagement{
+					Labels: kubecost.AssetLabels{
+						"baz": "other",
+					},
+				},
+			},
+		},
+		{
+			name: "region",
+			qp: map[string]string{
+				ParamFilterRegions: "r1,r2",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.Node{
+					Labels: kubecost.AssetLabels{
+						"label_topology_kubernetes_io_region": "r1",
+					},
+				},
+				&kubecost.Node{
+					Labels: kubecost.AssetLabels{
+						"label_topology_kubernetes_io_region": "r2",
+					},
+				},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.Node{
+					Labels: kubecost.AssetLabels{
+						"label_topology_kubernetes_io_region": "r3",
+					},
+				},
+			},
+		},
+		{
+			name: "complex",
+			qp: map[string]string{
+				ParamFilterRegions:  "r1,r2",
+				ParamFilterTypes:    "node",
+				ParamFilterAccounts: "a*",
+			},
+			shouldMatch: []kubecost.Asset{
+				&kubecost.Node{
+					Labels: kubecost.AssetLabels{
+						"label_topology_kubernetes_io_region": "r1",
+					},
+					Properties: &kubecost.AssetProperties{
+						Account: "a1",
+					},
+				},
+				&kubecost.Node{
+					Labels: kubecost.AssetLabels{
+						"label_topology_kubernetes_io_region": "r2",
+					},
+					Properties: &kubecost.AssetProperties{
+						Account: "a2",
+					},
+				},
+			},
+			shouldNotMatch: []kubecost.Asset{
+				&kubecost.Node{
+					Properties: &kubecost.AssetProperties{
+						Account: "b1",
+					},
+				},
+				&kubecost.Node{
+					Properties: &kubecost.AssetProperties{
+						Account: "3a",
+					},
+				},
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			// Convert map[string]string representation to the mapper
+			// library type
+			qpMap := mapper.NewMap()
+			for k, v := range c.qp {
+				qpMap.Set(k, v)
+			}
+			qpMapper := mapper.NewMapper(qpMap)
+
+			clustersMap := mockClusterMap{
+				m: map[string]*clusters.ClusterInfo{
+					"mapped-cluster-ID-1": {
+						ID:   "mapped-cluster-ID-ABC",
+						Name: "cluster ABC",
+					},
+				},
+			}
+
+			filterTree := AssetFilterFromParamsV1(qpMapper, clustersMap)
+			filter, err := assetCompiler.Compile(filterTree)
+			if err != nil {
+				t.Fatalf("compiling filter: %s", err)
+			}
+			for _, asset := range c.shouldMatch {
+				if !filter.Matches(asset) {
+					t.Errorf("should have matched: %s", asset.String())
+				}
+			}
+			for _, asset := range c.shouldNotMatch {
+				if filter.Matches(asset) {
+					t.Errorf("incorrectly matched: %s", asset.String())
+				}
+			}
+		})
+	}
+}

pkg/util/filterutil/filterutil.go

@@ -7,10 +7,12 @@ import (
 	"github.com/opencost/opencost/pkg/kubecost"
 	"github.com/opencost/opencost/pkg/log"
 	"github.com/opencost/opencost/pkg/prom"
+	"github.com/opencost/opencost/pkg/util/mapper"
 	"github.com/opencost/opencost/pkg/util/typeutil"
 
 	filter "github.com/opencost/opencost/pkg/filter21"
 	afilter "github.com/opencost/opencost/pkg/filter21/allocation"
+	assetfilter "github.com/opencost/opencost/pkg/filter21/asset"
 	"github.com/opencost/opencost/pkg/filter21/ast"
 	// cloudfilter "github.com/opencost/opencost/pkg/filter/cloud"
 )
@@ -27,6 +29,7 @@ import (
 var defaultFieldByType = map[string]any{
 	// typeutil.TypeOf[cloudfilter.CloudAggregationField](): cloudfilter.DefaultFieldByName,
 	typeutil.TypeOf[afilter.AllocationField](): afilter.DefaultFieldByName,
+	typeutil.TypeOf[assetfilter.AssetField]():  assetfilter.DefaultFieldByName,
 }
 
 // DefaultFieldByName looks up a specific T field instance by name and returns the default
@@ -65,8 +68,36 @@ const (
 	ParamFilterAnnotations = "filterAnnotations"
 	ParamFilterLabels      = "filterLabels"
 	ParamFilterServices    = "filterServices"
+
+	ParamFilterAccounts      = "filterAccounts"
+	ParamFilterCategories    = "filterCategories"
+	ParamFilterNames         = "filterNames"
+	ParamFilterProjects      = "filterProjects"
+	ParamFilterProviders     = "filterProviders"
+	ParamFilterProviderIDs   = "filterProviderIDs"
+	ParamFilterProviderIDsV2 = "filterProviderIds"
+	ParamFilterRegions       = "filterRegions"
+	ParamFilterTypes         = "filterTypes"
 )
 
+// ValidAssetFilterParams returns a list of all possible filter parameters
+func ValidAssetFilterParams() []string {
+	return []string{
+		ParamFilterAccounts,
+		ParamFilterCategories,
+		ParamFilterClusters,
+		ParamFilterLabels,
+		ParamFilterNames,
+		ParamFilterProjects,
+		ParamFilterProviders,
+		ParamFilterProviderIDs,
+		ParamFilterProviderIDsV2,
+		ParamFilterRegions,
+		ParamFilterServices,
+		ParamFilterTypes,
+	}
+}
+
 // AllocationPropToV1FilterParamKey maps allocation string property
 // representations to v1 filter param keys for legacy filter config support
 // (e.g. reports). Example mapping: "cluster" -> "filterClusters"
@@ -86,6 +117,22 @@ var AllocationPropToV1FilterParamKey = map[string]string{
 	kubecost.AllocationTeamProp:           ParamFilterTeams,
 }
 
+// Map to store Kubecost Asset property to Asset Filter types.
+// AssetPropToV1FilterParamKey maps asset string property representations to v1
+// filter param keys for legacy filter config support (e.g. reports). Example
+// mapping: "category" -> "filterCategories"
+var AssetPropToV1FilterParamKey = map[kubecost.AssetProperty]string{
+	kubecost.AssetNameProp:       ParamFilterNames,
+	kubecost.AssetTypeProp:       ParamFilterTypes,
+	kubecost.AssetAccountProp:    ParamFilterAccounts,
+	kubecost.AssetCategoryProp:   ParamFilterCategories,
+	kubecost.AssetClusterProp:    ParamFilterClusters,
+	kubecost.AssetProjectProp:    ParamFilterProjects,
+	kubecost.AssetProviderProp:   ParamFilterProviders,
+	kubecost.AssetProviderIDProp: ParamFilterProviderIDs,
+	kubecost.AssetServiceProp:    ParamFilterServices,
+}
+
 // AllHTTPParamKeys returns all HTTP GET parameters used for v1 filters. It is
 // intended to help validate HTTP queries in handlers to help avoid e.g.
 // spelling errors.
@@ -308,6 +355,132 @@ func AllocationFilterFromParamsV1(
 	return andFilter
 }
 
+func AssetFilterFromParamsV1(
+	qp mapper.PrimitiveMapReader,
+	clusterMap clusters.ClusterMap,
+) filter.Filter {
+
+	var filterOps []ast.FilterNode
+
+	// ClusterMap does not provide a cluster name -> cluster ID mapping in the
+	// interface, probably because there could be multiple IDs with the same
+	// name. However, V1 filter logic demands that the parameters to
+	// filterClusters= be checked against both cluster ID AND cluster name.
+	//
+	// To support expected filterClusters= behavior, we construct a mapping
+	// of cluster name -> cluster IDs (could be multiple IDs for the same name)
+	// so that we can create AllocationFilters that use only ClusterIDEquals.
+	//
+	//
+	// AllocationFilter intentionally does not support cluster name filters
+	// because those should be considered presentation-layer only.
+	clusterNameToIDs := map[string][]string{}
+	if clusterMap != nil {
+		cMap := clusterMap.AsMap()
+		for _, info := range cMap {
+			if info == nil {
+				continue
+			}
+
+			if _, ok := clusterNameToIDs[info.Name]; ok {
+				clusterNameToIDs[info.Name] = append(clusterNameToIDs[info.Name], info.ID)
+			} else {
+				clusterNameToIDs[info.Name] = []string{info.ID}
+			}
+		}
+	}
+
+	// The proliferation of > 0 guards in the function is to avoid constructing
+	// empty filter structs. While it is functionally equivalent to add empty
+	// filter structs (they evaluate to true always) there could be overhead
+	// when calling Matches() repeatedly for no purpose.
+
+	if filterClusters := qp.GetList(ParamFilterClusters, ","); len(filterClusters) > 0 {
+		var ops []ast.FilterNode
+
+		// filter my cluster identifier
+		ops = push(ops, filterV1SingleValueFromList(filterClusters, assetfilter.FieldClusterID))
+
+		for _, rawFilterValue := range filterClusters {
+			clusterNameFilter, wildcard := parseWildcardEnd(rawFilterValue)
+
+			clusterIDsToFilter := []string{}
+			for clusterName := range clusterNameToIDs {
+				if wildcard && strings.HasPrefix(clusterName, clusterNameFilter) {
+					clusterIDsToFilter = append(clusterIDsToFilter, clusterNameToIDs[clusterName]...)
+				} else if !wildcard && clusterName == clusterNameFilter {
+					clusterIDsToFilter = append(clusterIDsToFilter, clusterNameToIDs[clusterName]...)
+				}
+			}
+
+			for _, clusterID := range clusterIDsToFilter {
+				ops = append(ops, &ast.EqualOp{
+					Left: ast.Identifier{
+						Field: assetfilter.DefaultFieldByName(assetfilter.FieldClusterID),
+						Key:   "",
+					},
+					Right: clusterID,
+				})
+			}
+		}
+
+		clustersOp := opsToOr(ops)
+		filterOps = push(filterOps, clustersOp)
+	}
+
+	if raw := qp.GetList(ParamFilterAccounts, ","); len(raw) > 0 {
+		filterOps = push(filterOps, filterV1SingleValueFromList(raw, assetfilter.FieldAccount))
+	}
+
+	if raw := qp.GetList(ParamFilterCategories, ","); len(raw) > 0 {
+		filterOps = push(filterOps, filterV1SingleValueFromList(raw, assetfilter.FieldCategory))
+	}
+
+	if raw := qp.GetList(ParamFilterNames, ","); len(raw) > 0 {
+		filterOps = push(filterOps, filterV1SingleValueFromList(raw, assetfilter.FieldName))
+	}
+
+	if raw := qp.GetList(ParamFilterProjects, ","); len(raw) > 0 {
+		filterOps = push(filterOps, filterV1SingleValueFromList(raw, assetfilter.FieldProject))
+	}
+
+	if raw := qp.GetList(ParamFilterProviders, ","); len(raw) > 0 {
+		filterOps = push(filterOps, filterV1SingleValueFromList(raw, assetfilter.FieldProvider))
+	}
+
+	if raw := GetList(ParamFilterProviderIDs, ParamFilterProviderIDsV2, qp); len(raw) > 0 {
+		filterOps = push(filterOps, filterV1SingleValueFromList(raw, assetfilter.FieldProviderID))
+	}
+
+	if raw := qp.GetList(ParamFilterServices, ","); len(raw) > 0 {
+		filterOps = push(filterOps, filterV1SingleValueFromList(raw, assetfilter.FieldService))
+	}
+
+	if raw := qp.GetList(ParamFilterTypes, ","); len(raw) > 0 {
+		// Types have a special situation where we allow users to enter them
+		// capitalized or uncapitalized
+		for i := range raw {
+			raw[i] = strings.ToLower(raw[i])
+		}
+		filterOps = push(filterOps, filterV1SingleValueFromList(raw, assetfilter.FieldType))
+	}
+
+	if raw := qp.GetList(ParamFilterLabels, ","); len(raw) > 0 {
+		filterOps = push(filterOps, filterV1DoubleValueFromList(raw, assetfilter.FieldLabel))
+	}
+
+	if raw := qp.GetList(ParamFilterRegions, ","); len(raw) > 0 {
+		filterOps = push(filterOps, filterV1SingleLabelKeyFromList(raw, "label_topology_kubernetes_io_region", assetfilter.FieldLabel))
+	}
+
+	andFilter := opsToAnd(filterOps)
+	if andFilter == nil {
+		return &ast.VoidOp{} // no filter
+	}
+
+	return andFilter
+}
+
 // filterV1SingleValueFromList creates an OR of equality filters for a given
 // filter field.
 //
@@ -327,6 +500,22 @@ func filterV1SingleValueFromList[T ~string](rawFilterValues []string, filterFiel
 	return opsToOr(ops)
 }
 
+func filterV1SingleLabelKeyFromList[T ~string](rawFilterValues []string, labelName string, labelField T) ast.FilterNode {
+	var ops []ast.FilterNode
+	labelName = prom.SanitizeLabelName(labelName)
+
+	for _, filterValue := range rawFilterValues {
+		filterValue = strings.TrimSpace(filterValue)
+		filterValue, wildcard := parseWildcardEnd(filterValue)
+
+		subFilter := toEqualOp(labelField, labelName, filterValue, wildcard)
+
+		ops = append(ops, subFilter)
+	}
+
+	return opsToOr(ops)
+}
+
 // filterV1LabelAliasMappedFromList is like filterV1SingleValueFromList but is
 // explicitly for labels and annotations because "label-mapped" filters (like filterTeams=)
 // are actually label filters with a fixed label key.
@@ -351,7 +540,7 @@ func filterV1LabelAliasMappedFromList(rawFilterValues []string, labelName string
 //
 // The v1 query language (e.g. "filterLabels=app:foo,l2:bar") uses OR within
 // a field (e.g. label[app] = foo OR label[l2] = bar)
-func filterV1DoubleValueFromList(rawFilterValuesUnsplit []string, filterField afilter.AllocationField) ast.FilterNode {
+func filterV1DoubleValueFromList[T ~string](rawFilterValuesUnsplit []string, filterField T) ast.FilterNode {
 	var ops []ast.FilterNode
 
 	for _, unsplit := range rawFilterValuesUnsplit {
@@ -542,3 +731,13 @@ func toAllocationAliasOp(labelName string, filterValue string, wildcard bool) *a
 		},
 	}
 }
+
+// GetList provides a list of values from the first key if they exist, otherwise, it returns
+// the values from the second key.
+func GetList(primaryKey, secondaryKey string, qp mapper.PrimitiveMapReader) []string {
+	if raw := qp.GetList(primaryKey, ","); len(raw) > 0 {
+		return raw
+	}
+
+	return qp.GetList(secondaryKey, ",")
+}

pkg/util/filterutil/queryfilters_test.go

@@ -47,7 +47,7 @@ func allocGenerator(props kubecost.AllocationProperties) kubecost.Allocation {
 	return a
 }
 
-func TestFiltersFromParamsV1(t *testing.T) {
+func TestAllocationFiltersFromParamsV1(t *testing.T) {
 	// TODO: __unallocated__ case?
 	cases := []struct {
 		name           string

spec/opencost-specv01.md

@@ -1,16 +1,16 @@
 # OpenCost Specification
 
 
-The OpenCost Spec is a vendor-neutral specification for measuring and allocating infrastructure and container costs in Kubernetes environments. 
+The OpenCost Spec is a vendor-neutral specification for measuring and allocating infrastructure and container costs in Kubernetes environments.
 
 
 ## Introduction
 
 
-Kubernetes enables complex deployments of containerized workloads, which are often transient and consume variable amounts of cluster resources. While this enables teams to construct powerful solutions to a broad range of technical problems, it also creates complexities when measuring the resource utilization and costs of workloads and their associated infrastructure within the  dynamics of shared Kubernetes environments. 
+Kubernetes enables complex deployments of containerized workloads, which are often transient and consume variable amounts of cluster resources. While this enables teams to construct powerful solutions to a broad range of technical problems, it also creates complexities when measuring the resource utilization and costs of workloads and their associated infrastructure within the  dynamics of shared Kubernetes environments.
 
 
-As Kubernetes adoption increases within an organization, these complexities become a business-critical challenge to solve. In this document, we specify a vendor-agnostic methodology for accurately measuring and allocating the costs of a Kubernetes cluster to its hosted tenants. This community resource is maintained by Kubernetes practitioners and we welcome all contributions. 
+As Kubernetes adoption increases within an organization, these complexities become a business-critical challenge to solve. In this document, we specify a vendor-agnostic methodology for accurately measuring and allocating the costs of a Kubernetes cluster to its hosted tenants. This community resource is maintained by Kubernetes practitioners and we welcome all contributions.
 
 
 ## Foundational definitions
@@ -46,18 +46,21 @@ Cluster Asset Costs can be further segmented into **Resource Allocation Costs**
    <td><strong>Resource Allocation Costs</strong>
 <p>
 (for all assets)
+</p>
    </td>
    <td><strong>+</strong>
    </td>
    <td><strong>Resource Usage Costs</strong>
 <p>
 (for all assets)
+</p>
    </td>
    <td><strong>+</strong>
    </td>
    <td><strong>Cluster Overhead Costs</strong>
 <p>
 (for cluster)
+</p>
    </td>
   </tr>
 </table>
@@ -65,11 +68,11 @@ Cluster Asset Costs can be further segmented into **Resource Allocation Costs**
 
 The following chart shows these relationships:
 
-<img width="796" alt="image4" src="https://user-images.githubusercontent.com/453512/171577990-8f7c9a53-f5b1-4fbc-b2f6-75cd6ea67960.png">
+<img width="796" alt="image4" src="https://user-images.githubusercontent.com/453512/171577990-8f7c9a53-f5b1-4fbc-b2f6-75cd6ea67960.png"/>
 
 While billing models can differ by environment, below are common examples of segmentation by Allocation, Usage and Overhead Costs.
 
-<img width="292" alt="image1" src="https://user-images.githubusercontent.com/453512/171578190-d84dc3a7-1d20-4575-9bcc-2a5722de5eea.png">
+<img width="292" alt="image1" src="https://user-images.githubusercontent.com/453512/171578190-d84dc3a7-1d20-4575-9bcc-2a5722de5eea.png"/>
 
 
 Once calculated, these Asset Costs can then be distributed to the tenants that consume them, where Workload Costs plus Idle Costs equals Asset Costs. **Workload costs** are expenses that can be directly attributed to a set of Kubernetes workloads, e.g. a container, pod, deployment, etc. **Cluster Idle Costs** are the portion of Resource Allocation Costs that are not allocated to any workload[^1].
@@ -101,7 +104,7 @@ The following chart shows these relationships:
 
 ## Cluster Asset Costs
 
-Cluster Assets are observable entities within a Kubernetes cluster that directly incur costs related to their resources. Asset Costs consist of Resource Allocation Costs and Resource Usage Costs. Every Asset conforming to this specification MUST include at least one cost component with Amount, Unit and Rate attributes as well as a TotalCost value. 
+Cluster Assets are observable entities within a Kubernetes cluster that directly incur costs related to their resources. Asset Costs consist of Resource Allocation Costs and Resource Usage Costs. Every Asset conforming to this specification MUST include at least one cost component with Amount, Unit and Rate attributes as well as a TotalCost value.
 
 Attributes for measured Resource Allocation Costs:
 
@@ -109,7 +112,7 @@ Attributes for measured Resource Allocation Costs:
 
 * [float] Amount - the amount of resource reserved by the asset, e.g. 2 CPU cores
 * [float] Duration - time between the start and end of the allocation period measured in hours, e.g. 24 hours
-* [string] Unit - the amount’s unit of measurement, e.g. CPU cores 
+* [string] Unit - the amount’s unit of measurement, e.g. CPU cores
 * [float] HourlyRate - cost per one unit hour, e.g. $0.2 per CPU hourly rate
 * [float] Total Cost - defined as Amount * Duration * HourlyRate
 
@@ -189,7 +192,7 @@ Workloads are defined as entities to which Asset Costs are committed. Some resou
   <tr>
    <td>Storage Volume
    </td>
-   <td>The storage capacity of Persistent Volume Claim (PVC) requests measured in bytes or gigabytes. Attached at the Kubernetes pod-level. 
+   <td>The storage capacity of Persistent Volume Claim (PVC) requests measured in bytes or gigabytes. Attached at the Kubernetes pod-level.
    </td>
   </tr>
   <tr>
@@ -206,7 +209,7 @@ Workloads are defined as entities to which Asset Costs are committed. Some resou
   </tr>
 </table>
 
-The following workload cost aggregations are supported in a complete implementation in the OpenCost Spec: 
+The following workload cost aggregations are supported in a complete implementation in the OpenCost Spec:
 
 * container
 * pod
@@ -230,7 +233,7 @@ Shared Workload Costs, Cluster Idle Costs, and Overhead Costs are common example
 2. Proportionate to a tenant's consumption of Cluster Asset costs
 3. Custom metric, e.g. bytes of network egress
 
-A full implementation of the spec should support various methods of distributing shared costs. 
+A full implementation of the spec should support various methods of distributing shared costs.
 
 
 ## Idle Costs
@@ -260,6 +263,7 @@ Idle Costs can be calculated at both the Asset/Resource level as well as the Wor
    <td><strong>Cluster </strong>
 <p>
 <strong>Idle %</strong>
+</p>
    </td>
    <td><strong>=</strong>
    </td>
@@ -274,12 +278,12 @@ Idle Costs can be calculated at both the Asset/Resource level as well as the Wor
 
 
 
-## 
+##
 The following chart shows these relationships:
 ![image3](https://user-images.githubusercontent.com/453512/171579570-055bebe8-cc97-4129-9238-c4bcda8e123c.png)
 
 
-Asset Idle Cost can be calculated by individual assets, groups of assets, cluster(s), and by individual resources, e.g. CPU. Resources that are strictly billed on usage can be viewed to have 100% efficiency but should not be included when measuring idle percentage of a cluster. 
+Asset Idle Cost can be calculated by individual assets, groups of assets, cluster(s), and by individual resources, e.g. CPU. Resources that are strictly billed on usage can be viewed to have 100% efficiency but should not be included when measuring idle percentage of a cluster.
 
 Workload Idle Costs is a cost-weighted measurement of [requested](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container) resources that are unused. Workload Idle Costs can be calculated on any grouping of Kubernetes workloads, e.g. containers, pods, labels, annotations, namespaces, etc.
 
@@ -298,7 +302,7 @@ The state of a pod will affect the ability to assign costs and whether a resourc
 **Cluster Assets** – Observable entities within a Kubernetes cluster that directly incur costs related to their resources. Examples include nodes, persistent volumes, attached disks, load balancers.
 
 
-**Container** - An instance of a container image. You may have multiple copies of the same image running at the same time. [More info](https://kubernetes.io/docs/concepts/containers/) 
+**Container** - An instance of a container image. You may have multiple copies of the same image running at the same time. [More info](https://kubernetes.io/docs/concepts/containers/)
 
 
 **Image** - A template of a container which contains software (usually microservices) that needs to be run. [More info](https://kubernetes.io/docs/concepts/containers/images/)
@@ -310,10 +314,10 @@ The state of a pod will affect the ability to assign costs and whether a resourc
 **Pod** - A Kubernetes specific concept that consists of a group of containers. A pod is treated as a single block of resources that may be scheduled or scaled on a cluster. [More info](https://kubernetes.io/docs/concepts/workloads/pods/)
 
 
-**Container Orchestration** - Manages the cluster of server instances and maintains the lifecycle of containers and pods. Scheduling is a function of the container orchestrator which schedules pods/containers to run on a server instance. 
+**Container Orchestration** - Manages the cluster of server instances and maintains the lifecycle of containers and pods. Scheduling is a function of the container orchestrator which schedules pods/containers to run on a server instance.
 
 
-**Cluster** - A group of server instances 
+**Cluster** - A group of server instances
 
 
 **Namespace** - A Kubernetes concept which creates a ‘virtual’ cluster where pods/containers may be deployed and observed discreetly from other namespaces. [More info](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/)
@@ -347,7 +351,7 @@ Sampling Kubernetes resources is recommended with the following metrics / dataso
 ## Appendix C
 
 
-Working examples of OpenCost data to come! 
+Working examples of OpenCost data to come!
 
 
 ## Notes
@@ -357,5 +361,3 @@ Working examples of OpenCost data to come!
 
 [^2]:
      This is because containers are the smallest identifiable unit of "thing that uses resources." For example, the lowest level of reliable CPU usage information is usually a container.
-
-

test/cloud_test.go

@@ -194,7 +194,7 @@ func TestNodePriceFromCSVWithGPU(t *testing.T) {
 
 	c.DownloadPricingData()
 	k := c.GetKey(n.Labels, n)
-	resN, err := c.NodePricing(k)
+	resN, _, err := c.NodePricing(k)
 	if err != nil {
 		t.Errorf("Error in NodePricing: %s", err.Error())
 	} else {
@@ -210,7 +210,7 @@ func TestNodePriceFromCSVWithGPU(t *testing.T) {
 	}
 
 	k2 := c.GetKey(n2.Labels, n2)
-	resN2, err := c.NodePricing(k2)
+	resN2, _, err := c.NodePricing(k2)
 	if err != nil {
 		t.Errorf("Error in NodePricing: %s", err.Error())
 	} else {
@@ -249,7 +249,7 @@ func TestNodePriceFromCSVSpecialChar(t *testing.T) {
 	}
 	c.DownloadPricingData()
 	k := c.GetKey(n.Labels, n)
-	resN, err := c.NodePricing(k)
+	resN, _, err := c.NodePricing(k)
 	if err != nil {
 		t.Errorf("Error in NodePricing: %s", err.Error())
 	} else {
@@ -285,7 +285,7 @@ func TestNodePriceFromCSV(t *testing.T) {
 	}
 	c.DownloadPricingData()
 	k := c.GetKey(n.Labels, n)
-	resN, err := c.NodePricing(k)
+	resN, _, err := c.NodePricing(k)
 	if err != nil {
 		t.Errorf("Error in NodePricing: %s", err.Error())
 	} else {
@@ -302,7 +302,7 @@ func TestNodePriceFromCSV(t *testing.T) {
 	unknownN.Labels["foo"] = labelFooWant
 	unknownN.Labels["topology.kubernetes.io/region"] = "fakeregion"
 	k2 := c.GetKey(unknownN.Labels, unknownN)
-	resN2, _ := c.NodePricing(k2)
+	resN2, _, _ := c.NodePricing(k2)
 	if resN2 != nil {
 		t.Errorf("CSV provider should return nil on missing node")
 	}
@@ -314,7 +314,7 @@ func TestNodePriceFromCSV(t *testing.T) {
 		},
 	}
 	k3 := c.GetKey(n.Labels, n)
-	resN3, _ := c2.NodePricing(k3)
+	resN3, _, _ := c2.NodePricing(k3)
 	if resN3 != nil {
 		t.Errorf("CSV provider should return nil on missing csv")
 	}
@@ -361,7 +361,7 @@ func TestNodePriceFromCSVWithRegion(t *testing.T) {
 	}
 	c.DownloadPricingData()
 	k := c.GetKey(n.Labels, n)
-	resN, err := c.NodePricing(k)
+	resN, _, err := c.NodePricing(k)
 	if err != nil {
 		t.Errorf("Error in NodePricing: %s", err.Error())
 	} else {
@@ -371,7 +371,7 @@ func TestNodePriceFromCSVWithRegion(t *testing.T) {
 		}
 	}
 	k2 := c.GetKey(n2.Labels, n2)
-	resN2, err := c.NodePricing(k2)
+	resN2, _, err := c.NodePricing(k2)
 	if err != nil {
 		t.Errorf("Error in NodePricing: %s", err.Error())
 	} else {
@@ -381,7 +381,7 @@ func TestNodePriceFromCSVWithRegion(t *testing.T) {
 		}
 	}
 	k3 := c.GetKey(n3.Labels, n3)
-	resN3, err := c.NodePricing(k3)
+	resN3, _, err := c.NodePricing(k3)
 	if err != nil {
 		t.Errorf("Error in NodePricing: %s", err.Error())
 	} else {
@@ -398,7 +398,7 @@ func TestNodePriceFromCSVWithRegion(t *testing.T) {
 	unknownN.Labels["topology.kubernetes.io/region"] = "fakeregion"
 	unknownN.Labels["foo"] = labelFooWant
 	k4 := c.GetKey(unknownN.Labels, unknownN)
-	resN4, _ := c.NodePricing(k4)
+	resN4, _, _ := c.NodePricing(k4)
 	if resN4 != nil {
 		t.Errorf("CSV provider should return nil on missing node, instead returned %+v", resN4)
 	}
@@ -410,7 +410,7 @@ func TestNodePriceFromCSVWithRegion(t *testing.T) {
 		},
 	}
 	k5 := c.GetKey(n.Labels, n)
-	resN5, _ := c2.NodePricing(k5)
+	resN5, _, _ := c2.NodePricing(k5)
 	if resN5 != nil {
 		t.Errorf("CSV provider should return nil on missing csv")
 	}
@@ -501,7 +501,7 @@ func TestSourceMatchesFromCSV(t *testing.T) {
 	n2.Labels["foo"] = "labelFooWant"
 
 	k := c.GetKey(n2.Labels, n2)
-	resN, err := c.NodePricing(k)
+	resN, _, err := c.NodePricing(k)
 	if err != nil {
 		t.Errorf("Error in NodePricing: %s", err.Error())
 	} else {
@@ -567,7 +567,7 @@ func TestNodePriceFromCSVWithCase(t *testing.T) {
 
 	c.DownloadPricingData()
 	k := c.GetKey(n.Labels, n)
-	resN, err := c.NodePricing(k)
+	resN, _, err := c.NodePricing(k)
 	if err != nil {
 		t.Errorf("Error in NodePricing: %s", err.Error())
 	} else {
@@ -602,7 +602,7 @@ func TestNodePriceFromCSVByClass(t *testing.T) {
 	c.DownloadPricingData()
 
 	k := c.GetKey(n.Labels, n)
-	resN, err := c.NodePricing(k)
+	resN, _, err := c.NodePricing(k)
 	if err != nil {
 		t.Errorf("Error in NodePricing: %s", err.Error())
 	} else {
@@ -620,7 +620,7 @@ func TestNodePriceFromCSVByClass(t *testing.T) {
 	k2 := c.GetKey(n2.Labels, n)
 
 	c.DownloadPricingData()
-	resN2, err := c.NodePricing(k2)
+	resN2, _, err := c.NodePricing(k2)
 
 	if resN2 != nil {
 		t.Errorf("CSV provider should return nil on missing node, instead returned %+v", resN2)

ui/default.nginx.conf

@@ -60,6 +60,7 @@ server {
     listen [::]:9090;
     resolver 127.0.0.1 valid=5s;
     location /healthz {
+        access_log /dev/null;
         return 200 'OK';
     }
     location /model/ {