Merge branch 'develop' into feature/dependabot

.github/workflows/label-comments.yml

@@ -0,0 +1,45 @@
+name: needs-follow-up-label
+
+on:
+  issue_comment:
+    types: [created]
+  issues:
+    types: [opened, reopened, closed]
+
+jobs:
+  set-follow-up-label:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check comment actor org membership
+        id: response
+        run: |
+          echo "::set-output name=MEMBER_RESPONSE::$(curl -I -H 'Accept: application/vnd.github+json' -H 'Authorization: token ${{ github.token }}' 'https://api.github.com/orgs/kubecost/members/${{ github.actor }}')"
+
+      - name: "Check for non-4XX response"
+        id: membership
+        run: |
+          echo '${{ steps.response.outputs.MEMBER_RESPONSE }}' && echo "::set-output name=IS_MEMBER::$(grep 'HTTP/2 [2]' <<< '${{ steps.response.outputs.MEMBER_RESPONSE }}')"
+
+      - name: Apply needs-follow-up label if this is a new or reopened issue by user not in the org
+        if: ${{ steps.membership.outputs.IS_MEMBER == '' && github.event_name == 'issues' && (github.event.action == 'opened' || github.event.action == 'reopened') }}
+        uses: actions-ecosystem/action-add-labels@v1
+        with:
+          labels: needs-follow-up
+
+      - name: Apply needs-follow-up label if comment by a user not in the org
+        if: ${{ steps.membership.outputs.IS_MEMBER == '' && github.event_name == 'issue_comment' }}
+        uses: actions-ecosystem/action-add-labels@v1
+        with:
+          labels: needs-follow-up
+
+      - name: Remove needs-follow-up label if the issue has been closed
+        if: ${{ github.event_name == 'issues' && github.event.action == 'closed' }}
+        uses: actions-ecosystem/action-remove-labels@v1
+        with:
+          labels: needs-follow-up
+
+      - name: Remove needs-follow-up label if comment by a user in the org
+        if: ${{ steps.membership.outputs.IS_MEMBER != '' && github.event_name == 'issue_comment' }}
+        uses: actions-ecosystem/action-remove-labels@v1
+        with:
+          labels: needs-follow-up

.gitignore

@@ -5,10 +5,13 @@
 ui/.parcel-cache
 ui/.cache
 ui/dist
+ui/.env
 ui/node_modules/
 cmd/costmodel/costmodel
 cmd/costmodel/costmodel-amd64
 cmd/costmodel/costmodel-arm64
+cmd/costmodel/costmodel-tilt
+
 pkg/cloud/azureorphan_test.go
 
 # VS Code

ADOPTERS.MD

@@ -14,3 +14,4 @@ If you would like to be included in this table, please submit a PR to this file
 | Grafana Labs                               | *                                 | end user               | [How Grafana Labs uses and contributes to OpenCost](https://grafana.com/blog/2023/02/02/how-grafana-labs-uses-and-contributes-to-opencost-the-open-source-project-for-real-time-cost-monitoring-in-kubernetes/) |
 | Microsoft                                  | *                                 | Service Provider       | [Leverage OpenCost on Azure Kubernetes Service](http://aka.ms/aks/OpenCost-AKS) |
 | mindcurv group                             | *                                 | Consultancy            | [mindcurv group](https://mindcurv.com/en/) |
+| Zendesk                                    | *                                 | end user               | [Zendesk](https://www.zendesk.com/) |

CONTRIBUTING.md

@@ -32,6 +32,7 @@ Dependencies:
 1. Docker (with `buildx`)
 2. [just](https://github.com/casey/just) (if you don't want to install it , Just read the `justfile` and run the commands manually)
 3. Multi-arch `buildx` builders set up via https://github.com/tonistiigi/binfmt
+4. `manifest-tool` via https://github.com/estesp/manifest-tool
 4. `npm` (if you want to build the UI)
 
 ### Build the backend

Dockerfile.debug

@@ -0,0 +1,21 @@
+# This dockerfile is for development purposes only; do not use this for production deployments
+FROM golang:alpine
+# The prebuilt binary path. This Dockerfile assumes the binary will be built
+# outside of Docker.
+ARG binary_path
+
+WORKDIR /app
+RUN apk add --update --no-cache ca-certificates
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
+
+ADD --chmod=644 ./configs/default.json /models/default.json
+ADD --chmod=644 ./configs/azure.json /models/azure.json
+ADD --chmod=644 ./configs/aws.json /models/aws.json
+ADD --chmod=644 ./configs/gcp.json /models/gcp.json
+ADD --chmod=644 ./configs/alibaba.json /models/alibaba.json
+
+RUN echo "binary_path"
+COPY ${binary_path} main
+
+ENTRYPOINT ["/go/bin/dlv exec --listen=:40000 --api-version=2 --headless=true --accept-multiclient --log --continue /app/main"]
+EXPOSE 9003 40000

MAINTAINERS.md

@@ -7,6 +7,7 @@ Official list of [OpenCost Maintainers](https://github.com/orgs/opencost/teams/o
 | Maintainer | GitHub ID | Affiliation | Email |
 | --------------- | --------- | ----------- | ----------- |
 | Ajay Tripathy | @AjayTripathy | Kubecost | <Ajay@kubecost.com> |
+| Artur Khantimirov | @r2k1 | Microsoft | |
 | Matt Bolt | @​mbolt35 | Kubecost | <matt@kubecost.com> |
 | Matt Ray | @mattray | Kubecost | <mattray@kubecost.com> |
 | Michael Dresser | @michaelmdresser | Kubecost | <michael@kubecost.com> |

Tiltfile

@@ -0,0 +1,130 @@
+load('ext://helm_resource', 'helm_resource', 'helm_repo')
+load('ext://restart_process', 'docker_build_with_restart')
+
+# WARNING: this allows any k8s context for deployment
+#allow_k8s_contexts(k8s_context())
+# To allow a specific context for deployment:
+# allow_k8s_contexts('kubectl-context')
+# See https://docs.tilt.dev/api.html#api.allow_k8s_contexts for default allowed contexts
+
+config.define_string('arch', args=False, usage='amd64')
+config.define_string('docker-repo', args=False, usage='')
+cfg = config.parse()
+
+arch = cfg.get('arch')
+
+docker_platform = "linux/amd64"
+go_arch = "amd64"
+if arch == "arm64":
+    docker_platform = "linux/aarch64"
+    go_arch = "arm64"
+
+docker_repo = cfg.get('docker-repo')
+if docker_repo == None:
+    docker_repo = ''
+else:
+    docker_repo = docker_repo + "/"
+
+# Build and update opencost back end binary when code changes
+local_resource(
+    name='build-costmodel',
+    dir='.',
+    cmd='CGO_ENABLED=0 GOOS=linux GOARCH='+go_arch+' go build -o ./cmd/costmodel/costmodel-tilt ./cmd/costmodel/main.go',
+    deps=[
+        './cmd/costmodel/main.go',
+        './pkg',
+    ],
+    allow_parallel=True,
+    resource_deps=['build-go-mod-download'],
+)
+
+# Build back end docker container
+# If the binary is updated, update the running container and restart binary in dlv
+docker_build_with_restart(
+    ref=docker_repo+'opencost-costmodel',
+    context='.',
+    # remove --continue flag to make dlv wait until debugger is attached to start
+    entrypoint='/go/bin/dlv exec --listen=:40000 --api-version=2 --headless=true --accept-multiclient --log --continue /app/main',
+    dockerfile='Dockerfile.debug',
+    platform=docker_platform,
+
+    build_args={'binary_path':'./cmd/costmodel/costmodel-tilt'},
+    only=[
+        'cmd/costmodel/costmodel-tilt',
+        'configs',
+    ],
+    live_update=[
+       sync('./cmd/costmodel/costmodel-tilt', '/app/main'),
+    ],
+)
+
+# npm install if package.json changes
+local_resource(
+    name='build-npm-install',
+    dir='./ui',
+    cmd='npm install',
+    deps=[
+        './ui/package.json',
+    ],
+    allow_parallel=True,
+)
+
+# Build FE locally when code changes
+local_resource(
+    name='build-ui',
+    dir='./ui',
+    cmd='npx parcel build src/index.html',
+    deps=[
+        './ui/src',
+        './ui/package.json',
+    ],
+    allow_parallel=True,
+    resource_deps=['build-npm-install'],
+)
+
+# update container when relevant files change
+docker_build(
+    ref=docker_repo+'opencost-ui',
+    context='./ui',
+    dockerfile='./ui/Dockerfile.cross',
+    only=[
+        'dist',
+        'nginx.conf',
+        'default.nginx.conf.template',
+        'docker-entrypoint.sh',
+    ],
+    live_update=[
+       sync('./ui/dist', '/var/www'),
+    ],
+)
+
+# build yaml for deployment to k8s
+yaml = helm(
+    '../opencost-helm-chart/charts/opencost',
+    name='opencost',
+    values=['./tilt-values.yaml'],
+    # configuring opencost to also use the kubecost prometheus server below
+    set=[
+        'opencost.ui.image.fullImageName='+docker_repo+'opencost-ui',
+        'opencost.exporter.image.fullImageName='+docker_repo+'opencost-costmodel',
+        'opencost.prometheus.internal.namespaceName='+k8s_namespace(),
+    ]
+)
+k8s_yaml(yaml) # put resulting yaml into k8s
+k8s_resource(workload='opencost', port_forwards=['9003:9003','9090:9090','40000:40000'])
+
+helm_resource(
+    name='prometheus',
+    chart='prometheus-community/prometheus')
+k8s_resource(workload='prometheus', port_forwards=['9080:9090'])
+
+local_resource(
+    name='costmodel-test',
+    dir='.',
+    cmd='go test ./...',
+    deps=[
+        './pkg',
+    ],
+    allow_parallel=True,
+    resource_deps=['opencost'], # run tests after build to speed up deployment
+)

justfile

@@ -1,6 +1,6 @@
 commonenv := "CGO_ENABLED=0"
 
-version := "dev"
+version := `./tools/image-tag`
 commit := `git rev-parse --short HEAD`
 
 default:

kubernetes/exporter/opencost-exporter.yaml

@@ -155,7 +155,21 @@ spec:
               value: "AIzaSyD29bGxmHAVEOBYtgd8sYM2gM2ekfxQX4U" # The GCP Pricing API requires a key. This is supplied just for evaluation.
             - name: CLUSTER_ID
               value: "cluster-one" # Default cluster ID to use if cluster_id is not set in Prometheus metrics.
+            - name: EXPORT_CSV_FILE
+              value: "s3://path/to/csv"
+            - name: AWS_ACCESS_KEY_ID  
+              value: "XXXXXXXXXXXXXXX" ## AWS Access KeyID
+            - name: AWS_SECRET_ACCESS_KEY
+              value: "XXXXXXXXXXXXXXX" ## AWS Secret Access Key
+            - name: AWS_REGION
+              value: "us-west-2" ## AWS Region where bucket is hosted
           imagePullPolicy: Always
+          volumeMounts:
+          - name: tmp-volume
+            mountPath: /tmp
+      volumes:
+      - name: tmp-volume
+        emptyDir: {}
 ---
 
 # Expose the cost model with a service

pkg/cloud/aws/provider_test.go

@@ -2,7 +2,7 @@ package aws
 
 import (
 	"bytes"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"reflect"
@@ -306,7 +306,7 @@ func Test_populate_pricing(t *testing.T) {
 	`
 
 	testResponse := http.Response{
-		Body: ioutil.NopCloser(bytes.NewBufferString(awsUSEastString)),
+		Body: io.NopCloser(bytes.NewBufferString(awsUSEastString)),
 		Request: &http.Request{
 			URL: &url.URL{
 				Scheme: "https",
@@ -445,7 +445,7 @@ func Test_populate_pricing(t *testing.T) {
 	}
 
 	testResponse = http.Response{
-		Body: ioutil.NopCloser(bytes.NewBufferString(awsCnString)),
+		Body: io.NopCloser(bytes.NewBufferString(awsCnString)),
 		Request: &http.Request{
 			URL: &url.URL{
 				Scheme: "https",

pkg/cloud/azure/authorizer.go

@@ -1,80 +1,60 @@
 package azure
 
 import (
-	"encoding/json"
 	"fmt"
 
-	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/opencost/opencost/pkg/util/json"
 )
 
-const AccessKeyAuthorizerType = "AzureAccessKey"
+const DefaultCredentialAuthorizerType = "AzureDefaultCredential"
 
+// Authorizer configs provide credentials from azidentity to connect to Azure services.
 type Authorizer interface {
 	cloud.Authorizer
-	GetBlobCredentials() (azblob.Credential, error)
+	GetCredential() (azcore.TokenCredential, error)
 }
 
 // SelectAuthorizerByType is an implementation of AuthorizerSelectorFn and acts as a register for Authorizer types
 func SelectAuthorizerByType(typeStr string) (Authorizer, error) {
 	switch typeStr {
-	case AccessKeyAuthorizerType:
-		return &AccessKey{}, nil
+	case DefaultCredentialAuthorizerType:
+		return &DefaultAzureCredentialHolder{}, nil
 	default:
 		return nil, fmt.Errorf("azure: provider authorizer type '%s' is not valid", typeStr)
 	}
 }
 
-type AccessKey struct {
-	AccessKey string `json:"accessKey"`
-	Account   string `json:"account"`
-}
+type DefaultAzureCredentialHolder struct{}
+
+func (dac *DefaultAzureCredentialHolder) MarshalJSON() ([]byte, error) {
+	fmap := make(map[string]any, 1)
+	fmap[cloud.AuthorizerTypeProperty] = DefaultCredentialAuthorizerType
 
-func (ak *AccessKey) MarshalJSON() ([]byte, error) {
-	fmap := make(map[string]any, 3)
-	fmap[cloud.AuthorizerTypeProperty] = AccessKeyAuthorizerType
-	fmap["accessKey"] = ak.AccessKey
-	fmap["account"] = ak.Account
 	return json.Marshal(fmap)
 }
 
-func (ak *AccessKey) Validate() error {
-	if ak.AccessKey == "" {
-		return fmt.Errorf("AccessKey: missing access key")
-	}
-	if ak.Account == "" {
-		return fmt.Errorf("AccessKey: missing account")
-	}
+func (dac *DefaultAzureCredentialHolder) Validate() error {
 	return nil
 }
 
-func (ak *AccessKey) Equals(config cloud.Config) bool {
+func (dac *DefaultAzureCredentialHolder) Equals(config cloud.Config) bool {
 	if config == nil {
 		return false
 	}
-	thatConfig, ok := config.(*AccessKey)
+	_, ok := config.(*DefaultAzureCredentialHolder)
 	if !ok {
 		return false
 	}
-
-	if ak.AccessKey != thatConfig.AccessKey {
-		return false
-	}
-	if ak.Account != thatConfig.Account {
-		return false
-	}
-
 	return true
 }
 
-func (ak *AccessKey) Sanitize() cloud.Config {
-	return &AccessKey{
-		AccessKey: cloud.Redacted,
-		Account:   ak.Account,
-	}
+func (dac *DefaultAzureCredentialHolder) Sanitize() cloud.Config {
+	return &DefaultAzureCredentialHolder{}
 }
 
-func (ak *AccessKey) GetBlobCredentials() (azblob.Credential, error) {
-	// Create a default request pipeline using your storage account name and account key.
-	return azblob.NewSharedKeyCredential(ak.Account, ak.AccessKey)
+func (dac *DefaultAzureCredentialHolder) GetCredential() (azcore.TokenCredential, error) {
+	return azidentity.NewDefaultAzureCredential(nil)
 }

pkg/cloud/azure/storageauthorizer.go

@@ -0,0 +1,124 @@
+package azure
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/opencost/opencost/pkg/cloud"
+)
+
+const SharedKeyAuthorizerType = "AzureAccessKey"
+
+// StorageAuthorizer is a service specific Authorizer for Azure Storage, it exists so that we can support existing Shared
+// Key configurations while allowing the Authorizer to have a service agnostic api
+type StorageAuthorizer interface {
+	cloud.Authorizer
+	GetBlobClient(serviceURL string) (*azblob.Client, error)
+}
+
+// SelectStorageAuthorizerByType is an implementation of AuthorizerSelectorFn and acts as a register for Authorizer types
+func SelectStorageAuthorizerByType(typeStr string) (StorageAuthorizer, error) {
+	switch typeStr {
+	case SharedKeyAuthorizerType:
+		return &SharedKeyCredential{}, nil
+	default:
+		authorizer, err := SelectAuthorizerByType(typeStr)
+		if err != nil {
+			return nil, err
+		}
+		return &AuthorizerHolder{authorizer}, nil
+	}
+}
+
+// SharedKeyCredential is a StorageAuthorizer with credentials which cannot be used to authorize other services. This
+// is a legacy auth method which is not included in azidentity
+type SharedKeyCredential struct {
+	AccessKey string `json:"accessKey"`
+	Account   string `json:"account"`
+}
+
+func (skc *SharedKeyCredential) MarshalJSON() ([]byte, error) {
+	fmap := make(map[string]any, 3)
+	fmap[cloud.AuthorizerTypeProperty] = SharedKeyAuthorizerType
+	fmap["accessKey"] = skc.AccessKey
+	fmap["account"] = skc.Account
+	return json.Marshal(fmap)
+}
+
+func (skc *SharedKeyCredential) Validate() error {
+	if skc.AccessKey == "" {
+		return fmt.Errorf("SharedKeyCredential: missing access key")
+	}
+	if skc.Account == "" {
+		return fmt.Errorf("SharedKeyCredential: missing account")
+	}
+	return nil
+}
+
+func (skc *SharedKeyCredential) Equals(config cloud.Config) bool {
+	if config == nil {
+		return false
+	}
+	thatConfig, ok := config.(*SharedKeyCredential)
+	if !ok {
+		return false
+	}
+
+	if skc.AccessKey != thatConfig.AccessKey {
+		return false
+	}
+	if skc.Account != thatConfig.Account {
+		return false
+	}
+
+	return true
+}
+
+func (skc *SharedKeyCredential) Sanitize() cloud.Config {
+	return &SharedKeyCredential{
+		AccessKey: cloud.Redacted,
+		Account:   skc.Account,
+	}
+}
+
+func (skc *SharedKeyCredential) GetBlobClient(serviceURL string) (*azblob.Client, error) {
+	credential, err := azblob.NewSharedKeyCredential(skc.Account, skc.AccessKey)
+	if err != nil {
+		return nil, err
+	}
+	client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, credential, nil)
+	return client, err
+}
+
+// AuthorizerHolder is a StorageAuthorizer implementation that wraps an Authorizer implementation
+type AuthorizerHolder struct {
+	Authorizer
+}
+
+func (ah *AuthorizerHolder) Equals(config cloud.Config) bool {
+	if config == nil {
+		return false
+	}
+	that, ok := config.(*AuthorizerHolder)
+	if !ok {
+		return false
+	}
+
+	return ah.Authorizer.Equals(that.Authorizer)
+}
+
+func (ah *AuthorizerHolder) Sanitize() cloud.Config {
+	return &AuthorizerHolder{Authorizer: ah.Authorizer.Sanitize().(Authorizer)}
+}
+
+func (ah *AuthorizerHolder) GetBlobClient(serviceURL string) (*azblob.Client, error) {
+	// Create a default request pipeline using your storage account name and account key.
+	cred, err := ah.GetCredential()
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := azblob.NewClient(serviceURL, cred, nil)
+	return client, err
+}

pkg/cloud/azure/storagebillingparser.go

@@ -9,7 +9,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
 	"github.com/opencost/opencost/pkg/cloud"
 	"github.com/opencost/opencost/pkg/log"
 )
@@ -36,13 +37,14 @@ func (asbp *AzureStorageBillingParser) ParseBillingData(start, end time.Time, re
 		return err
 	}
 
-	containerURL, err := asbp.getContainer()
+	serviceURL := fmt.Sprintf(asbp.StorageConnection.getBlobURLTemplate(), asbp.Account, "")
+	client, err := asbp.Authorizer.GetBlobClient(serviceURL)
 	if err != nil {
 		asbp.ConnectionStatus = cloud.FailedConnection
 		return err
 	}
 	ctx := context.Background()
-	blobNames, err := asbp.getMostRecentBlobs(start, end, containerURL, ctx)
+	blobNames, err := asbp.getMostRecentBlobs(start, end, client, ctx)
 	if err != nil {
 		asbp.ConnectionStatus = cloud.FailedConnection
 		return err
@@ -54,7 +56,7 @@ func (asbp *AzureStorageBillingParser) ParseBillingData(start, end time.Time, re
 	}
 
 	for _, blobName := range blobNames {
-		blobBytes, err2 := asbp.DownloadBlob(blobName, containerURL, ctx)
+		blobBytes, err2 := asbp.DownloadBlob(blobName, client, ctx)
 		if err2 != nil {
 			asbp.ConnectionStatus = cloud.FailedConnection
 			return err2
@@ -101,7 +103,7 @@ func (asbp *AzureStorageBillingParser) parseCSV(start, end time.Time, reader *cs
 	return nil
 }
 
-func (asbp *AzureStorageBillingParser) getMostRecentBlobs(start, end time.Time, containerURL *azblob.ContainerURL, ctx context.Context) ([]string, error) {
+func (asbp *AzureStorageBillingParser) getMostRecentBlobs(start, end time.Time, client *azblob.Client, ctx context.Context) ([]string, error) {
 	log.Infof("Azure Storage: retrieving most recent reports from: %v - %v", start, end)
 
 	// Get list of month substrings for months contained in the start to end range
@@ -109,33 +111,36 @@ func (asbp *AzureStorageBillingParser) getMostRecentBlobs(start, end time.Time,
 	if err != nil {
 		return nil, err
 	}
-	mostResentBlobs := make(map[string]azblob.BlobItemInternal)
-	for marker := (azblob.Marker{}); marker.NotDone(); {
-		// Get a result segment starting with the blob indicated by the current Marker.
-		listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{})
+	mostResentBlobs := make(map[string]container.BlobItem)
+
+	pager := client.NewListBlobsFlatPager(asbp.Container, &azblob.ListBlobsFlatOptions{
+		Include: container.ListBlobsInclude{Deleted: false, Versions: false},
+	})
+
+	for pager.More() {
+		resp, err := pager.NextPage(ctx)
 		if err != nil {
 			return nil, err
 		}
 
-		// ListBlobs returns the start of the next segment; you MUST use this to get
-		// the next segment (after processing the current result segment).
-		marker = listBlob.NextMarker
-
 		// Using the list of months strings find the most resent blob for each month in the range
-		for _, blobInfo := range listBlob.Segment.BlobItems {
+		for _, blobInfo := range resp.Segment.BlobItems {
+			if blobInfo.Name == nil {
+				continue
+			}
+			// If Container Path configuration exists, check if it is in the blobs name
+			if asbp.Path != "" && !strings.Contains(*blobInfo.Name, asbp.Path) {
+				continue
+			}
 			for _, month := range monthStrs {
-				if strings.Contains(blobInfo.Name, month) {
-					// If Container Path configuration exists, check if it is in the blobs name
-					if asbp.Path != "" && !strings.Contains(blobInfo.Name, asbp.Path) {
-						continue
-					}
-
+				if strings.Contains(*blobInfo.Name, month) {
+					// check if blob is the newest seen for this month
 					if prevBlob, ok := mostResentBlobs[month]; ok {
 						if prevBlob.Properties.CreationTime.After(*blobInfo.Properties.CreationTime) {
 							continue
 						}
 					}
-					mostResentBlobs[month] = blobInfo
+					mostResentBlobs[month] = *blobInfo
 				}
 			}
 		}
@@ -145,7 +150,7 @@ func (asbp *AzureStorageBillingParser) getMostRecentBlobs(start, end time.Time,
 	var blobNames []string
 	for _, month := range monthStrs {
 		if blob, ok := mostResentBlobs[month]; ok {
-			blobNames = append(blobNames, blob.Name)
+			blobNames = append(blobNames, *blob.Name)
 		}
 	}
 

pkg/cloud/azure/storageconfiguration.go

@@ -9,12 +9,12 @@ import (
 )
 
 type StorageConfiguration struct {
-	SubscriptionID string     `json:"subscriptionID"`
-	Account        string     `json:"account"`
-	Container      string     `json:"container"`
-	Path           string     `json:"path"`
-	Cloud          string     `json:"cloud"`
-	Authorizer     Authorizer `json:"authorizer"`
+	SubscriptionID string            `json:"subscriptionID"`
+	Account        string            `json:"account"`
+	Container      string            `json:"container"`
+	Path           string            `json:"path"`
+	Cloud          string            `json:"cloud"`
+	Authorizer     StorageAuthorizer `json:"authorizer"`
 }
 
 // Check ensures that all required fields are set, and throws an error if they are not
@@ -93,7 +93,7 @@ func (sc *StorageConfiguration) Sanitize() cloud.Config {
 		Container:      sc.Container,
 		Path:           sc.Path,
 		Cloud:          sc.Cloud,
-		Authorizer:     sc.Authorizer.Sanitize().(Authorizer),
+		Authorizer:     sc.Authorizer.Sanitize().(StorageAuthorizer),
 	}
 }
 
@@ -153,7 +153,7 @@ func (sc *StorageConfiguration) UnmarshalJSON(b []byte) error {
 	if !ok {
 		return fmt.Errorf("StorageConfiguration: UnmarshalJSON: missing authorizer")
 	}
-	authorizer, err := cloud.AuthorizerFromInterface(authAny, SelectAuthorizerByType)
+	authorizer, err := cloud.AuthorizerFromInterface(authAny, SelectStorageAuthorizerByType)
 	if err != nil {
 		return fmt.Errorf("StorageConfiguration: UnmarshalJSON: %s", err.Error())
 	}
@@ -167,8 +167,8 @@ func ConvertAzureStorageConfigToConfig(asc AzureStorageConfig) cloud.KeyedConfig
 		return nil
 	}
 
-	var authorizer Authorizer
-	authorizer = &AccessKey{
+	var authorizer StorageAuthorizer
+	authorizer = &SharedKeyCredential{
 		AccessKey: asc.AccessKey,
 		Account:   asc.AccountName,
 	}

pkg/cloud/azure/storageconfiguration_test.go

@@ -14,14 +14,14 @@ func TestStorageConfiguration_Validate(t *testing.T) {
 		config   StorageConfiguration
 		expected error
 	}{
-		"valid config Azure AccessKey": {
+		"valid config Azure SharedKeyCredential": {
 			config: StorageConfiguration{
 				SubscriptionID: "subscriptionID",
 				Account:        "account",
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -35,11 +35,11 @@ func TestStorageConfiguration_Validate(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					Account: "account",
 				},
 			},
-			expected: fmt.Errorf("AccessKey: missing access key"),
+			expected: fmt.Errorf("SharedKeyCredential: missing access key"),
 		},
 		"missing authorizer": {
 			config: StorageConfiguration{
@@ -59,7 +59,7 @@ func TestStorageConfiguration_Validate(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -73,7 +73,7 @@ func TestStorageConfiguration_Validate(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -87,7 +87,7 @@ func TestStorageConfiguration_Validate(t *testing.T) {
 				Container:      "",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -101,7 +101,7 @@ func TestStorageConfiguration_Validate(t *testing.T) {
 				Container:      "container",
 				Path:           "",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -115,7 +115,7 @@ func TestStorageConfiguration_Validate(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -155,7 +155,7 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -166,14 +166,36 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
 			},
 			expected: true,
 		},
-
+		"matching config AuthorizerHolder": {
+			left: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AuthorizerHolder{
+					Authorizer: &DefaultAzureCredentialHolder{},
+				},
+			},
+			right: &StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AuthorizerHolder{
+					Authorizer: &DefaultAzureCredentialHolder{},
+				},
+			},
+			expected: true,
+		},
 		"missing both authorizer": {
 			left: StorageConfiguration{
 				SubscriptionID: "subscriptionID",
@@ -208,7 +230,7 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -222,7 +244,7 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -237,6 +259,30 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 			},
 			expected: false,
 		},
+		"differing storage authorizer": {
+			left: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &SharedKeyCredential{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			right: &StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AuthorizerHolder{
+					Authorizer: &DefaultAzureCredentialHolder{},
+				},
+			},
+			expected: false,
+		},
 		"different subscriptionID": {
 			left: StorageConfiguration{
 				SubscriptionID: "subscriptionID",
@@ -244,7 +290,7 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -255,7 +301,7 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -269,7 +315,7 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -280,7 +326,7 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -294,7 +340,7 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -305,7 +351,7 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 				Container:      "container2",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -319,7 +365,7 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -330,7 +376,7 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 				Container:      "container",
 				Path:           "path2",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -344,7 +390,7 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -355,7 +401,7 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud2",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
@@ -369,12 +415,12 @@ func TestStorageConfiguration_Equals(t *testing.T) {
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
 			},
-			right: &AccessKey{
+			right: &SharedKeyCredential{
 				AccessKey: "accessKey",
 				Account:   "account",
 			},
@@ -409,19 +455,31 @@ func TestStorageConfiguration_JSON(t *testing.T) {
 				Authorizer:     nil,
 			},
 		},
-		"AccessKey Authorizer": {
+		"SharedKeyCredential Authorizer": {
 			config: StorageConfiguration{
 				SubscriptionID: "subscriptionID",
 				Account:        "account",
 				Container:      "container",
 				Path:           "path",
 				Cloud:          "cloud",
-				Authorizer: &AccessKey{
+				Authorizer: &SharedKeyCredential{
 					AccessKey: "accessKey",
 					Account:   "account",
 				},
 			},
 		},
+		"Default AuthorizerHolder Authorizer": {
+			config: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AuthorizerHolder{
+					Authorizer: &DefaultAzureCredentialHolder{},
+				},
+			},
+		},
 	}
 
 	for name, testCase := range testCases {

pkg/cloud/azure/storageconnection.go

@@ -4,10 +4,9 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"net/url"
 	"strings"
 
-	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
 	"github.com/opencost/opencost/pkg/cloud"
 	"github.com/opencost/opencost/pkg/log"
 )
@@ -35,25 +34,6 @@ func (sc *StorageConnection) Equals(config cloud.Config) bool {
 	return sc.StorageConfiguration.Equals(&thatConfig.StorageConfiguration)
 }
 
-func (sc *StorageConnection) getContainer() (*azblob.ContainerURL, error) {
-
-	credential, err := sc.Authorizer.GetBlobCredentials()
-	if err != nil {
-		return nil, err
-	}
-
-	p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
-
-	// From the Azure portal, get your storage account blob service URL endpoint.
-	URL, _ := url.Parse(
-		fmt.Sprintf(sc.getBlobURLTemplate(), sc.Account, sc.Container))
-
-	// Create a ContainerURL object that wraps the container URL and a request
-	// pipeline to make requests.
-	containerURL := azblob.NewContainerURL(*URL, p)
-	return &containerURL, nil
-}
-
 // getBlobURLTemplate returns the correct BlobUrl for whichever Cloud storage account is specified by the AzureCloud configuration
 // defaults to the Public Cloud template
 func (sc *StorageConnection) getBlobURLTemplate() string {
@@ -65,22 +45,24 @@ func (sc *StorageConnection) getBlobURLTemplate() string {
 	return "https://%s.blob.core.windows.net/%s"
 }
 
-func (sc *StorageConnection) DownloadBlob(blobName string, containerURL *azblob.ContainerURL, ctx context.Context) ([]byte, error) {
+func (sc *StorageConnection) DownloadBlob(blobName string, client *azblob.Client, ctx context.Context) ([]byte, error) {
 	log.Infof("Azure Storage: retrieving blob: %v", blobName)
 
-	blobURL := containerURL.NewBlobURL(blobName)
-	downloadResponse, err := blobURL.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	downloadResponse, err := client.DownloadStream(ctx, sc.Container, blobName, nil)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("Azure: DownloadBlob: failed to download %w", err)
 	}
 	// NOTE: automatically retries are performed if the connection fails
-	bodyStream := downloadResponse.Body(azblob.RetryReaderOptions{MaxRetryRequests: 20})
+	retryReader := downloadResponse.NewRetryReader(ctx, &azblob.RetryReaderOptions{})
+	defer retryReader.Close()
 
 	// read the body into a buffer
 	downloadedData := bytes.Buffer{}
-	_, err = downloadedData.ReadFrom(bodyStream)
+
+	_, err = downloadedData.ReadFrom(retryReader)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("Azure: DownloadBlob: failed to read downloaded data %w", err)
 	}
+
 	return downloadedData.Bytes(), nil
 }

pkg/cloud/config/configurations_test.go

@@ -31,7 +31,7 @@ var (
 					Container:      "containerName",
 					Path:           "containerPath",
 					Cloud:          "azureCloud",
-					Authorizer: &azure.AccessKey{
+					Authorizer: &azure.SharedKeyCredential{
 						AccessKey: "accessKey",
 						Account:   "accountName",
 					},
@@ -214,11 +214,11 @@ func TestConfigurations_UnmarshalJSON(t *testing.T) {
 		input    any
 		expected *Configurations
 	}{
-		"Azure Storage AccessKey": {
+		"Azure Storage SharedKeyCredential": {
 			input:    azureConfiguration,
 			expected: azureConfiguration,
 		},
-		"Azure Storage AccessKey Conversion": {
+		"Azure Storage SharedKeyCredential Conversion": {
 			input:    azureMultiCloudConf,
 			expected: azureConfiguration,
 		},

pkg/cloud/gcp/bigqueryintegration.go

@@ -2,16 +2,13 @@ package gcp
 
 import (
 	"context"
-	"encoding/json"
+	"errors"
 	"fmt"
-	"regexp"
 	"strings"
 	"time"
 
-	"cloud.google.com/go/bigquery"
 	"github.com/opencost/opencost/pkg/kubecost"
 	"github.com/opencost/opencost/pkg/log"
-	"github.com/opencost/opencost/pkg/util/timeutil"
 	"google.golang.org/api/iterator"
 )
 
@@ -28,6 +25,7 @@ const (
 	LabelsColumnName             = "labels"
 	ResourceNameColumnName       = "resource"
 	CostColumnName               = "cost"
+	ListCostColumnName           = "list_cost"
 	CreditsColumnName            = "credits"
 )
 
@@ -35,8 +33,12 @@ const BiqQueryWherePartitionFmt = `DATE(_PARTITIONTIME) >= "%s" AND DATE(_PARTIT
 const BiqQueryWhereDateFmt = `usage_start_time >= "%s" AND usage_start_time < "%s"`
 
 func (bqi *BigQueryIntegration) GetCloudCost(start time.Time, end time.Time) (*kubecost.CloudCostSetRange, error) {
-	// Build Query
+	cudRates, err := bqi.GetFlexibleCUDRates(start, end)
+	if err != nil {
+		return nil, fmt.Errorf("error retrieving CUD rates: %w", err)
+	}
 
+	// Build Query
 	selectColumns := []string{
 		fmt.Sprintf("TIMESTAMP_TRUNC(usage_start_time, day) as %s", UsageDateColumnName),
 		fmt.Sprintf("billing_account_id as %s", BillingAccountIDColumnName),
@@ -46,7 +48,8 @@ func (bqi *BigQueryIntegration) GetCloudCost(start time.Time, end time.Time) (*k
 		fmt.Sprintf("resource.name as %s", ResourceNameColumnName),
 		fmt.Sprintf("TO_JSON_STRING(labels) as %s", LabelsColumnName),
 		fmt.Sprintf("SUM(cost) as %s", CostColumnName),
-		fmt.Sprintf("IFNULL(SUM((Select SUM(amount) FROM bd.credits)),0) as %s", CreditsColumnName),
+		fmt.Sprintf("SUM(cost_at_list) as %s", ListCostColumnName),
+		fmt.Sprintf("ARRAY_CONCAT_AGG(credits) as %s", CreditsColumnName),
 	}
 
 	groupByColumns := []string{
@@ -59,15 +62,7 @@ func (bqi *BigQueryIntegration) GetCloudCost(start time.Time, end time.Time) (*k
 		ResourceNameColumnName,
 	}
 
-	partitionStart := start
-	partitionEnd := end.AddDate(0, 0, 2)
-	wherePartition := fmt.Sprintf(BiqQueryWherePartitionFmt, partitionStart.Format("2006-01-02"), partitionEnd.Format("2006-01-02"))
-	whereDate := fmt.Sprintf(BiqQueryWhereDateFmt, start.Format("2006-01-02"), end.Format("2006-01-02"))
-
-	whereConjuncts := []string{
-		wherePartition,
-		whereDate,
-	}
+	whereConjuncts := GetWhereConjuncts(start, end)
 
 	columnStr := strings.Join(selectColumns, ", ")
 	table := fmt.Sprintf(" `%s` bd ", bqi.GetBillingDataDataset())
@@ -95,8 +90,11 @@ func (bqi *BigQueryIntegration) GetCloudCost(start time.Time, end time.Time) (*k
 	}
 
 	// Parse query into CloudCostSetRange
+
 	for {
-		var ccl CloudCostLoader
+		ccl := CloudCostLoader{
+			FlexibleCUDRates: cudRates,
+		}
 		err = iter.Next(&ccl)
 		if err == iterator.Done {
 			break
@@ -115,256 +113,126 @@ func (bqi *BigQueryIntegration) GetCloudCost(start time.Time, end time.Time) (*k
 
 }
 
-type CloudCostLoader struct {
-	CloudCost *kubecost.CloudCost
+// GetWhereConjuncts creates a list of Where filter statements that filter for usage start date and partition time
+// additional filters can be added before combining into the final where clause
+func GetWhereConjuncts(start time.Time, end time.Time) []string {
+	partitionStart := start
+	partitionEnd := end.AddDate(0, 0, 2)
+	wherePartition := fmt.Sprintf(BiqQueryWherePartitionFmt, partitionStart.Format("2006-01-02"), partitionEnd.Format("2006-01-02"))
+	whereDate := fmt.Sprintf(BiqQueryWhereDateFmt, start.Format("2006-01-02"), end.Format("2006-01-02"))
+	return []string{wherePartition, whereDate}
 }
 
-// Load populates the fields of a CloudCostValues with bigquery.Value from provided slice
-func (ccl *CloudCostLoader) Load(values []bigquery.Value, schema bigquery.Schema) error {
-
-	// Create Cloud Cost Properties
-	properties := kubecost.CloudCostProperties{
-		Provider: kubecost.GCPProvider,
-	}
-	var window kubecost.Window
-	var description string
-	var listCost float64
-	var credits float64
-
-	for i, field := range schema {
-		if field == nil {
-			log.DedupedErrorf(5, "GCP: BigQuery: found nil field in schema")
-			continue
-		}
+// FlexibleCUDRates are the total amount paid / total amount credited per day for all Flexible CUDs. Since credited will be a negative value
+// this will be a negative ratio. This can then be multiplied with the credits from Flexible CUDs on specific line items to determine
+// the amount paid for the credit it received. This allows us to amortize the Flexible CUD costs which are not associated with resources
+// in the billing export. AmountPayed itself may have some credits on it so a Rate and a NetRate are created.
+// Having both allow us to populate AmortizedCost and AmortizedNetCost respectively.
+type FlexibleCUDRates struct {
+	NetRate float64
+	Rate    float64
+}
 
-		switch field.Name {
-		case UsageDateColumnName:
-			usageDate, ok := values[i].(time.Time)
-			if !ok {
-				// It would be very surprising if an unparsable time came back from the API, so it should be ok to return here.
-				return fmt.Errorf("error parsing usage date: %v", values[0])
-			}
-			// start and end will be the day that the usage occurred on
-			s := usageDate
-			e := s.Add(timeutil.Day)
-			window = kubecost.NewWindow(&s, &e)
-		case BillingAccountIDColumnName:
-			invoiceEntityID, ok := values[i].(string)
-			if !ok {
-				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", BillingAccountIDColumnName, values[i])
-				invoiceEntityID = ""
-			}
-			properties.InvoiceEntityID = invoiceEntityID
-		case ProjectIDColumnName:
-			accountID, ok := values[i].(string)
-			if !ok {
-				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", ProjectIDColumnName, values[i])
-				accountID = ""
-			}
-			properties.AccountID = accountID
-		case ServiceDescriptionColumnName:
-			service, ok := values[i].(string)
-			if !ok {
-				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", ServiceDescriptionColumnName, values[i])
-				service = ""
-			}
-			properties.Service = service
-		case SKUDescriptionColumnName:
-			d, ok := values[i].(string)
-			if !ok {
-				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", SKUDescriptionColumnName, values[i])
-				d = ""
-			}
-			description = d
-		case LabelsColumnName:
-			labelJSON, ok := values[i].(string)
-			if !ok {
-				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", LabelsColumnName, values[i])
-			}
-			labelList := []map[string]string{}
-			err := json.Unmarshal([]byte(labelJSON), &labelList)
-			if err != nil {
-				log.Warnf("GCP Cloud Assets: error unmarshaling GCP CloudCost labels: %s", err)
-			}
-			labels := map[string]string{}
-			for _, pair := range labelList {
-				key := pair["key"]
-				value := pair["value"]
-				labels[key] = value
-			}
-			properties.Labels = labels
-		case ResourceNameColumnName:
-			resouceNameValue := values[i]
-			if resouceNameValue == nil {
-				properties.ProviderID = ""
-				continue
-			}
-			resource, ok := resouceNameValue.(string)
-			if !ok {
-				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", ResourceNameColumnName, values[i])
-				properties.ProviderID = ""
-				continue
-			}
-
-			properties.ProviderID = ParseProviderID(resource)
-		case CostColumnName:
-			cost, ok := values[i].(float64)
-			if !ok {
-				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", CostColumnName, values[i])
-				cost = 0.0
-			}
-			listCost = cost
-		case CreditsColumnName:
-			creditSum, ok := values[i].(float64)
-			if !ok {
-				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", CreditsColumnName, values[i])
-				creditSum = 0.0
-			}
-			credits = creditSum
-		default:
-			log.DedupedErrorf(5, "GCP: BigQuery: found unrecognized column name %s", field.Name)
-		}
+// GetFlexibleCUDRates returns a map of FlexibleCUDRates keyed on the start time of the day which those
+// FlexibleCUDRates were derived from.
+func (bqi *BigQueryIntegration) GetFlexibleCUDRates(start time.Time, end time.Time) (map[time.Time]FlexibleCUDRates, error) {
+	costsByDate, err := bqi.queryFlexibleCUDTotalCosts(start, end)
+	if err != nil {
+		return nil, fmt.Errorf("GetFlexibleCUDRates: %w", err)
 	}
 
-	// Check required Fields
-	if window.IsOpen() {
-		return fmt.Errorf("GCP: BigQuery: error parsing, item had invalid window")
+	creditsByDate, err := bqi.queryFlexibleCUDTotalCredits(start, end)
+	if err != nil {
+		return nil, fmt.Errorf("GetFlexibleCUDRates: %w", err)
 	}
 
-	// Determine Category
-	properties.Category = SelectCategory(properties.Service, description)
-
-	// sum credit and cost for NetCost
-	netCost := listCost + credits
-
-	// Using the NetCost as a 'placeholder' for these costs now, until we can revisit and spend the time to do
-	// the calculations correctly
-	amortizedCost := netCost
-	amortizedNetCost := netCost
-	invoicedCost := netCost
+	results := map[time.Time]FlexibleCUDRates{}
+	for date, amountCredited := range creditsByDate {
+		// Protection against divide by zero
+		if amountCredited == 0 {
+			log.Warnf("GetFlexibleCUDRates: 0 value total credit for Flexible CUDs for date %s", date.Format(time.RFC3339))
+			continue
+		}
+		amountPayed, ok := costsByDate[date]
+		if !ok {
+			log.Warnf("GetFlexibleCUDRates: could not find Flexible CUD payments for date %s", date.Format(time.RFC3339))
+			continue
+		}
 
-	// percent k8s is determined by the presence of labels
-	k8sPercent := 0.0
-	if IsK8s(properties.Labels) {
-		k8sPercent = 1.0
-	}
+		// amountPayed itself may have some credits on it so a Rate and a NetRate are created.
+		// Having both allow us to populate AmortizedCost and AmortizedNetCost respectively.
+		results[date] = FlexibleCUDRates{
+			NetRate: (amountPayed.cost + amountPayed.credits) / amountCredited,
+			Rate:    amountPayed.cost / amountCredited,
+		}
 
-	ccl.CloudCost = &kubecost.CloudCost{
-		Properties: &properties,
-		Window:     window,
-		ListCost: kubecost.CostMetric{
-			Cost:              listCost,
-			KubernetesPercent: k8sPercent,
-		},
-		AmortizedCost: kubecost.CostMetric{
-			Cost:              amortizedCost,
-			KubernetesPercent: k8sPercent,
-		},
-		AmortizedNetCost: kubecost.CostMetric{
-			Cost:              amortizedNetCost,
-			KubernetesPercent: k8sPercent,
-		},
-		InvoicedCost: kubecost.CostMetric{
-			Cost:              invoicedCost,
-			KubernetesPercent: k8sPercent,
-		},
-		NetCost: kubecost.CostMetric{
-			Cost:              netCost,
-			KubernetesPercent: k8sPercent,
-		},
 	}
-
-	return nil
+	return results, nil
 }
 
-func IsK8s(labels map[string]string) bool {
-	if _, ok := labels["goog-gke-volume"]; ok {
-		return true
-	}
+func (bqi *BigQueryIntegration) queryFlexibleCUDTotalCosts(start time.Time, end time.Time) (map[time.Time]flexibleCUDCostTotals, error) {
+	queryFmt := `
+		SELECT
+		  TIMESTAMP_TRUNC(usage_start_time, day) as usage_date, 
+		  sum(cost), 
+		  IFNULL(SUM((Select SUM(amount) FROM bd.credits)),0),
+		FROM %s
+		WHERE %s
+		GROUP BY usage_date, sku.description
+	`
 
-	if _, ok := labels["goog-gke-node"]; ok {
-		return true
-	}
+	table := fmt.Sprintf(" `%s` bd ", bqi.GetBillingDataDataset())
+	whereConjuncts := GetWhereConjuncts(start, end)
+	whereConjuncts = append(whereConjuncts, "sku.description like 'Commitment - dollar based v1:%'")
+	whereClause := strings.Join(whereConjuncts, " AND ")
+	query := fmt.Sprintf(queryFmt, table, whereClause)
 
-	if _, ok := labels["goog-k8s-cluster-name"]; ok {
-		return true
+	iter, err := bqi.Query(context.Background(), query)
+	if err != nil {
+		return nil, fmt.Errorf("queryCUDAmountPayed: query error %w", err)
 	}
-
-	return false
-}
-
-var parseProviderIDRx = regexp.MustCompile("^.+\\/(.+)?") // Capture "gke-cluster-3-default-pool-xxxx-yy" from "projects/###/instances/gke-cluster-3-default-pool-xxxx-yy"
-
-func ParseProviderID(id string) string {
-	match := parseProviderIDRx.FindStringSubmatch(id)
-	if len(match) == 0 {
-		return id
+	var loader FlexibleCUDCostTotalsLoader
+	for {
+		err = iter.Next(&loader)
+		if errors.Is(err, iterator.Done) {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("queryCUDAmountPayed: load error %w", err)
+		}
 	}
-	return match[len(match)-1]
+	return loader.values, nil
 }
 
-func SelectCategory(service, description string) string {
-	s := strings.ToLower(service)
-	d := strings.ToLower(description)
-
-	// Network descriptions
-	if strings.Contains(d, "download") {
-		return kubecost.NetworkCategory
-	}
-	if strings.Contains(d, "network") {
-		return kubecost.NetworkCategory
-	}
-	if strings.Contains(d, "ingress") {
-		return kubecost.NetworkCategory
-	}
-	if strings.Contains(d, "egress") {
-		return kubecost.NetworkCategory
-	}
-	if strings.Contains(d, "static ip") {
-		return kubecost.NetworkCategory
-	}
-	if strings.Contains(d, "external ip") {
-		return kubecost.NetworkCategory
-	}
-	if strings.Contains(d, "load balanced") {
-		return kubecost.NetworkCategory
-	}
-	if strings.Contains(d, "licensing fee") {
-		return kubecost.OtherCategory
-	}
+func (bqi *BigQueryIntegration) queryFlexibleCUDTotalCredits(start time.Time, end time.Time) (map[time.Time]float64, error) {
+	queryFmt := `SELECT
+	TIMESTAMP_TRUNC(usage_start_time, day) as usage_date,
+	sum(credits.amount)
+	FROM %s
+	CROSS JOIN UNNEST(bd.credits) AS credits
+	WHERE %s
+	GROUP BY usage_date, credits.id
+	`
 
-	// Storage Descriptions
-	if strings.Contains(d, "storage") {
-		return kubecost.StorageCategory
-	}
-	if strings.Contains(d, "pd capacity") {
-		return kubecost.StorageCategory
-	}
-	if strings.Contains(d, "pd iops") {
-		return kubecost.StorageCategory
-	}
-	if strings.Contains(d, "pd snapshot") {
-		return kubecost.StorageCategory
-	}
+	table := fmt.Sprintf(" `%s` bd ", bqi.GetBillingDataDataset())
+	whereConjuncts := GetWhereConjuncts(start, end)
+	whereConjuncts = append(whereConjuncts, "credits.type = 'COMMITTED_USAGE_DISCOUNT_DOLLAR_BASE'")
+	whereClause := strings.Join(whereConjuncts, " AND ")
+	query := fmt.Sprintf(queryFmt, table, whereClause)
 
-	// Service Defaults
-	if strings.Contains(s, "storage") {
-		return kubecost.StorageCategory
-	}
-	if strings.Contains(s, "compute") {
-		return kubecost.ComputeCategory
-	}
-	if strings.Contains(s, "sql") {
-		return kubecost.StorageCategory
-	}
-	if strings.Contains(s, "bigquery") {
-		return kubecost.StorageCategory
+	iter, err := bqi.Query(context.Background(), query)
+	if err != nil {
+		return nil, fmt.Errorf("queryFlexibleCUDTotalCredits: query error %w", err)
 	}
-	if strings.Contains(s, "kubernetes") {
-		return kubecost.ManagementCategory
-	} else if strings.Contains(s, "pub/sub") {
-		return kubecost.NetworkCategory
+	var loader FlexibleCUDCreditTotalsLoader
+	for {
+		err = iter.Next(&loader)
+		if errors.Is(err, iterator.Done) {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("queryFlexibleCUDTotalCredits: load error %w", err)
+		}
 	}
-
-	return kubecost.OtherCategory
+	return loader.values, nil
 }

pkg/cloud/gcp/bigqueryintegration_types.go

@@ -0,0 +1,310 @@
+package gcp
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"cloud.google.com/go/bigquery"
+	"github.com/opencost/opencost/pkg/kubecost"
+	"github.com/opencost/opencost/pkg/log"
+	"github.com/opencost/opencost/pkg/util/json"
+	"github.com/opencost/opencost/pkg/util/timeutil"
+)
+
+type CloudCostLoader struct {
+	CloudCost        *kubecost.CloudCost
+	FlexibleCUDRates map[time.Time]FlexibleCUDRates
+}
+
+// Load populates the fields of a CloudCostValues with bigquery.Value from provided slice
+func (ccl *CloudCostLoader) Load(values []bigquery.Value, schema bigquery.Schema) error {
+
+	// Create Cloud Cost Properties
+	properties := kubecost.CloudCostProperties{
+		Provider: kubecost.GCPProvider,
+	}
+	var window kubecost.Window
+	var description string
+	var cost float64
+	var listCost float64
+	var creditAmount float64
+	var cudCreditAmount float64
+	var flexibleCUDCreditAmount float64
+
+	for i, field := range schema {
+		if field == nil {
+			log.DedupedErrorf(5, "GCP: BigQuery: found nil field in schema")
+			continue
+		}
+
+		// ignore nil values
+		if values[i] == nil {
+			continue
+		}
+
+		switch field.Name {
+		case UsageDateColumnName:
+			usageDate, ok := values[i].(time.Time)
+			if !ok {
+				// It would be very surprising if an unparsable time came back from the API, so it should be ok to return here.
+				return fmt.Errorf("error parsing usage date: %v", values[0])
+			}
+			// start and end will be the day that the usage occurred on
+			s := usageDate
+			e := s.Add(timeutil.Day)
+			window = kubecost.NewClosedWindow(s, e)
+		case BillingAccountIDColumnName:
+			invoiceEntityID, ok := values[i].(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", BillingAccountIDColumnName, values[i])
+				invoiceEntityID = ""
+			}
+			properties.InvoiceEntityID = invoiceEntityID
+		case ProjectIDColumnName:
+			accountID, ok := values[i].(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", ProjectIDColumnName, values[i])
+				accountID = ""
+			}
+			properties.AccountID = accountID
+		case ServiceDescriptionColumnName:
+			service, ok := values[i].(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", ServiceDescriptionColumnName, values[i])
+				service = ""
+			}
+			properties.Service = service
+		case SKUDescriptionColumnName:
+			d, ok := values[i].(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", SKUDescriptionColumnName, values[i])
+				d = ""
+			}
+			description = d
+		case LabelsColumnName:
+			labelJSON, ok := values[i].(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", LabelsColumnName, values[i])
+			}
+			labelList := []map[string]string{}
+			err := json.Unmarshal([]byte(labelJSON), &labelList)
+			if err != nil {
+				log.Warnf("GCP Cloud Assets: error unmarshaling GCP CloudCost labels: %s", err)
+			}
+			labels := map[string]string{}
+			for _, pair := range labelList {
+				key := pair["key"]
+				value := pair["value"]
+				labels[key] = value
+			}
+			properties.Labels = labels
+		case ResourceNameColumnName:
+			resouceNameValue := values[i]
+			if resouceNameValue == nil {
+				properties.ProviderID = ""
+				continue
+			}
+			resource, ok := resouceNameValue.(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", ResourceNameColumnName, values[i])
+				properties.ProviderID = ""
+				continue
+			}
+
+			properties.ProviderID = ParseProviderID(resource)
+		case CostColumnName:
+			costValue, ok := values[i].(float64)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", CostColumnName, values[i])
+				costValue = 0.0
+			}
+			cost = costValue
+		case ListCostColumnName:
+			listCostValue, ok := values[i].(float64)
+			if !ok {
+				log.Errorf("error parsing GCP CloudCost %s: %v", ListCostColumnName, values[i])
+				listCostValue = 0
+			}
+			listCost = listCostValue
+		case CreditsColumnName:
+			creditSlice, ok := values[i].([]bigquery.Value)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", CreditsColumnName, values[i])
+			}
+			for _, credit := range creditSlice {
+				creditValues, ok := credit.([]bigquery.Value)
+				if !ok {
+					log.DedupedErrorf(5, "error parsing GCP CloudCost credit values: %v", creditValues)
+					continue
+				}
+				amount, ok := creditValues[1].(float64)
+				if !ok {
+					log.DedupedErrorf(5, "error parsing GCP CloudCost credit amount: %v", creditValues[1])
+					continue
+				}
+				creditType, ok := creditValues[4].(string)
+				if !ok {
+					log.DedupedErrorf(5, "error parsing GCP CloudCost credit type: %v", creditValues[4])
+					continue
+				}
+				switch creditType {
+				case "COMMITTED_USAGE_DISCOUNT":
+					cudCreditAmount += amount
+				case "COMMITTED_USAGE_DISCOUNT_DOLLAR_BASE":
+					flexibleCUDCreditAmount += amount
+				default:
+					creditAmount += amount
+				}
+			}
+		default:
+			log.DedupedErrorf(5, "GCP: BigQuery: found unrecognized column name %s", field.Name)
+		}
+	}
+
+	// Check required Fields
+	if window.IsOpen() {
+		return fmt.Errorf("GCP: BigQuery: error parsing, item had invalid window")
+	}
+
+	// Determine amount paid for credit received from Global CUD
+	var flexibleCUDPayedAmount float64
+	var flexibleCUDNetPayedAmount float64
+	if ccl.FlexibleCUDRates != nil {
+		if rates, ok := ccl.FlexibleCUDRates[*window.Start()]; ok {
+			flexibleCUDNetPayedAmount = flexibleCUDCreditAmount * rates.NetRate
+			flexibleCUDPayedAmount = flexibleCUDCreditAmount * rates.Rate
+		}
+	}
+
+	// Determine Category
+	properties.Category = SelectCategory(properties.Service, description)
+
+	// price_at_list is a new column in the billing export which may be nil
+	if listCost == 0.0 {
+		listCost = cost
+	}
+
+	// Net Cost is cost with all credit amounts applied
+	netCost := cost + creditAmount + cudCreditAmount + flexibleCUDCreditAmount
+
+	// Amortized Cost is Cost plus CUD credits and amortized CUD payments
+	amortizedCost := cost + cudCreditAmount + flexibleCUDCreditAmount + flexibleCUDPayedAmount
+
+	// Amortized Net Cost is Cost with all credits and amortized CUD payments
+	amortizedNetCost := cost + creditAmount + cudCreditAmount + flexibleCUDCreditAmount + flexibleCUDNetPayedAmount
+
+	// Using the NetCost as a 'placeholder' for these costs now, until we can revisit and spend the time to do
+	// the calculations correctly
+	invoicedCost := netCost
+
+	// Update Cost for Commitments that will have matching resource id's and should not their non-amortized costs rolled
+	// into values
+	if strings.HasPrefix(description, "Commitment v1") {
+		listCost = 0
+		netCost = 0
+	}
+
+	// Update Cost for Global CUDs to prevent double counting values, which are added in during amortization
+	if strings.HasPrefix(description, "Commitment - dollar based v1:") {
+		amortizedCost = 0
+		amortizedNetCost = 0
+	}
+
+	// percent k8s is determined by the presence of labels
+	k8sPercent := 0.0
+	if IsK8s(properties.Labels) {
+		k8sPercent = 1.0
+	}
+
+	ccl.CloudCost = &kubecost.CloudCost{
+		Properties: &properties,
+		Window:     window,
+		ListCost: kubecost.CostMetric{
+			Cost:              listCost,
+			KubernetesPercent: k8sPercent,
+		},
+		AmortizedCost: kubecost.CostMetric{
+			Cost:              amortizedCost,
+			KubernetesPercent: k8sPercent,
+		},
+		AmortizedNetCost: kubecost.CostMetric{
+			Cost:              amortizedNetCost,
+			KubernetesPercent: k8sPercent,
+		},
+		InvoicedCost: kubecost.CostMetric{
+			Cost:              invoicedCost,
+			KubernetesPercent: k8sPercent,
+		},
+		NetCost: kubecost.CostMetric{
+			Cost:              netCost,
+			KubernetesPercent: k8sPercent,
+		},
+	}
+
+	return nil
+}
+
+type FlexibleCUDCreditTotalsLoader struct {
+	values map[time.Time]float64
+}
+
+func (ctl *FlexibleCUDCreditTotalsLoader) Load(values []bigquery.Value, schema bigquery.Schema) error {
+
+	usageDate, ok := values[0].(time.Time)
+	if !ok {
+		// It would be very surprising if an unparsable time came back from the API, so it should be ok to return here.
+		return fmt.Errorf("error parsing usage date: %v", values[0])
+	}
+
+	amount, ok := values[1].(float64)
+	if !ok {
+		return fmt.Errorf("error parsing amount: %v", values[1])
+	}
+
+	if ctl.values == nil {
+		ctl.values = map[time.Time]float64{}
+	}
+
+	ctl.values[usageDate] = amount
+
+	return nil
+}
+
+type flexibleCUDCostTotals struct {
+	cost    float64
+	credits float64
+}
+
+type FlexibleCUDCostTotalsLoader struct {
+	values map[time.Time]flexibleCUDCostTotals
+}
+
+func (ctl *FlexibleCUDCostTotalsLoader) Load(values []bigquery.Value, schema bigquery.Schema) error {
+	usageDate, ok := values[0].(time.Time)
+	if !ok {
+		// It would be very surprising if an unparsable time came back from the API, so it should be ok to return here.
+		return fmt.Errorf("error parsing usage date: %v", values[0])
+	}
+
+	cost, ok := values[1].(float64)
+	if !ok {
+		return fmt.Errorf("error parsing cost: %v", values[1])
+	}
+
+	credits, ok := values[2].(float64)
+	if !ok {
+		return fmt.Errorf("error parsing credits: %v", values[2])
+	}
+
+	if ctl.values == nil {
+		ctl.values = map[time.Time]flexibleCUDCostTotals{}
+	}
+
+	ctl.values[usageDate] = flexibleCUDCostTotals{
+		cost:    cost,
+		credits: credits,
+	}
+
+	return nil
+}

pkg/cloud/gcp/cloudcost.go

@@ -0,0 +1,100 @@
+package gcp
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/opencost/opencost/pkg/kubecost"
+)
+
+func IsK8s(labels map[string]string) bool {
+	if _, ok := labels["goog-gke-volume"]; ok {
+		return true
+	}
+
+	if _, ok := labels["goog-gke-node"]; ok {
+		return true
+	}
+
+	if _, ok := labels["goog-k8s-cluster-name"]; ok {
+		return true
+	}
+
+	return false
+}
+
+var parseProviderIDRx = regexp.MustCompile("^.+\\/(.+)?") // Capture "gke-cluster-3-default-pool-xxxx-yy" from "projects/###/instances/gke-cluster-3-default-pool-xxxx-yy"
+
+func ParseProviderID(id string) string {
+	match := parseProviderIDRx.FindStringSubmatch(id)
+	if len(match) == 0 {
+		return id
+	}
+	return match[len(match)-1]
+}
+
+func SelectCategory(service, description string) string {
+	s := strings.ToLower(service)
+	d := strings.ToLower(description)
+
+	// Network descriptions
+	if strings.Contains(d, "download") {
+		return kubecost.NetworkCategory
+	}
+	if strings.Contains(d, "network") {
+		return kubecost.NetworkCategory
+	}
+	if strings.Contains(d, "ingress") {
+		return kubecost.NetworkCategory
+	}
+	if strings.Contains(d, "egress") {
+		return kubecost.NetworkCategory
+	}
+	if strings.Contains(d, "static ip") {
+		return kubecost.NetworkCategory
+	}
+	if strings.Contains(d, "external ip") {
+		return kubecost.NetworkCategory
+	}
+	if strings.Contains(d, "load balanced") {
+		return kubecost.NetworkCategory
+	}
+	if strings.Contains(d, "licensing fee") {
+		return kubecost.OtherCategory
+	}
+
+	// Storage Descriptions
+	if strings.Contains(d, "storage") {
+		return kubecost.StorageCategory
+	}
+	if strings.Contains(d, "pd capacity") {
+		return kubecost.StorageCategory
+	}
+	if strings.Contains(d, "pd iops") {
+		return kubecost.StorageCategory
+	}
+	if strings.Contains(d, "pd snapshot") {
+		return kubecost.StorageCategory
+	}
+
+	// Service Defaults
+	if strings.Contains(s, "storage") {
+		return kubecost.StorageCategory
+	}
+	if strings.Contains(s, "compute") {
+		return kubecost.ComputeCategory
+	}
+	if strings.Contains(s, "sql") {
+		return kubecost.StorageCategory
+	}
+	if strings.Contains(s, "bigquery") {
+		return kubecost.StorageCategory
+	}
+	if strings.Contains(s, "kubernetes") {
+		return kubecost.ManagementCategory
+	} else if strings.Contains(s, "pub/sub") {
+		return kubecost.NetworkCategory
+	}
+
+	return kubecost.OtherCategory
+}

pkg/cloud/gcp/provider.go

@@ -86,7 +86,8 @@ var gcpRegions = []string{
 }
 
 var (
-	nvidiaGPURegex = regexp.MustCompile("(Nvidia Tesla [^ ]+) ")
+	nvidiaTeslaGPURegex = regexp.MustCompile("(Nvidia Tesla [^ ]+) ")
+	nvidiaGPURegex      = regexp.MustCompile("(Nvidia [^ ]+) ")
 	// gce://guestbook-12345/...
 	//  => guestbook-12345
 	gceRegex = regexp.MustCompile("gce://([^/]*)/*")
@@ -772,13 +773,23 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]models.Key, pvKeys m
 				}
 
 				var gpuType string
-				for matchnum, group := range nvidiaGPURegex.FindStringSubmatch(product.Description) {
+				for matchnum, group := range nvidiaTeslaGPURegex.FindStringSubmatch(product.Description) {
 					if matchnum == 1 {
 						gpuType = strings.ToLower(strings.Join(strings.Split(group, " "), "-"))
 						log.Debugf("GCP Billing API: GPU type found: '%s'", gpuType)
 					}
 				}
 
+				// If a 'Nvidia Tesla' is not found, try 'Nvidia'
+				if gpuType == "" {
+					for matchnum, group := range nvidiaGPURegex.FindStringSubmatch(product.Description) {
+						if matchnum == 1 {
+							gpuType = strings.ToLower(strings.Join(strings.Split(group, " "), "-"))
+							log.Debugf("GCP Billing API: GPU type found: '%s'", gpuType)
+						}
+					}
+				}
+
 				candidateKeys := []string{}
 				if gcp.ValidPricingKeys == nil {
 					gcp.ValidPricingKeys = make(map[string]bool)
@@ -985,7 +996,6 @@ func (gcp *GCP) parsePages(inputKeys map[string]models.Key, pvKeys map[string]mo
 
 	url := gcp.getBillingAPIURL(gcp.APIKey, c.CurrencyCode)
 
-	log.Infof("Fetch GCP Billing Data from URL: %s", url)
 	var parsePagesHelper func(string) error
 	parsePagesHelper = func(pageToken string) error {
 		if pageToken == "done" {

pkg/cloud/scaleway/provider.go

@@ -207,7 +207,13 @@ func (key *scalewayPVKey) Features() string {
 
 func (c *Scaleway) GetPVKey(pv *v1.PersistentVolume, parameters map[string]string, defaultRegion string) models.PVKey {
 	// the csi volume handle is the form <az>/<volume-id>
-	zone := strings.Split(pv.Spec.CSI.VolumeHandle, "/")[0]
+	zone := ""
+	if pv.Spec.CSI != nil {
+		zoneVolID := strings.Split(pv.Spec.CSI.VolumeHandle, "/")
+		if len(zoneVolID) > 0 {
+			zone = zoneVolID[0]
+		}
+	}
 	return &scalewayPVKey{
 		Labels:                 pv.Labels,
 		StorageClassName:       pv.Spec.StorageClassName,

pkg/cloudcost/memoryrepository_test.go

@@ -2,6 +2,7 @@ package cloudcost
 
 import (
 	"reflect"
+	"sort"
 	"testing"
 	"time"
 
@@ -172,6 +173,8 @@ func TestMemoryRepository_Keys(t *testing.T) {
 				t.Errorf("Keys() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
+			sort.Strings(got)
+			sort.Strings(tt.want)
 			if !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("Keys() got = %v, want %v", got, tt.want)
 			}

pkg/cloudcost/querier.go

@@ -29,7 +29,7 @@ const DefaultChartItemsLength int = 10
 // ViewQuerier defines a contract for return View types to the QueryService to service the View Api
 type ViewQuerier interface {
 	QueryViewGraph(ViewQueryRequest, context.Context) (ViewGraphData, error)
-	QueryViewTotals(ViewQueryRequest, context.Context) (*ViewTableRow, int, error)
+	QueryViewTotals(ViewQueryRequest, context.Context) (*ViewTotals, error)
 	QueryViewTable(ViewQueryRequest, context.Context) (ViewTableRows, error)
 }
 

pkg/cloudcost/queryservice.go

@@ -106,11 +106,6 @@ func (s *QueryService) GetCloudCostViewGraphHandler() func(w http.ResponseWriter
 	}
 }
 
-type CloudCostViewTotalsResponse struct {
-	NumResults int           `json:"numResults"`
-	Combined   *ViewTableRow `json:"combined"`
-}
-
 func (s *QueryService) GetCloudCostViewTotalsHandler() func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	// Return valid handler func
 	return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
@@ -136,17 +131,12 @@ func (s *QueryService) GetCloudCostViewTotalsHandler() func(w http.ResponseWrite
 			return
 		}
 
-		totals, count, err := s.ViewQuerier.QueryViewTotals(*request, ctx)
+		resp, err := s.ViewQuerier.QueryViewTotals(*request, ctx)
 		if err != nil {
 			http.Error(w, fmt.Sprintf("Internal server error: %s", err), http.StatusInternalServerError)
 			return
 		}
 
-		resp := CloudCostViewTotalsResponse{
-			NumResults: count,
-			Combined:   totals,
-		}
-
 		_, spanResp := tracer.Start(ctx, "write response")
 		w.Header().Set("Content-Type", "application/json")
 		protocol.WriteData(w, resp)

pkg/cloudcost/queryservice_helper.go

@@ -38,6 +38,11 @@ func ParseCloudCostRequest(qp httputil.QueryParams) (*QueryRequest, error) {
 		aggregateBy = append(aggregateBy, prop)
 	}
 
+	// if we're aggregating by nothing (aka `item` on the frontend) then aggregate by all
+	if len(aggregateBy) == 0 {
+		aggregateBy = []string{kubecost.CloudCostInvoiceEntityIDProp, kubecost.CloudCostAccountIDProp, kubecost.CloudCostProviderProp, kubecost.CloudCostProviderIDProp, kubecost.CloudCostCategoryProp, kubecost.CloudCostServiceProp}
+	}
+
 	accumulate := kubecost.ParseAccumulate(qp.Get("accumulate", ""))
 
 	var filter filter21.Filter

pkg/cloudcost/queryservice_helper_test.go

@@ -41,7 +41,7 @@ func TestParseCloudCostRequest(t *testing.T) {
 			want: &QueryRequest{
 				Start:       start,
 				End:         end,
-				AggregateBy: nil,
+				AggregateBy: []string{kubecost.CloudCostInvoiceEntityIDProp, kubecost.CloudCostAccountIDProp, kubecost.CloudCostProviderProp, kubecost.CloudCostProviderIDProp, kubecost.CloudCostCategoryProp, kubecost.CloudCostServiceProp},
 				Accumulate:  "",
 				Filter:      nil,
 			},
@@ -77,7 +77,7 @@ func TestParseCloudCostRequest(t *testing.T) {
 			want: &QueryRequest{
 				Start:       start,
 				End:         end,
-				AggregateBy: nil,
+				AggregateBy: []string{kubecost.CloudCostInvoiceEntityIDProp, kubecost.CloudCostAccountIDProp, kubecost.CloudCostProviderProp, kubecost.CloudCostProviderIDProp, kubecost.CloudCostCategoryProp, kubecost.CloudCostServiceProp},
 				Accumulate:  kubecost.AccumulateOptionWeek,
 				Filter:      nil,
 			},
@@ -91,7 +91,7 @@ func TestParseCloudCostRequest(t *testing.T) {
 			want: &QueryRequest{
 				Start:       start,
 				End:         end,
-				AggregateBy: nil,
+				AggregateBy: []string{kubecost.CloudCostInvoiceEntityIDProp, kubecost.CloudCostAccountIDProp, kubecost.CloudCostProviderProp, kubecost.CloudCostProviderIDProp, kubecost.CloudCostCategoryProp, kubecost.CloudCostServiceProp},
 				Accumulate:  kubecost.AccumulateOptionNone,
 				Filter:      nil,
 			},
@@ -105,7 +105,7 @@ func TestParseCloudCostRequest(t *testing.T) {
 			want: &QueryRequest{
 				Start:       start,
 				End:         end,
-				AggregateBy: nil,
+				AggregateBy: []string{kubecost.CloudCostInvoiceEntityIDProp, kubecost.CloudCostAccountIDProp, kubecost.CloudCostProviderProp, kubecost.CloudCostProviderIDProp, kubecost.CloudCostCategoryProp, kubecost.CloudCostServiceProp},
 				Accumulate:  kubecost.AccumulateOptionNone,
 				Filter:      validFilter,
 			},

pkg/cloudcost/repositoryquerier.go

@@ -114,42 +114,45 @@ func (rq *RepositoryQuerier) QueryViewGraph(request ViewQueryRequest, ctx contex
 	return sets, nil
 }
 
-func (rq *RepositoryQuerier) QueryViewTotals(request ViewQueryRequest, ctx context.Context) (*ViewTableRow, int, error) {
+func (rq *RepositoryQuerier) QueryViewTotals(request ViewQueryRequest, ctx context.Context) (*ViewTotals, error) {
 	ccasr, err := rq.Query(request.QueryRequest, ctx)
 	if err != nil {
-		return nil, -1, fmt.Errorf("QueryViewTotals: query failed: %w", err)
+		return nil, fmt.Errorf("QueryViewTotals: query failed: %w", err)
 	}
 	acc, err := ccasr.AccumulateAll()
 	if err != nil {
-		return nil, -1, fmt.Errorf("QueryViewTotals: accumulate failed: %w", err)
+		return nil, fmt.Errorf("QueryViewTotals: accumulate failed: %w", err)
 	}
 	if acc.IsEmpty() {
-		return nil, 0, nil
+		return nil, nil
 	}
 	count := len(acc.CloudCosts)
 
 	total, err := acc.Aggregate([]string{})
 	if err != nil {
-		return nil, -1, fmt.Errorf("QueryViewTotals: aggregate total failed: %w", err)
+		return nil, fmt.Errorf("QueryViewTotals: aggregate total failed: %w", err)
 	}
 
 	if total.IsEmpty() {
-		return nil, -1, fmt.Errorf("QueryViewTotals: missing total: %w", err)
+		return nil, fmt.Errorf("QueryViewTotals: missing total: %w", err)
 	}
 
 	if len(total.CloudCosts) != 1 {
-		return nil, -1, fmt.Errorf("QueryViewTotals: total did not aggregate: %w", err)
+		return nil, fmt.Errorf("QueryViewTotals: total did not aggregate: %w", err)
 	}
 
 	cm, err := total.CloudCosts[""].GetCostMetric(request.CostMetricName)
 	if err != nil {
-		return nil, -1, fmt.Errorf("QueryViewTotals: failed to retrieve cost metric: %w", err)
-	}
-	return &ViewTableRow{
-		Name:              "Totals",
-		KubernetesPercent: cm.KubernetesPercent,
-		Cost:              cm.Cost,
-	}, count, nil
+		return nil, fmt.Errorf("QueryViewTotals: failed to retrieve cost metric: %w", err)
+	}
+	return &ViewTotals{
+		NumResults: count,
+		Combined: &ViewTableRow{
+			Name:              "Totals",
+			KubernetesPercent: cm.KubernetesPercent,
+			Cost:              cm.Cost,
+		},
+	}, nil
 }
 
 func (rq *RepositoryQuerier) QueryViewTable(request ViewQueryRequest, ctx context.Context) (ViewTableRows, error) {
@@ -168,8 +171,14 @@ func (rq *RepositoryQuerier) QueryViewTable(request ViewQueryRequest, ctx contex
 		if err2 != nil {
 			return nil, fmt.Errorf("QueryViewTable: failed to retrieve cost metric: %w", err)
 		}
+		var labels map[string]string
+		if cloudCost.Properties != nil {
+			labels = cloudCost.Properties.Labels
+		}
+
 		vtr := &ViewTableRow{
 			Name:              key,
+			Labels:            labels,
 			KubernetesPercent: costMetric.KubernetesPercent,
 			Cost:              costMetric.Cost,
 		}

pkg/cloudcost/view.go

@@ -31,9 +31,10 @@ func (vtrs ViewTableRows) Equal(that ViewTableRows) bool {
 }
 
 type ViewTableRow struct {
-	Name              string  `json:"name"`
-	KubernetesPercent float64 `json:"kubernetesPercent"`
-	Cost              float64 `json:"cost"`
+	Name              string            `json:"name"`
+	Labels            map[string]string `json:"labels"`
+	KubernetesPercent float64           `json:"kubernetesPercent"`
+	Cost              float64           `json:"cost"`
 }
 
 func (vtr *ViewTableRow) Equal(that *ViewTableRow) bool {
@@ -41,6 +42,20 @@ func (vtr *ViewTableRow) Equal(that *ViewTableRow) bool {
 		return false
 	}
 
+	if len(vtr.Labels) != len(that.Labels) {
+		return false
+	}
+
+	for key, value := range vtr.Labels {
+		thatValue, ok := that.Labels[key]
+		if !ok {
+			return false
+		}
+		if value != thatValue {
+			return false
+		}
+	}
+
 	if !mathutil.Approximately(vtr.KubernetesPercent, that.KubernetesPercent) {
 		return false
 	}
@@ -105,3 +120,8 @@ func (vgdsi ViewGraphDataSetItem) Equal(that ViewGraphDataSetItem) bool {
 
 	return true
 }
+
+type ViewTotals struct {
+	NumResults int           `json:"numResults"`
+	Combined   *ViewTableRow `json:"combined"`
+}

pkg/cmd/costmodel/costmodel.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"net/http/pprof"
 	"time"
 
 	"github.com/julienschmidt/httprouter"
@@ -62,12 +63,22 @@ func Execute(opts *CostModelOpts) error {
 	a.Router.GET("/cloudCost/rebuild", a.CloudCostPipelineService.GetCloudCostRebuildHandler())
 	a.Router.GET("/cloudCost/repair", a.CloudCostPipelineService.GetCloudCostRepairHandler())
 
+	if env.IsPProfEnabled() {
+		a.Router.HandlerFunc(http.MethodGet, "/debug/pprof/", pprof.Index)
+		a.Router.HandlerFunc(http.MethodGet, "/debug/pprof/cmdline", pprof.Cmdline)
+		a.Router.HandlerFunc(http.MethodGet, "/debug/pprof/profile", pprof.Profile)
+		a.Router.HandlerFunc(http.MethodGet, "/debug/pprof/symbol", pprof.Symbol)
+		a.Router.HandlerFunc(http.MethodGet, "/debug/pprof/trace", pprof.Trace)
+		a.Router.Handler(http.MethodGet, "/debug/pprof/goroutine", pprof.Handler("goroutine"))
+		a.Router.Handler(http.MethodGet, "/debug/pprof/heap", pprof.Handler("heap"))
+	}
+
 	rootMux.Handle("/", a.Router)
 	rootMux.Handle("/metrics", promhttp.Handler())
 	telemetryHandler := metrics.ResponseMetricMiddleware(rootMux)
 	handler := cors.AllowAll().Handler(telemetryHandler)
 
-	return http.ListenAndServe(":9003", errors.PanicHandlerMiddleware(handler))
+	return http.ListenAndServe(fmt.Sprint(":", env.GetAPIPort()), errors.PanicHandlerMiddleware(handler))
 }
 
 func StartExportWorker(ctx context.Context, model costmodel.AllocationModel) error {

pkg/costmodel/cluster_helpers.go

@@ -636,13 +636,11 @@ func buildLabelsMap(
 			Name:    node,
 		}
 
-		m[key] = make(map[string]string)
-
-		for name, value := range result.Metric {
-			if val, ok := value.(string); ok {
-				m[key][name] = val
-			}
-		}
+		// The QueryResult.GetLabels function needs to be called to sanitize the
+		// ingested label data. This removes the label_ prefix that prometheus
+		// adds to emitted labels. It also keeps from ingesting prometheus labels
+		// that aren't a part of the asset.
+		m[key] = result.GetLabels()
 	}
 	return m
 }

pkg/costmodel/cluster_helpers_test.go

@@ -2,6 +2,7 @@ package costmodel
 
 import (
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 
@@ -1108,3 +1109,74 @@ func TestAssetCustompricing(t *testing.T) {
 	}
 
 }
+
+func TestBuildLabelsMap(t *testing.T) {
+	const (
+		labelKey1   = "testlabelkey1"
+		labelValue1 = "testlabel1-value"
+		labelKey2   = "test-label-key-2"
+		labelValue2 = "testlabel2.value"
+		nonLabelKey = "instance_type"
+		labelPrefix = "label_"
+	)
+
+	startTimestamp := float64(windowStart.Unix())
+
+	nodePromResult := []*prom.QueryResult{
+		{
+			Metric: map[string]interface{}{
+				"cluster_id":             "cluster1",
+				"node":                   "node1",
+				"instance_type":          "type1",
+				"provider_id":            "provider1",
+				"label_testlabelkey1":    "testlabel1-value",
+				"label_test-label-key-2": "testlabel2.value",
+			},
+			Values: []*util.Vector{
+				{
+					Timestamp: startTimestamp,
+					Value:     0.5,
+				},
+			},
+		},
+		{
+			Metric: map[string]interface{}{
+				"cluster_id":             "cluster1",
+				"node":                   "node2",
+				"instance_type":          "type1",
+				"provider_id":            "provider1",
+				"label_testlabelkey1":    "testlabel1-value",
+				"label_test-label-key-2": "testlabel2.value",
+			},
+			Values: []*util.Vector{
+				{
+					Timestamp: startTimestamp,
+					Value:     0.5,
+				},
+			},
+		},
+	}
+
+	nodeLabelMap := buildLabelsMap(nodePromResult)
+	// Test that for all nodes and all label keys in the map there isn't a key with the label_ prefix.
+	for _, labelMap := range nodeLabelMap {
+		for key, value := range labelMap {
+			if strings.HasPrefix(key, labelPrefix) {
+				t.Errorf("Asset label maps aren't sanitized. Expected no '%v' prefix in %v", labelPrefix, key)
+			}
+			// Test that the label value isn't touched
+			if key == labelKey1 && value != labelValue1 {
+				t.Errorf("Label Value didn't match. Got %v, but Expected: %v", value, labelValue1)
+			}
+			// Test that the label value isn't touched
+			if key == labelKey2 && value != labelValue2 {
+				t.Errorf("Label Value didn't match. Got %v, but Expected: %v", value, labelValue2)
+			}
+		}
+		// Test that keys that don't have the label_ prefix aren't in the resultant label map.
+		_, ok := labelMap[nonLabelKey]
+		if ok {
+			t.Errorf("Non-label keys are included in label mapping for asset labels. Expected '%v' to not exist'.", nonLabelKey)
+		}
+	}
+}

pkg/costmodel/costmodel.go

@@ -31,6 +31,8 @@ const (
 
 	profileThreshold = 1000 * 1000 * 1000 // 1s (in ns)
 
+	unmountedPVsContainer = "unmounted-pvs"
+
 	apiPrefix         = "/api/v1"
 	epAlertManagers   = apiPrefix + "/alertmanagers"
 	epLabelValues     = apiPrefix + "/label/:name/values"
@@ -724,16 +726,13 @@ func findUnmountedPVCostData(clusterMap clusters.ClusterMap, unmountedPVs map[st
 
 		namespaceAnnotations, _ := namespaceAnnotationsMapping[ns+","+clusterID]
 
-		// Should be a unique "Unmounted" cost data type
-		name := "unmounted-pvs"
-
-		metric := NewContainerMetricFromValues(ns, name, name, "", clusterID)
+		metric := NewContainerMetricFromValues(ns, unmountedPVsContainer, unmountedPVsContainer, "", clusterID)
 		key := metric.Key()
 
 		if costData, ok := costs[key]; !ok {
 			costs[key] = &CostData{
-				Name:            name,
-				PodName:         name,
+				Name:            unmountedPVsContainer,
+				PodName:         unmountedPVsContainer,
 				NodeName:        "",
 				Annotations:     namespaceAnnotations,
 				Namespace:       ns,
@@ -1115,7 +1114,60 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 			gpuc = 0.0
 		}
 
-		if newCnode.GPU != "" && newCnode.GPUCost == "" {
+		// Special case for SUSE rancher, since it won't behave with normal
+		// calculations, courtesy of the instance type not being "real" (a
+		// recognizable AWS instance type.)
+		if newCnode.InstanceType == "rke2" {
+			log.Infof(
+				"Found a SUSE Rancher node %s, defaulting and skipping math",
+				cp.GetKey(nodeLabels, n).Features(),
+			)
+
+			defaultCPUCorePrice, err := strconv.ParseFloat(cfg.CPU, 64)
+			if err != nil {
+				log.Errorf("Could not parse default cpu price")
+				defaultCPUCorePrice = 0
+			}
+			if math.IsNaN(defaultCPUCorePrice) {
+				log.Warnf("defaultCPU parsed as NaN. Setting to 0.")
+				defaultCPUCorePrice = 0
+			}
+
+			defaultRAMPrice, err := strconv.ParseFloat(cfg.RAM, 64)
+			if err != nil {
+				log.Errorf("Could not parse default ram price")
+				defaultRAMPrice = 0
+			}
+			if math.IsNaN(defaultRAMPrice) {
+				log.Warnf("defaultRAM parsed as NaN. Setting to 0.")
+				defaultRAMPrice = 0
+			}
+
+			defaultGPUPrice, err := strconv.ParseFloat(cfg.GPU, 64)
+			if err != nil {
+				log.Errorf("Could not parse default gpu price")
+				defaultGPUPrice = 0
+			}
+			if math.IsNaN(defaultGPUPrice) {
+				log.Warnf("defaultGPU parsed as NaN. Setting to 0.")
+				defaultGPUPrice = 0
+			}
+			// Just say no to doing the ratios!
+			cpuCost := defaultCPUCorePrice * cpu
+			gpuCost := defaultGPUPrice * gpuc
+			ramCost := defaultRAMPrice * ram
+			nodeCost := cpuCost + gpuCost + ramCost
+
+			newCnode.Cost = fmt.Sprintf("%f", nodeCost)
+			newCnode.VCPUCost = fmt.Sprintf("%f", cpuCost)
+			newCnode.GPUCost = fmt.Sprintf("%f", gpuCost)
+			newCnode.RAMCost = fmt.Sprintf("%f", ramCost)
+			newCnode.RAMBytes = fmt.Sprintf("%f", ram)
+
+		} else if newCnode.GPU != "" && newCnode.GPUCost == "" {
+			// was the big thing to investigate. All the funky ratio math
+			// we were doing was messing with their default pricing. for SUSE Rancher.
+
 			// We couldn't find a gpu cost, so fix cpu and ram, then accordingly
 			log.Infof("GPU without cost found for %s, calculating...", cp.GetKey(nodeLabels, n).Features())
 
@@ -2502,9 +2554,8 @@ func (cm *CostModel) QueryAllocation(window kubecost.Window, resolution, step ti
 					}
 
 					if totals == nil {
-						log.Errorf("unable to locate asset totals for allocation %s", key)
-						return nil, fmt.Errorf("unable to locate allocation totals for allocation")
-
+						log.Errorf("unable to locate asset totals for allocation %s, corresponding PARC is being skipped", key)
+						continue
 					}
 
 					parc.CPUTotalCost = totals.CPUCost

pkg/costmodel/metrics.go

@@ -421,7 +421,7 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 			var ok bool
 			defaultRegion, ok = util.GetRegion(nodeList[0].Labels)
 			if !ok {
-				log.DedupedWarningf(5, "Failed to locate default region")
+				log.DedupedWarningf(5, "Failed to read default region from labels on node %s", nodeList[0].Name)
 			}
 		}
 
@@ -471,7 +471,7 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 			// TODO: Pass CloudProvider into CostModel on instantiation so this isn't so awkward
 			nodes, err := cmme.Model.GetNodeCost(cmme.CloudProvider)
 			if err != nil {
-				log.Warnf("Metric emission: error getting Node cost: %s", err)
+				log.Warnf("Error getting Node cost: %s", err)
 			}
 			for nodeName, node := range nodes {
 				// Emit costs, guarding against NaN inputs for custom pricing.
@@ -535,24 +535,27 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 				// don't record cpuCost, ramCost, or gpuCost in the case of wild outliers
 				// k8s api sometimes causes cost spikes as described here:
 				// https://github.com/opencost/opencost/issues/927
-				if cpuCost < outlierFactor*avgCosts.CpuCostAverage {
+				cpuOutlierCutoff := outlierFactor * avgCosts.CpuCostAverage
+				if cpuCost < cpuOutlierCutoff {
 					cmme.CPUPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID, node.ArchType).Set(cpuCost)
 					avgCosts.CpuCostAverage = (avgCosts.CpuCostAverage*avgCosts.NumCpuDataPoints + cpuCost) / (avgCosts.NumCpuDataPoints + 1)
 					avgCosts.NumCpuDataPoints += 1
 				} else {
-					log.Warnf("CPU cost outlier detected; skipping data point.")
+					log.Debugf("CPU cost outlier detected; skipping data point: %s had %f as cost, which is above %f.", nodeName, cpuCost, cpuOutlierCutoff)
 				}
-				if ramCost < outlierFactor*avgCosts.RamCostAverage {
+				ramOutlierCutoff := outlierFactor * avgCosts.RamCostAverage
+				if ramCost < ramOutlierCutoff {
 					cmme.RAMPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID, node.ArchType).Set(ramCost)
 					avgCosts.RamCostAverage = (avgCosts.RamCostAverage*avgCosts.NumRamDataPoints + ramCost) / (avgCosts.NumRamDataPoints + 1)
 					avgCosts.NumRamDataPoints += 1
 				} else {
-					log.Warnf("RAM cost outlier detected; skipping data point.")
+					log.Debugf("RAM cost outlier detected; skipping data point: %s had %f as cost, which is above %f.", nodeName, ramCost, ramOutlierCutoff)
 				}
 				// skip redording totalCost if any constituent costs were outliers
-				if cpuCost < outlierFactor*avgCosts.CpuCostAverage &&
-					ramCost < outlierFactor*avgCosts.RamCostAverage {
+				if cpuCost < cpuOutlierCutoff && ramCost < ramOutlierCutoff {
 					cmme.NodeTotalPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID, node.ArchType).Set(totalCost)
+				} else {
+					log.Debugf("CPU and RAM outlier detected, not recording node %s total cost %f", nodeName, totalCost)
 				}
 
 				nodeCostAverages[labelKey] = avgCosts
@@ -568,7 +571,7 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 			// TODO: Pass CloudProvider into CostModel on instantiation so this isn't so awkward
 			loadBalancers, err := cmme.Model.GetLBCost(cmme.CloudProvider)
 			if err != nil {
-				log.Warnf("Metric emission: error getting LoadBalancer cost: %s", err)
+				log.Warnf("Error getting LoadBalancer cost: %s", err)
 			}
 			for lbKey, lb := range loadBalancers {
 				// TODO: parse (if necessary) and calculate cost associated with loadBalancer based on dynamic cloud prices fetched into each lb struct on GetLBCost() call
@@ -644,7 +647,7 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 
 				parameters, ok := storageClassMap[pv.Spec.StorageClassName]
 				if !ok {
-					log.Debugf("Unable to find parameters for storage class \"%s\". Does pv \"%s\" have a storageClassName?", pv.Spec.StorageClassName, pv.Name)
+					log.Debugf("Unable to find parameters for storage class \"%s\". Pv \"%s\" might have an empty or invalid storageClassName.", pv.Spec.StorageClassName, pv.Name)
 				}
 				var region string
 				if r, ok := util.GetRegion(pv.Labels); ok {
@@ -662,7 +665,7 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 				GetPVCost(cacPv, pv, cmme.CloudProvider, region)
 				c, _ := strconv.ParseFloat(cacPv.Cost, 64)
 				cmme.PersistentVolumePriceRecorder.WithLabelValues(pv.Name, pv.Name, cacPv.ProviderID).Set(c)
-				labelKey := getKeyFromLabelStrings(pv.Name, pv.Name)
+				labelKey := getKeyFromLabelStrings(pv.Name, pv.Name, cacPv.ProviderID)
 				pvSeen[labelKey] = true
 			}
 
@@ -670,44 +673,44 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 			// longer exist
 			for labelString, seen := range nodeSeen {
 				if !seen {
-					log.Debugf("Removing %s from nodes", labelString)
+					log.Debugf("Removing metrics for %s, no data observed recently", labelString)
 					labels := getLabelStringsFromKey(labelString)
 
 					ok := cmme.NodeTotalPriceRecorder.DeleteLabelValues(labels...)
 					if ok {
-						log.Debugf("removed %s from totalprice", labelString)
+						log.Debugf("No data observed for node with labels %v, removed from totalprice", labels)
 					} else {
-						log.Errorf("FAILURE TO REMOVE %s from totalprice", labelString)
+						log.Warnf("Failed to remove label set %v from metric node_total_hourly_cost. Failure to remove stale metrics may result in inaccurate data.", labels)
 					}
 					ok = cmme.NodeSpotRecorder.DeleteLabelValues(labels...)
 					if ok {
-						log.Debugf("removed %s from spot records", labelString)
+						log.Debugf("No data observed for node with labels %v, removed from spot records", labels)
 					} else {
-						log.Errorf("FAILURE TO REMOVE %s from spot records", labelString)
+						log.Warnf("Failed to remove label set %v from metric kubecost_node_is_spot. Failure to remove stale metrics may result in inaccurate data.", labels)
 					}
 					ok = cmme.CPUPriceRecorder.DeleteLabelValues(labels...)
 					if ok {
-						log.Debugf("removed %s from cpuprice", labelString)
+						log.Debugf("No data observed for node with labels %v, removed from cpuprice", labels)
 					} else {
-						log.Errorf("FAILURE TO REMOVE %s from cpuprice", labelString)
+						log.Warnf("Failed to remove label set %v from metric node_cpu_hourly_cost. Failure to remove stale metrics may result in inaccurate data.", labels)
 					}
 					ok = cmme.GPUPriceRecorder.DeleteLabelValues(labels...)
 					if ok {
-						log.Debugf("removed %s from gpuprice", labelString)
+						log.Debugf("No data observed for node with labels %v, removed from gpuprice", labels)
 					} else {
-						log.Errorf("FAILURE TO REMOVE %s from gpuprice", labelString)
+						log.Warnf("Failed to remove label set %v from metric node_gpu_hourly_cost. Failure to remove stale metrics may result in inaccurate data.", labels)
 					}
 					ok = cmme.GPUCountRecorder.DeleteLabelValues(labels...)
 					if ok {
-						log.Debugf("removed %s from gpucount", labelString)
+						log.Debugf("No data observed for node with labels %v, removed from gpucount", labels)
 					} else {
-						log.Errorf("FAILURE TO REMOVE %s from gpucount", labelString)
+						log.Warnf("Failed to remove label set %v from metric node_gpu_count. Failure to remove stale metrics may result in inaccurate data.", labels)
 					}
 					ok = cmme.RAMPriceRecorder.DeleteLabelValues(labels...)
 					if ok {
-						log.Debugf("removed %s from ramprice", labelString)
+						log.Debugf("No data observed for node with labels %v, removed from ramprice", labels)
 					} else {
-						log.Errorf("FAILURE TO REMOVE %s from ramprice", labelString)
+						log.Warnf("Failed to remove label set %v from metric node_ram_hourly_cost. Failure to remove stale metrics may result in inaccurate data.", labels)
 					}
 					delete(nodeSeen, labelString)
 					delete(nodeCostAverages, labelString)
@@ -720,7 +723,7 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 					labels := getLabelStringsFromKey(labelString)
 					ok := cmme.LBCostRecorder.DeleteLabelValues(labels...)
 					if !ok {
-						log.Errorf("Metric emission: failed to delete LoadBalancer with labels: %v", labels)
+						log.Warnf("Failed to remove label set %v from metric kubecost_load_balancer_cost. Failure to remove stale metrics may result in inaccurate data.", labels)
 					}
 					delete(loadBalancerSeen, labelString)
 				} else {
@@ -730,17 +733,21 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 			for labelString, seen := range containerSeen {
 				if !seen {
 					labels := getLabelStringsFromKey(labelString)
-					ok := cmme.RAMAllocationRecorder.DeleteLabelValues(labels...)
-					if !ok {
-						log.Errorf("Metric emission: failed to delete RAMAllocation with labels: %v", labels)
-					}
-					ok = cmme.CPUAllocationRecorder.DeleteLabelValues(labels...)
-					if !ok {
-						log.Errorf("Metric emission: failed to delete CPUAllocation with labels: %v", labels)
-					}
-					ok = cmme.GPUAllocationRecorder.DeleteLabelValues(labels...)
-					if !ok {
-						log.Errorf("Metric emission: failed to delete GPUAllocation with labels: %v", labels)
+					if len(labels) >= 2 && labels[1] != unmountedPVsContainer { // special "pod" to contain the unmounted PVs - does not have RAM/CPU/...
+						ok := cmme.RAMAllocationRecorder.DeleteLabelValues(labels...)
+						if !ok {
+							log.Warnf("Failed to remove label set %v from metric container_memory_allocation_bytes. Failure to remove stale metrics may result in inaccurate data.", labels)
+						}
+						ok = cmme.CPUAllocationRecorder.DeleteLabelValues(labels...)
+						if !ok {
+							log.Warnf("Failed to remove label set %v from metric container_cpu_allocation. Failure to remove stale metrics may result in inaccurate data.", labels)
+						}
+						ok = cmme.GPUAllocationRecorder.DeleteLabelValues(labels...)
+						if !ok {
+							log.Warnf("Failed to remove label set %v from metric container_gpu_allocation. Failure to remove stale metrics may result in inaccurate data.", labels)
+						}
+					} else {
+						log.Debugf("Did not try to delete RAM/CPU/GPU for fake '%s' container: %v", unmountedPVsContainer, labels)
 					}
 					delete(containerSeen, labelString)
 				} else {
@@ -752,7 +759,7 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 					labels := getLabelStringsFromKey(labelString)
 					ok := cmme.PersistentVolumePriceRecorder.DeleteLabelValues(labels...)
 					if !ok {
-						log.Errorf("Metric emission: failed to delete PVPrice with labels: %v", labels)
+						log.Warnf("Failed to remove label set %v from metric pv_hourly_cost. Failure to remove stale metrics may result in inaccurate data.", labels)
 					}
 					delete(pvSeen, labelString)
 				} else {
@@ -764,7 +771,7 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 					labels := getLabelStringsFromKey(labelString)
 					ok := cmme.PVAllocationRecorder.DeleteLabelValues(labels...)
 					if !ok {
-						log.Errorf("Metric emission: failed to delete PVAllocation with labels: %v", labels)
+						log.Warnf("Failed to remove label set %v from metric pod_pvc_allocation. Failure to remove stale metrics may result in inaccurate data.", labels)
 					}
 					delete(pvcSeen, labelString)
 				} else {

pkg/env/costmodelenv.go

@@ -11,6 +11,8 @@ import (
 )
 
 const (
+	APIPortEnvVar = "API_PORT"
+
 	AWSAccessKeyIDEnvVar     = "AWS_ACCESS_KEY_ID"
 	AWSAccessKeySecretEnvVar = "AWS_SECRET_ACCESS_KEY"
 	AWSClusterIDEnvVar       = "AWS_CLUSTER_ID"
@@ -51,6 +53,8 @@ const (
 	ThanosOffsetEnvVar       = "THANOS_QUERY_OFFSET"
 	ThanosMaxSourceResEnvVar = "THANOS_MAX_SOURCE_RESOLUTION"
 
+	PProfEnabledEnvVar = "PPROF_ENABLED"
+
 	LogCollectionEnabledEnvVar    = "LOG_COLLECTION_ENABLED"
 	ProductAnalyticsEnabledEnvVar = "PRODUCT_ANALYTICS_ENABLED"
 	ErrorReportingEnabledEnvVar   = "ERROR_REPORTING_ENABLED"
@@ -137,10 +141,20 @@ func GetExportCSVLabelsList() []string {
 	return GetList(ExportCSVLabelsList, ",")
 }
 
+func IsPProfEnabled() bool {
+	return GetBool(PProfEnabledEnvVar, false)
+}
+
 func GetExportCSVMaxDays() int {
 	return GetInt(ExportCSVMaxDays, 90)
 }
 
+// GetAPIPort returns the environment variable value for APIPortEnvVar which
+// is the port number the API is available on.
+func GetAPIPort() int {
+	return GetInt(APIPortEnvVar, 9003)
+}
+
 // GetKubecostConfigBucket returns a file location for a mounted bucket configuration which is used to store
 // a subset of kubecost configurations that require sharing via remote storage.
 func GetKubecostConfigBucket() string {

pkg/env/costmodelenv_test.go

@@ -5,6 +5,44 @@ import (
 	"testing"
 )
 
+func TestGetAPIPort(t *testing.T) {
+	tests := []struct {
+		name string
+		want int
+		pre  func()
+	}{
+		{
+			name: "Ensure the default API port '9003'",
+			want: 9003,
+		},
+		{
+			name: "Ensure the default API port '9003' when API_PORT is set to ''",
+			want: 9003,
+			pre: func() {
+				os.Setenv("API_PORT", "")
+			},
+		},
+		{
+			name: "Ensure the API port '9004' when API_PORT is set to '9004'",
+			want: 9004,
+			pre: func() {
+				os.Setenv("API_PORT", "9004")
+			},
+		},
+	}
+	for _, tt := range tests {
+		if tt.pre != nil {
+			tt.pre()
+		}
+		t.Run(tt.name, func(t *testing.T) {
+			if got := GetAPIPort(); got != tt.want {
+				t.Errorf("GetAPIPort() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+
+}
+
 func TestIsCacheDisabled(t *testing.T) {
 	tests := []struct {
 		name string

pkg/kubecost/asset.go

@@ -4,7 +4,6 @@ import (
 	"encoding"
 	"fmt"
 	"math"
-	"regexp"
 	"strings"
 	"time"
 
@@ -12,6 +11,7 @@ import (
 	"github.com/opencost/opencost/pkg/filter21/ast"
 	"github.com/opencost/opencost/pkg/filter21/matcher"
 	"github.com/opencost/opencost/pkg/log"
+	"github.com/opencost/opencost/pkg/prom"
 	"github.com/opencost/opencost/pkg/util/json"
 	"github.com/opencost/opencost/pkg/util/timeutil"
 )
@@ -3841,6 +3841,7 @@ func (asr *AssetSetRange) InsertRange(that *AssetSetRange) error {
 	}
 
 	var err error
+	var as *AssetSet
 	for _, thatAS := range that.Assets {
 		if thatAS == nil || err != nil {
 			continue
@@ -3852,7 +3853,7 @@ func (asr *AssetSetRange) InsertRange(that *AssetSetRange) error {
 			err = fmt.Errorf("cannot merge AssetSet into window that does not exist: %s", thatAS.Window.String())
 			continue
 		}
-		as, err := asr.Get(i)
+		as, err = asr.Get(i)
 		if err != nil {
 			err = fmt.Errorf("AssetSetRange index does not exist: %d", i)
 			continue
@@ -4138,8 +4139,8 @@ func GetNodePoolName(provider string, labels map[string]string) string {
 }
 
 func getPoolNameHelper(label string, labels map[string]string) string {
-	sanitizedLabel := regexp.MustCompile(`[^a-zA-Z0-9 ]+`).ReplaceAllString(label, "_")
-	if poolName, found := labels[fmt.Sprintf("label_%s", sanitizedLabel)]; found {
+	sanitizedLabel := prom.SanitizeLabelName(label)
+	if poolName, found := labels[sanitizedLabel]; found {
 		return poolName
 	} else {
 		log.Warnf("unable to derive node pool name from node labels")

tilt-values.yaml

@@ -0,0 +1,118 @@
+# DO NOT USE FOR DEPLOYMENT. This file is intended to be used with a Tiltfile
+# and for development purposes only. Please refer to
+# https://github.com/opencost/opencost-helm-chart
+service:
+  enabled: true
+  # --  Kubernetes Service type
+  type: ClusterIP
+  # -- extra ports.  Useful for sidecar pods such as oauth-proxy
+  extraPorts:
+    - name: debug
+      port: 40000
+      targetPort: 40000
+
+opencost:
+  exporter:
+    # -- The GCP Pricing API requires a key. This is supplied just for evaluation.
+    cloudProviderApiKey: ""
+    # -- Default cluster ID to use if cluster_id is not set in Prometheus metrics.
+    defaultClusterId: 'tilt-cluster'
+  livenessProbe:
+    # -- Whether probe is enabled
+    enabled: true
+    # -- Number of seconds before probe is initiated
+    initialDelaySeconds: 120
+    # -- Probe frequency in seconds
+    periodSeconds: 10
+    # -- Number of failures for probe to be considered failed
+    failureThreshold: 3
+  # Readiness probe configuration
+  readinessProbe:
+    # -- Whether probe is enabled
+    enabled: true
+    # -- Number of seconds before probe is initiated
+    initialDelaySeconds: 120
+    # -- Probe frequency in seconds
+    periodSeconds: 10
+    # -- Number of failures for probe to be considered failed
+    failureThreshold: 3
+
+  # Persistent volume claim for storing the data. eg: csv file
+  persistence:
+    enabled: false
+
+  aws:
+    # -- AWS secret access key
+    secret_access_key: ""
+    # -- AWS secret key id
+    access_key_id: ""
+
+  customPricing:
+    # -- Enables custom pricing configuration
+    enabled: false
+    # -- Customize the configmap name used for custom pricing
+    configmapName: custom-pricing-model
+    # -- Path for the pricing configuration.
+    configPath: /tmp/custom-config
+    # -- Configures the pricing model provided in the values file.
+    createConfigmap: true
+    # -- Sets the provider type for the custom pricing file.
+    provider: custom
+    # -- More information about these values here: https://www.opencost.io/docs/configuration/on-prem#custom-pricing-using-the-opencost-helm-chart
+    costModel:
+      description: Modified pricing configuration.
+      CPU: 1.25
+      spotCPU: 0.006655
+      RAM: 0.50
+      spotRAM: 0.000892
+      GPU: 0.95
+      storage: 0.25
+      zoneNetworkEgress: 0.01
+      regionNetworkEgress: 0.01
+      internetNetworkEgress: 0.12
+
+  dataRetention:
+    dailyResolutionDays: 15
+
+  cloudCost:
+    # -- Enable cloud cost ingestion and querying, dependant on valid integration credentials
+    enabled: false
+    # -- Number of hours between each run of the Cloud Cost pipeline
+    refreshRateHours: 6
+    # -- Number of days into the past that a Cloud Cost standard run will query for
+    runWindowDays: 3
+    # -- The number of standard runs before a Month-to-Date run occurs
+    monthToDateInterval: 6
+    # -- The max number of days that any single query will be made to construct Cloud Costs
+    queryWindowDays: 7
+
+  metrics:
+    serviceMonitor:
+      # -- Create ServiceMonitor resource for scraping metrics using PrometheusOperator
+      enabled: false
+      # -- Additional labels to add to the ServiceMonitor
+      additionalLabels: {}
+      # -- Specify if the ServiceMonitor will be deployed into a different namespace (blank deploys into same namespace as chart)
+      namespace: ""
+      # -- Interval at which metrics should be scraped
+      scrapeInterval: 30s
+      # -- Timeout after which the scrape is ended
+      scrapeTimeout: 10s
+      # -- HonorLabels chooses the metric's labels on collisions with target labels
+      honorLabels: true
+      # -- RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds relabelings for a few standard Kubernetes fields
+      relabelings: []
+      # -- MetricRelabelConfigs to apply to samples before ingestion
+      metricRelabelings: []
+      # -- HTTP scheme used for scraping. Defaults to `http`
+      scheme: http
+  prometheus:
+    internal:
+      enabled: true
+      # -- Service name of in-cluster Prometheus
+      serviceName: prometheus-server
+      # -- Service port of in-cluster Prometheus
+      port: 80
+  ui:
+    # -- Enable OpenCost UI
+    enabled: true

ui/Dockerfile

@@ -7,8 +7,17 @@ RUN npx parcel build src/index.html
 
 FROM nginx:alpine
 
+ARG version=dev
+ARG	commit=HEAD
+ENV VERSION=${version}
+ENV HEAD=${commit}
+
+ENV API_PORT=9003
+ENV API_SERVER=0.0.0.0
+ENV UI_PORT=9090
+
 COPY --from=builder /opt/ui/dist /var/www
-COPY default.nginx.conf /etc/nginx/conf.d/
+COPY default.nginx.conf.template /etc/nginx/conf.d/default.nginx.conf.template
 COPY nginx.conf /etc/nginx/
 COPY ./docker-entrypoint.sh /usr/local/bin/
 

ui/Dockerfile.cross

@@ -1,7 +1,16 @@
 FROM nginx:alpine
 
+ARG version=dev
+ARG	commit=HEAD
+ENV VERSION=${version}
+ENV HEAD=${commit}
+
+ENV API_PORT=9003
+ENV API_SERVER=0.0.0.0
+ENV UI_PORT=9090
+
 COPY ./dist /var/www
-COPY default.nginx.conf /etc/nginx/conf.d/
+COPY default.nginx.conf.template /etc/nginx/conf.d/default.nginx.conf.template
 COPY nginx.conf /etc/nginx/
 COPY ./docker-entrypoint.sh /usr/local/bin/
 

ui/default.nginx.conf → ui/default.nginx.conf.template

@@ -35,7 +35,7 @@ gzip_types
 upstream model {
     # Update to the cost model endpoint
     # Example: host.docker.internal:9003;
-    server 0.0.0.0:9003;
+    server ${API_SERVER}:${API_PORT};
 }
 
 server {
@@ -58,8 +58,8 @@ server {
     }
 
     add_header ETag "1.96.0";
-    listen 9090;
-    listen [::]:9090;
+    listen ${UI_PORT};
+    listen [::]:${UI_PORT};
     resolver 127.0.0.1 valid=5s;
     location /healthz {
         access_log /dev/null;

ui/docker-entrypoint.sh

@@ -4,10 +4,14 @@ set -e
 if [[ ! -z "$BASE_URL_OVERRIDE" ]]; then
     echo "running with BASE_URL=${BASE_URL_OVERRIDE}"
     sed -i "s^{PLACEHOLDER_BASE_URL}^$BASE_URL_OVERRIDE^g" /var/www/*.js
-else 
+else
     echo "running with BASE_URL=${BASE_URL}"
     sed -i "s^{PLACEHOLDER_BASE_URL}^$BASE_URL^g" /var/www/*.js
 fi
 
+envsubst '$API_PORT $API_SERVER $UI_PORT' < /etc/nginx/conf.d/default.nginx.conf.template > /etc/nginx/conf.d/default.nginx.conf
+
+echo "Starting ui version $VERSION ($HEAD)"
+
 # Run the parent (nginx) container's entrypoint script
-exec /docker-entrypoint.sh "$@"
+exec /docker-entrypoint.sh "$@"

ui/justfile

@@ -1,3 +1,6 @@
+version := `../tools/image-tag`
+commit := `git rev-parse --short HEAD`
+
 default:
     just --list
 
@@ -13,6 +16,8 @@ build IMAGETAG: build-local
         -f 'Dockerfile.cross' \
         --provenance=false \
         -t {{IMAGETAG}}-amd64 \
+        --build-arg version={{version}} \
+        --build-arg commit={{commit}} \
         --push \
         .
 
@@ -22,6 +27,8 @@ build IMAGETAG: build-local
         -f 'Dockerfile.cross' \
         --provenance=false \
         -t {{IMAGETAG}}-arm64 \
+        --build-arg version={{version}} \
+        --build-arg commit={{commit}} \
         --push \
         .
 

ui/package.json

@@ -21,7 +21,7 @@
     "@material-ui/icons": "^4.11.2",
     "@material-ui/pickers": "^3.3.10",
     "@material-ui/styles": "^4.11.5",
-    "axios": "^1.4.0",
+    "axios": "^1.6.0",
     "date-fns": "^2.30.0",
     "material-design-icons-iconfont": "^6.1.0",
     "prop-types": "^15.7.2",

ui/src/cloudCost/cloudCostDetails.js

@@ -33,7 +33,7 @@ const CloudCostDetails = ({
 
   const nextFilters = [
     ...(filters ?? []),
-    { property: "providerIds", value: selectedProviderId },
+    { property: "providerID", value: selectedProviderId },
   ];
 
   async function fetchData() {
@@ -122,7 +122,7 @@ const CloudCostDetails = ({
         title={`Costs over the last ${window}`}
         style={{ margin: "10%" }}
       >
-        <Paper>
+        <Paper style={{ padding: 20 }}>
           <Typography style={{ marginTop: "1rem" }} variant="body1">
             {selectedItem}
           </Typography>

ui/src/cloudCostReports.js

@@ -4,7 +4,7 @@ import Header from "./components/Header";
 import IconButton from "@material-ui/core/IconButton";
 import RefreshIcon from "@material-ui/icons/Refresh";
 import { makeStyles } from "@material-ui/styles";
-import { Paper, Typography } from "@material-ui/core";
+import { Box, Link, Paper, Typography } from "@material-ui/core";
 import CircularProgress from "@material-ui/core/CircularProgress";
 import { get, find } from "lodash";
 import { useLocation, useHistory } from "react-router";
@@ -135,12 +135,12 @@ const CloudCostReports = () => {
           {
             primary: "Failed to load report data",
             secondary:
-              "Please update Kubecost to the latest version, then contact support if problems persist.",
+            "Please update OpenCost to the latest version, and open an Issue if problems persist.",
           },
         ]);
       } else {
         let secondary =
-          "Please contact Kubecost support with a bug report if problems persist.";
+          "Please open an Issue with OpenCost if problems persist.";
         if (err.message.length > 0) {
           secondary = err.message;
         }
@@ -175,7 +175,6 @@ const CloudCostReports = () => {
       return {
         property,
         value,
-        name: aggMap[property] || property,
       };
     });
     setFilters(newFilters);
@@ -204,6 +203,30 @@ const CloudCostReports = () => {
     setTitle(generateTitle({ window, aggregateBy, costMetric }));
   }, [window, aggregateBy, costMetric, filters]);
 
+  const hasCloudCostEnabled = aggregateBy.includes("item")
+    ? true // this is kind of hacky but something weird is happening
+    : // when drilling down will address in a later PR - @jjarrett21
+      !!cloudCostData.cloudCostStatus?.length;
+
+  const enabledWarnings = [
+    {
+      primary: "There are no Cloud Cost integrations currently configured.",
+      secondary: (
+        <>
+          Learn more about setting up Cloud Costs{" "}
+          <Link
+            href={
+              "https://www.opencost.io/docs/configuration/#cloud-costs"
+            }
+            target="_blank"
+          >
+            here
+          </Link>
+        </>
+      ),
+    },
+  ];
+
   return (
     <Page active="cloud.html">
       <Header>
@@ -212,13 +235,19 @@ const CloudCostReports = () => {
         </IconButton>
       </Header>
 
-      {!loading && errors.length > 0 && (
+      {!loading && !hasCloudCostEnabled && (
+        <div style={{ marginBottom: 20 }}>
+          <Warnings warnings={enabledWarnings} />
+        </div>
+      )}
+
+      {!loading && errors.length > 0 && hasCloudCostEnabled && (
         <div style={{ marginBottom: 20 }}>
           <Warnings warnings={errors} />
         </div>
       )}
 
-      {init && (
+      {init && hasCloudCostEnabled && (
         <Paper id="cloud-cost">
           <div className={classes.reportHeader}>
             <div className={classes.titles}>
@@ -237,6 +266,7 @@ const CloudCostReports = () => {
               aggregationOptions={aggregationOptions}
               aggregateBy={aggregateBy}
               setAggregateBy={(agg) => {
+                setFilters([])
                 searchParams.set("agg", agg);
                 routerHistory.push({
                   search: `?${searchParams.toString()}`,

ui/src/components/Warnings.js

@@ -1,21 +1,21 @@
-import React from 'react'
-import { makeStyles } from '@material-ui/styles'
-import List from '@material-ui/core/List'
-import ListItem from '@material-ui/core/ListItem'
-import ListItemIcon from '@material-ui/core/ListItemIcon'
-import ListItemText from '@material-ui/core/ListItemText'
-import Paper from '@material-ui/core/Paper'
-import WarningIcon from '@material-ui/icons/Warning'
+import React from "react";
+import { makeStyles } from "@material-ui/styles";
+import List from "@material-ui/core/List";
+import ListItem from "@material-ui/core/ListItem";
+import ListItemIcon from "@material-ui/core/ListItemIcon";
+import ListItemText from "@material-ui/core/ListItemText";
+import Paper from "@material-ui/core/Paper";
+import WarningIcon from "@material-ui/icons/Warning";
 
 const useStyles = makeStyles({
   root: {},
-})
+});
 
-const Warnings = ({warnings}) => {
-  const classes = useStyles()
+const Warnings = ({ warnings }) => {
+  const classes = useStyles();
 
   if (!warnings || warnings.length === 0) {
-    return null
+    return null;
   }
 
   return (
@@ -31,7 +31,7 @@ const Warnings = ({warnings}) => {
         ))}
       </List>
     </Paper>
-  )
-}
+  );
+};
 
-export default Warnings
+export default Warnings;

ui/src/services/cloudCostDayTotals.js

@@ -1,5 +1,5 @@
 import axios from "axios";
-import { getCloudFilters } from "../util";
+import { parseFilters } from "../util";
 import { costMetricToPropName } from "../cloudCost/tokens";
 
 function formatItemsForCost({ data, costType }) {
@@ -21,12 +21,11 @@ class CloudCostDayTotalsService {
     if (this.BASE_URL.includes("PLACEHOLDER_BASE_URL")) {
       this.BASE_URL = `http://localhost:9090/model`;
     }
-
     if (aggregate.includes("item")) {
       const resp = await axios.get(
         `${
           this.BASE_URL
-        }/cloudCost?window=${window}&costMetric=${costMetric}${getCloudFilters(
+        }/cloudCost?window=${window}&costMetric=${costMetric}&filter=${parseFilters(
           filters
         )}`
       );

ui/src/services/cloudCostTop.js

@@ -1,5 +1,5 @@
 import axios from "axios";
-import { getCloudFilters, formatSampleItemsForGraph } from "../util";
+import { formatSampleItemsForGraph, parseFilters } from "../util";
 
 class CloudCostTopService {
   BASE_URL = process.env.BASE_URL || "{PLACEHOLDER_BASE_URL}";
@@ -13,7 +13,7 @@ class CloudCostTopService {
       window,
       aggregate,
       costMetric,
-      filters,
+      filter: parseFilters(filters ?? []),
       limit: 1000,
     };
 
@@ -21,7 +21,7 @@ class CloudCostTopService {
       const resp = await axios.get(
         `${
           this.BASE_URL
-        }/cloudCost?window=${window}&costMetric=${costMetric}${getCloudFilters(
+        }/cloudCost?window=${window}&costMetric=${costMetric}&filter=${parseFilters(
           filters
         )}`
       );
@@ -43,10 +43,13 @@ class CloudCostTopService {
       params,
     });
 
+    const status = await axios.get(`${this.BASE_URL}/cloudCost/status`);
+
     return {
       tableRows: tableView.data.data,
       graphData: graphView.data.data,
       tableTotal: totalsView.data.data.combined,
+      cloudCostStatus: status.data.data,
     };
   }
 }

ui/src/util.js

@@ -344,35 +344,6 @@ export function checkCustomWindow(window) {
   return customDateRegex.test(window);
 }
 
-export function getCloudFilters(filters) {
-  const filterNamesMap = {
-    "invoice entity": "filterInvoiceEntityIDs",
-    provider: "filterProviders",
-    providerids: "filterProviderIDs",
-    service: "filterServices",
-    account: "filterAccountIDs",
-  };
-  const params = new URLSearchParams();
-  const labelFilters = [];
-
-  for (let filter of filters) {
-    const mapped = filterNamesMap[filter.property.toLowerCase()];
-
-    if (mapped) {
-      params.set(mapped, filter.value);
-    } else if (filter.property === "Labels") {
-      labelFilters.push(filter.value);
-    } else if (filter.property.startsWith(":")) {
-      labelFilters.push(`${filter.property.slice(6)}:${filter.value}`);
-    }
-  }
-  if (labelFilters.length) {
-    params.set("filterLabels", labelFilters.join(","));
-  }
-
-  return `&${params.toString()}`;
-}
-
 export function formatSampleItemsForGraph({ data, costMetric }) {
   const costMetricPropName = costMetric
     ? costMetricToPropName[costMetric]
@@ -412,29 +383,31 @@ export function formatSampleItemsForGraph({ data, costMetric }) {
         cloudCostItem[costMetricPropName].kubernetesPercent;
     });
   });
-  const tableRows = Object.entries(accumulator).map(
-    ([
-      name,
-      {
+  const tableRows = Object.entries(accumulator)
+    .map(
+      ([
+        name,
+        {
+          cost,
+          start,
+          end,
+          providerID,
+          kubernetesCost,
+          kubernetesPercent,
+          labelName,
+        },
+      ]) => ({
         cost,
+        name,
+        kubernetesCost,
+        kubernetesPercent,
         start,
         end,
         providerID,
-        kubernetesCost,
-        kubernetesPercent,
         labelName,
-      },
-    ]) => ({
-      cost,
-      name,
-      kubernetesCost,
-      kubernetesPercent,
-      start,
-      end,
-      providerID,
-      labelName,
-    })
-  );
+      })
+    )
+    .sort((a, b) => (a.cost > b.cost ? -1 : 1));
 
   const tableTotal = tableRows.reduce(
     (tr1, tr2) => ({
@@ -457,6 +430,18 @@ export function formatSampleItemsForGraph({ data, costMetric }) {
   return { graphData, tableRows, tableTotal };
 }
 
+export function parseFilters(filters) {
+  if (typeof filters === "string") {
+    return filters;
+  }
+  // remove dups (via context ) and format
+  return (
+    [...new Set(filters.map((f) => `${f.property}:"${f.value}"`))].join(
+      encodeURIComponent("+")
+    ) || ""
+  );
+}
+
 export default {
   rangeToCumulative,
   cumulativeToTotals,