Merge branch 'develop' into mattray-patch-1

Signed-off-by: Cliff Colvin <ccolvin@kubecost.com>

.github/workflows/build-and-publish-release.yml

@@ -57,12 +57,11 @@ jobs:
         id: branch
         run: |
           VERSION_NUMBER=${{ steps.version_number.outputs.RELEASE_VERSION }}
-          echo "BRANCH_NAME=v${VERSION_NUMBER%.*}" >> $GITHUB_ENV
+          echo "BRANCH_NAME=v${VERSION_NUMBER%.*}" >> $GITHUB_OUTPUT
 
       - name: Checkout Repo
         uses: actions/checkout@v4
         with:
-          repository: 'opencost/opencost'
           ref: '${{ steps.branch.outputs.BRANCH_NAME }}'
           path: ./opencost
 
@@ -85,9 +84,9 @@ jobs:
       - name: Set OpenCost Image Tags
         id: tags
         run: |
-          echo "IMAGE_TAG=ghcr.io/opencost/opencost:${{ steps.sha.outputs.OC_SHORTHASH }}" >> $GITHUB_OUTPUT
-          echo "IMAGE_TAG_LATEST=ghcr.io/opencost/opencost:latest" >> $GITHUB_OUTPUT
-          echo "IMAGE_TAG_VERSION=ghcr.io/opencost/opencost:${{ steps.version_number.outputs.RELEASE_VERSION }}" >> $GITHUB_OUTPUT
+          echo "IMAGE_TAG=ghcr.io/${{ github.repository_owner }}/opencost:${{ steps.sha.outputs.OC_SHORTHASH }}" >> $GITHUB_OUTPUT
+          echo "IMAGE_TAG_LATEST=ghcr.io/${{ github.repository_owner }}/opencost:latest" >> $GITHUB_OUTPUT
+          echo "IMAGE_TAG_VERSION=ghcr.io/${{ github.repository_owner }}/opencost:${{ steps.version_number.outputs.RELEASE_VERSION }}" >> $GITHUB_OUTPUT
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3

.github/workflows/build-test.yaml

@@ -87,9 +87,9 @@ jobs:
       - name: Upload code coverage
         uses: actions/upload-artifact@v4
         with:
-          name: oc-code-coverage
+          name: code-coverage
           path: |
            coverage.out
            pr_num.txt
            base.txt
-           head.txt
+           head.txt

.github/workflows/sonar.yaml

@@ -1,4 +1,4 @@
-name: Sonar Code Coverage Upload
+name: Sonar
 on:
   workflow_run:
     workflows: ["Build/Test"]
@@ -8,43 +8,74 @@ jobs:
     name: Sonar
     runs-on: ubuntu-latest
     if: github.event.workflow_run.conclusion == 'success'
+    permissions:
+      checks: write
+      contents: read
+      actions: read
     steps:
+      - uses: LouisBrunner/checks-action@v2.0.0
+        if: always()
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          name: Quality Gate
+          status: in_progress
+          sha: ${{ github.event.workflow_run.head_sha }}
       - uses: actions/checkout@v4
         with:
           repository: ${{ github.event.workflow_run.head_repository.full_name }}
           ref: ${{ github.event.workflow_run.head_branch }}
           fetch-depth: 0
-      - name: 'Download code coverage'
-        uses: actions/github-script@v7
+      - name: Download coverage artifacts
+        uses: actions/download-artifact@v4
         with:
-          script: |
-            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               run_id: context.payload.workflow_run.id,
-            });
-            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
-              return artifact.name == "oc-code-coverage"
-            })[0];
-            let download = await github.rest.actions.downloadArtifact({
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               artifact_id: matchArtifact.id,
-               archive_format: 'zip',
-            });
-            let fs = require('fs');
-            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/oc-code-coverage.zip`, Buffer.from(download.data));
-      - name: 'Unzip code coverage'
-        run: unzip oc-code-coverage.zip -d coverage
-      - name: set env vars 
+          name: code-coverage
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ github.token }}
+          path: pr-artifact
+      - name: Validate Coverage Vars
+        id: validate-vars
+        if: github.event.workflow_run.head_branch != 'develop'
+        shell: bash
+        run: | 
+          # check the PR number
+          pr_content=$(cat pr-artifact/pr_num.txt | tr -d '\n' | tr -d ' ')
+
+          # Check if the content matches a single number
+          if [[ "$pr_content" =~ ^[0-9]+$ ]]; then
+            echo "The file 'pr_num.txt' contains a single number: $pr_content"
+          else
+            echo "The file 'pr_num.txt' does not contain a single number."
+            exit 1
+          fi
+
+          base_content=$(cat pr-artifact/base.txt | tr -d '\n' | tr -d ' ')
+          if git check-ref-format --allow-onelevel "$base_content"; then
+            echo "The file 'base.txt' contains a valid git ref: $base_content"
+          else
+            echo "The file 'base.txt' does not contain a valid git ref: $base_content"
+            exit 1
+          fi
+
+          head_content=$(cat pr-artifact/head.txt | tr -d '\n' | tr -d ' ')
+          if git check-ref-format --allow-onelevel "$head_content"; then
+            echo "The file 'head.txt' contains a valid git ref: $head_content"
+          else
+            echo "The file 'head.txt' does not contain a valid git ref: $head_content"
+            exit 1
+          fi
+
+      - name: set vars 
+        id: set-vars
         run: | 
-          echo "SONAR_PR_NUM=$(cat coverage/pr_num.txt)" >> $GITHUB_ENV
-          echo "SONAR_BASE=$(cat coverage/base.txt)" >> $GITHUB_ENV
-          echo "SONAR_HEAD=$(cat coverage/head.txt)" >> $GITHUB_ENV
+          echo "SONAR_PR_NUM=$(cat pr-artifact/pr_num.txt | tr -d '\n' | tr -d ' ')" >> $GITHUB_OUTPUT
+          echo "SONAR_BASE=$(cat pr-artifact/base.txt | tr -d '\n' | tr -d ' ')" >> $GITHUB_OUTPUT
+          echo "SONAR_HEAD=$(cat pr-artifact/head.txt | tr -d '\n' | tr -d ' ')" >> $GITHUB_OUTPUT
+          # move coverage file to root where sonar properties file is expecting it
+          cp pr-artifact/coverage.out coverage.out
       # on develop branch, only run a baseline scan
       - name: SonarCloud Scan (Baseline)
         uses: sonarsource/sonarcloud-github-action@master
-        if: env.SONAR_HEAD == 'develop'
+        if: github.event.workflow_run.head_branch == 'develop'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
@@ -54,18 +85,48 @@ jobs:
             -Dsonar.projectKey=opencost_opencost
             -Dsonar.organization=opencost
             -Dsonar.branch.name=develop
-            -Dsonar.branch.target=develop
       - name: SonarCloud Scan (PR)
         uses: sonarsource/sonarcloud-github-action@master
-        if: env.SONAR_HEAD != 'develop'
+        if: github.event.workflow_run.head_branch != 'develop'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         with:
           args: >
             -Dsonar.scm.revision=${{ github.event.workflow_run.head_sha }}
-            -Dsonar.pullrequest.key=${{ env.SONAR_PR_NUM }}
-            -Dsonar.pullrequest.branch=${{ env.SONAR_HEAD }}
-            -Dsonar.pullrequest.base=${{ env.SONAR_BASE }}
+            -Dsonar.pullrequest.key=${{ steps.set-vars.outputs.SONAR_PR_NUM }}
+            -Dsonar.pullrequest.branch="${{ steps.set-vars.outputs.SONAR_HEAD }}"
+            -Dsonar.pullrequest.base="${{ steps.set-vars.outputs.SONAR_BASE }}"
             -Dsonar.projectKey=opencost_opencost
-            -Dsonar.organization=opencost
+            -Dsonar.organization=opencost
+      - name: SonarQube Quality Gate check
+        id: sonarqube-quality-gate-check
+        continue-on-error: true
+        uses: sonarsource/sonarqube-quality-gate-action@master
+        # fail step after specific time.
+        timeout-minutes: 5
+        env:
+             SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+             SONAR_HOST_URL: "https://sonarcloud.io"
+      - uses: LouisBrunner/checks-action@v2.0.0
+        id: fail-quality-gate
+        if: steps.sonarqube-quality-gate-check.outputs.quality-gate-status != 'PASSED'
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          name: Quality Gate
+          status: completed
+          conclusion: failure
+          sha: ${{ github.event.workflow_run.head_sha }}
+          output: |
+            {"summary":"Failed - see https://sonarcloud.io/summary/new_code?id=opencost_opencostl&pullRequest=${{ steps.set-vars.outputs.SONAR_PR_NUM }}","text_description":"Quality Gate failed. Check the [SonarCloud Dashboard](https://sonarcloud.io/dashboard?id=opencost_opencost&pullRequest=${{ steps.set-vars.outputs.SONAR_PR_NUM }}) for more details."}
+      - uses: LouisBrunner/checks-action@v2.0.0
+        id: pass-quality-gate
+        if: steps.sonarqube-quality-gate-check.outputs.quality-gate-status == 'PASSED'
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          name: Quality Gate
+          status: completed
+          conclusion: success
+          sha: ${{ github.event.workflow_run.head_sha }}
+          output: |
+            {"summary":"Passed","text_description":"Quality Gate passed. Check the [SonarCloud Dashboard](https://sonarcloud.io/dashboard?id=opencost_opencost&pullRequest=${{ steps.set-vars.outputs.SONAR_PR_NUM }}) for more details."} 

.gitignore

@@ -9,6 +9,8 @@ cmd/costmodel/costmodel-tilt
 
 pkg/cloud/azureorphan_test.go
 
+pkg/cloud/oracle/cloud-integration.json
+
 # VS Code
 .vscode
 

MAINTAINERS.md

@@ -8,15 +8,16 @@ Official list of [OpenCost Maintainers](https://github.com/orgs/opencost/teams/o
 | --------------- | --------- | ----------- | ----------- |
 | Ajay Tripathy | @AjayTripathy | Kubecost | <Ajay@kubecost.com> |
 | Alex Meijer | @ameijer | Kubecost | <ameijer@kubecost.com> |
-| Artur Khantimirov | @r2k1 | Microsoft | |
+| Artur Khantimirov | @r2k1 | Microsoft | <akhantimirov@microsoft.com> |
 | Matt Bolt | @​mbolt35 | Kubecost | <matt@kubecost.com> |
-| Matt Ray | @mattray | Kubecost | <mattray@kubecost.com> |
 | Niko Kovacevic | @nikovacevic | Kubecost | <niko@kubecost.com> |
 | Sean Holcomb | @Sean-Holcomb | Kubecost | <Sean@kubecost.com> |
 | Thomas Evans | @teevans | Kubecost | <thomas@kubecost.com> |
 
 ## Opencost Emeritus Committers
 We would like to acknowledge previous committers and their huge contributions to our collective success:
+
 | Maintainer | GitHub ID | Affiliation | Email |
 | --------------- | --------- | ----------- | ----------- |
-| Michael Dresser | @michaelmdresser | Kubecost | <michaelmdresser@gmail.com> |
+| Michael Dresser | @michaelmdresser | Kubecost (former) | <michaelmdresser@gmail.com> |
+| Matt Ray | @mattray | Kubecost (former) | <mattray@kubecost.com> |

README.md

@@ -1,7 +1,7 @@
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/6219/badge)](https://www.bestpractices.dev/projects/6219)
 
-<img src="./opencost-header.png"/>
+![](./opencost-header.png)
 
 # OpenCost — your favorite open source cost monitoring tool for Kubernetes and cloud spend
 

core/go.mod

@@ -1,6 +1,6 @@
 module github.com/opencost/opencost/core
 
-go 1.21.0
+go 1.22.0
 
 require (
 	github.com/davecgh/go-spew v1.1.1
@@ -16,7 +16,7 @@ require (
 	golang.org/x/sync v0.6.0
 	golang.org/x/text v0.14.0
 	google.golang.org/grpc v1.62.0
-	google.golang.org/protobuf v1.32.0
+	google.golang.org/protobuf v1.33.0
 	k8s.io/api v0.25.3
 	k8s.io/apimachinery v0.25.3
 )
@@ -47,8 +47,8 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
-	golang.org/x/net v0.21.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

core/go.sum

@@ -386,8 +386,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -463,8 +463,8 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -641,8 +641,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
-google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=

core/pkg/filter/cloudcost/fields.go

@@ -8,11 +8,15 @@ import (
 type CloudCostField string
 
 const (
-	FieldInvoiceEntityID CloudCostField = CloudCostField(fieldstrings.FieldInvoiceEntityID)
-	FieldAccountID       CloudCostField = CloudCostField(fieldstrings.FieldAccountID)
-	FieldProvider        CloudCostField = CloudCostField(fieldstrings.FieldProvider)
-	FieldProviderID      CloudCostField = CloudCostField(fieldstrings.FieldProviderID)
-	FieldCategory        CloudCostField = CloudCostField(fieldstrings.FieldCategory)
-	FieldService         CloudCostField = CloudCostField(fieldstrings.FieldService)
-	FieldLabel           CloudCostField = CloudCostField(fieldstrings.FieldLabel)
+	FieldInvoiceEntityID   CloudCostField = CloudCostField(fieldstrings.FieldInvoiceEntityID)
+	FieldInvoiceEntityName CloudCostField = CloudCostField(fieldstrings.FieldInvoiceEntityName)
+	FieldAccountID         CloudCostField = CloudCostField(fieldstrings.FieldAccountID)
+	FieldAccountName       CloudCostField = CloudCostField(fieldstrings.FieldAccountName)
+	FieldRegionID          CloudCostField = CloudCostField(fieldstrings.FieldRegionID)
+	FieldAvailabilityZone  CloudCostField = CloudCostField(fieldstrings.FieldAvailabilityZone)
+	FieldProvider          CloudCostField = CloudCostField(fieldstrings.FieldProvider)
+	FieldProviderID        CloudCostField = CloudCostField(fieldstrings.FieldProviderID)
+	FieldCategory          CloudCostField = CloudCostField(fieldstrings.FieldCategory)
+	FieldService           CloudCostField = CloudCostField(fieldstrings.FieldService)
+	FieldLabel             CloudCostField = CloudCostField(fieldstrings.FieldLabel)
 )

core/pkg/filter/cloudcost/parser.go

@@ -6,7 +6,11 @@ import "github.com/opencost/opencost/core/pkg/filter/ast"
 // valid left-hand comparators
 var cloudCostFilterFields []*ast.Field = []*ast.Field{
 	ast.NewField(FieldInvoiceEntityID),
+	ast.NewField(FieldInvoiceEntityName),
 	ast.NewField(FieldAccountID),
+	ast.NewField(FieldAccountName),
+	ast.NewField(FieldRegionID),
+	ast.NewField(FieldAvailabilityZone),
 	ast.NewField(FieldProvider),
 	ast.NewField(FieldProviderID),
 	ast.NewField(FieldCategory),

core/pkg/filter/cloudcost/parser_test.go

@@ -0,0 +1,76 @@
+package cloudcost
+
+import (
+	"testing"
+)
+
+func TestNewCloudCostFilterParserParse(t *testing.T) {
+	parser := NewCloudCostFilterParser()
+	testCases := map[string]struct {
+		input       string
+		expectError bool
+	}{
+		"Empty": {
+			input:       ``,
+			expectError: false,
+		},
+		"InvoiceEntityID": {
+			input:       `invoiceEntityID: "123"`,
+			expectError: false,
+		},
+		"InvoiceEntityName": {
+			input:       `invoiceEntityName: "foo"`,
+			expectError: false,
+		},
+		"AccountID": {
+			input:       `accountID: "123"`,
+			expectError: false,
+		},
+		"AccountName": {
+			input:       `accountName: "foo"`,
+			expectError: false,
+		},
+		"RegionID": {
+			input:       `regionID: "us-west-1"`,
+			expectError: false,
+		},
+		"AvailabilityZone": {
+			input:       `availabilityZone: "us-west-1a"`,
+			expectError: false,
+		},
+		"Provider": {
+			input:       `provider: "aws"`,
+			expectError: false,
+		},
+		"ProviderID": {
+			input:       `providerID: "i-123"`,
+			expectError: false,
+		},
+		"Category": {
+			input:       `category: "compute"`,
+			expectError: false,
+		},
+		"Service": {
+			input:       `service: "ec2"`,
+			expectError: false,
+		},
+		"Label": {
+			input:       `label[foo]:"bar"`,
+			expectError: false,
+		},
+		"InvalidField": {
+			input:       `foo: "bar"`,
+			expectError: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			_, err := parser.Parse(tc.input)
+			if (err != nil) != tc.expectError {
+				t.Errorf("expected error: %v, got: %v", tc.expectError, err)
+			}
+		})
+	}
+
+}

core/pkg/filter/fieldstrings/fieldstrings.go

@@ -24,8 +24,12 @@ const (
 	FieldAccount    string = "account"
 	FieldService    string = "service"
 
-	FieldInvoiceEntityID string = "invoiceEntityID"
-	FieldAccountID       string = "accountID"
+	FieldInvoiceEntityID   string = "invoiceEntityID"
+	FieldInvoiceEntityName string = "invoiceEntityName"
+	FieldAccountID         string = "accountID"
+	FieldAccountName       string = "accountName"
+	FieldRegionID          string = "regionID"
+	FieldAvailabilityZone  string = "availabilityZone"
 
 	AliasDepartment  string = "department"
 	AliasEnvironment string = "environment"

core/pkg/log/log.go

@@ -49,6 +49,14 @@ func InitLogging(showLogLevelSetMessage bool) {
 
 }
 
+func GetLogger() *zerolog.Logger {
+	return &log.Logger
+}
+
+func SetLogger(l *zerolog.Logger) {
+	log.Logger = *l
+}
+
 func GetLogLevel() string {
 	return zerolog.GlobalLevel().String()
 }

core/pkg/log/log_test.go

@@ -0,0 +1,64 @@
+package log
+
+import (
+	"bytes"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/rs/zerolog"
+)
+
+func TestGetLogger(t *testing.T) {
+	initialLogger := GetLogger()
+	if initialLogger == nil {
+		t.Error("GetLogger() returned nil")
+	}
+
+	secondLogger := GetLogger()
+	if initialLogger != secondLogger {
+		t.Error("GetLogger() returned different loggers on subsequent calls")
+	}
+}
+
+func TestSetLogger(t *testing.T) {
+	var buf bytes.Buffer
+	newLogger := zerolog.New(&buf).With().Str("test", "value").Logger()
+	SetLogger(&newLogger)
+
+	// Log a message using the global logger
+	Infof("Test message")
+
+	// Parse the logged message
+	loggedData := parseLogMessage(t, buf.String())
+
+	// Check if the "test" field is present in the logged message
+	if value, exists := loggedData["test"]; !exists || value != "value" {
+		t.Error("SetLogger() did not set the logger with expected context")
+	}
+}
+
+func TestLoggerConsistency(t *testing.T) {
+	var buf bytes.Buffer
+	newLogger := zerolog.New(&buf).With().Str("test", "consistency").Logger()
+	SetLogger(&newLogger)
+
+	// Log a message using the global logger
+	Infof("Consistency test message")
+
+	// Parse the logged message
+	loggedData := parseLogMessage(t, buf.String())
+
+	// Check if the "test" field is present in the logged message
+	if value, exists := loggedData["test"]; !exists || value != "consistency" {
+		t.Error("Logger inconsistency: Updated logger does not have expected context")
+	}
+}
+
+func parseLogMessage(t *testing.T, logMessage string) map[string]interface{} {
+	var loggedData map[string]interface{}
+	if err := json.Unmarshal([]byte(strings.TrimSpace(logMessage)), &loggedData); err != nil {
+		t.Fatalf("Failed to parse logged message: %v", err)
+	}
+	return loggedData
+}

core/pkg/opencost/allocation.go

@@ -63,9 +63,11 @@ type Allocation struct {
 	CPUCoreUsageAverage        float64               `json:"cpuCoreUsageAverage"`
 	CPUCost                    float64               `json:"cpuCost"`
 	CPUCostAdjustment          float64               `json:"cpuCostAdjustment"`
+	CPUCostIdle                float64               `json:"cpuCostIdle"` //@bingen:field[ignore]
 	GPUHours                   float64               `json:"gpuHours"`
 	GPUCost                    float64               `json:"gpuCost"`
 	GPUCostAdjustment          float64               `json:"gpuCostAdjustment"`
+	GPUCostIdle                float64               `json:"gpuCostIdle"` //@bingen:field[ignore]
 	NetworkTransferBytes       float64               `json:"networkTransferBytes"`
 	NetworkReceiveBytes        float64               `json:"networkReceiveBytes"`
 	NetworkCost                float64               `json:"networkCost"`
@@ -82,6 +84,7 @@ type Allocation struct {
 	RAMBytesUsageAverage       float64               `json:"ramByteUsageAverage"`
 	RAMCost                    float64               `json:"ramCost"`
 	RAMCostAdjustment          float64               `json:"ramCostAdjustment"`
+	RAMCostIdle                float64               `json:"ramCostIdle"` //@bingen:field[ignore]
 	SharedCost                 float64               `json:"sharedCost"`
 	ExternalCost               float64               `json:"externalCost"`
 	// RawAllocationOnly is a pointer so if it is not present it will be
@@ -669,11 +672,13 @@ func (a *Allocation) Clone() *Allocation {
 		CPUCoreRequestAverage:          a.CPUCoreRequestAverage,
 		CPUCoreUsageAverage:            a.CPUCoreUsageAverage,
 		CPUCost:                        a.CPUCost,
+		CPUCostIdle:                    a.CPUCostIdle,
 		CPUCostAdjustment:              a.CPUCostAdjustment,
 		GPUHours:                       a.GPUHours,
 		GPURequestAverage:              a.GPURequestAverage,
 		GPUUsageAverage:                a.GPUUsageAverage,
 		GPUCost:                        a.GPUCost,
+		GPUCostIdle:                    a.GPUCostIdle,
 		GPUCostAdjustment:              a.GPUCostAdjustment,
 		NetworkTransferBytes:           a.NetworkTransferBytes,
 		NetworkReceiveBytes:            a.NetworkReceiveBytes,
@@ -690,6 +695,7 @@ func (a *Allocation) Clone() *Allocation {
 		RAMBytesRequestAverage:         a.RAMBytesRequestAverage,
 		RAMBytesUsageAverage:           a.RAMBytesUsageAverage,
 		RAMCost:                        a.RAMCost,
+		RAMCostIdle:                    a.RAMCostIdle,
 		RAMCostAdjustment:              a.RAMCostAdjustment,
 		SharedCost:                     a.SharedCost,
 		ExternalCost:                   a.ExternalCost,
@@ -731,6 +737,9 @@ func (a *Allocation) Equal(that *Allocation) bool {
 	if !util.IsApproximately(a.CPUCost, that.CPUCost) {
 		return false
 	}
+	if !util.IsApproximately(a.CPUCostIdle, that.CPUCostIdle) {
+		return false
+	}
 	if !util.IsApproximately(a.CPUCostAdjustment, that.CPUCostAdjustment) {
 		return false
 	}
@@ -740,6 +749,9 @@ func (a *Allocation) Equal(that *Allocation) bool {
 	if !util.IsApproximately(a.GPUCost, that.GPUCost) {
 		return false
 	}
+	if !util.IsApproximately(a.GPUCostIdle, that.GPUCostIdle) {
+		return false
+	}
 	if !util.IsApproximately(a.GPUCostAdjustment, that.GPUCostAdjustment) {
 		return false
 	}
@@ -779,6 +791,9 @@ func (a *Allocation) Equal(that *Allocation) bool {
 	if !util.IsApproximately(a.RAMCost, that.RAMCost) {
 		return false
 	}
+	if !util.IsApproximately(a.RAMCostIdle, that.RAMCostIdle) {
+		return false
+	}
 	if !util.IsApproximately(a.RAMCostAdjustment, that.RAMCostAdjustment) {
 		return false
 	}
@@ -920,7 +935,7 @@ func (a *Allocation) CPUEfficiency() float64 {
 		return a.CPUCoreUsageAverage / a.CPUCoreRequestAverage
 	}
 
-	if a.CPUCoreUsageAverage == 0.0 || a.CPUCost == 0.0 {
+	if a.CPUCoreUsageAverage == 0.0 || a.CPUTotalCost() == 0.0 {
 		return 0.0
 	}
 
@@ -939,7 +954,7 @@ func (a *Allocation) RAMEfficiency() float64 {
 		return a.RAMBytesUsageAverage / a.RAMBytesRequestAverage
 	}
 
-	if a.RAMBytesUsageAverage == 0.0 || a.RAMCost == 0.0 {
+	if a.RAMBytesUsageAverage == 0.0 || a.RAMTotalCost() == 0.0 {
 		return 0.0
 	}
 
@@ -958,7 +973,7 @@ func (a *Allocation) GPUEfficiency() float64 {
 		return a.GPUUsageAverage / a.GPURequestAverage
 	}
 
-	if a.GPUUsageAverage == 0.0 || a.GPUCost == 0.0 {
+	if a.GPUUsageAverage == 0.0 || a.GPUTotalCost() == 0.0 {
 		return 0.0
 	}
 
@@ -1249,6 +1264,9 @@ func (a *Allocation) add(that *Allocation) {
 	a.CPUCost += that.CPUCost
 	a.GPUCost += that.GPUCost
 	a.RAMCost += that.RAMCost
+	a.CPUCostIdle += that.CPUCostIdle
+	a.GPUCostIdle += that.GPUCostIdle
+	a.RAMCostIdle += that.RAMCostIdle
 	a.NetworkCost += that.NetworkCost
 	a.NetworkCrossZoneCost += that.NetworkCrossZoneCost
 	a.NetworkCrossRegionCost += that.NetworkCrossRegionCost
@@ -2545,6 +2563,10 @@ func (a *Allocation) SanitizeNaN() {
 		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for CPUCost: name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
 		a.CPUCost = 0
 	}
+	if math.IsNaN(a.CPUCostIdle) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for CPUCostIdle: name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.CPUCostIdle = 0
+	}
 	if math.IsNaN(a.CPUCoreRequestAverage) {
 		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for CPUCoreRequestAverage: name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
 		a.CPUCoreRequestAverage = 0
@@ -2577,6 +2599,10 @@ func (a *Allocation) SanitizeNaN() {
 		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for GPUCost name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
 		a.GPUCost = 0
 	}
+	if math.IsNaN(a.GPUCostIdle) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for GPUCostIdle name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.GPUCostIdle = 0
+	}
 	if math.IsNaN(a.GPUCostAdjustment) {
 		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for GPUCostAdjustment name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
 		a.GPUCostAdjustment = 0
@@ -2637,6 +2663,10 @@ func (a *Allocation) SanitizeNaN() {
 		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for RAMCost name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
 		a.RAMCost = 0
 	}
+	if math.IsNaN(a.RAMCostIdle) {
+		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for RAMCostIdle name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
+		a.RAMCostIdle = 0
+	}
 	if math.IsNaN(a.RAMCostAdjustment) {
 		log.DedupedWarningf(5, "Allocation: Unexpected NaN found for RAMCostAdjustment name:%s, window:%s, properties:%s", a.Name, a.Window.String(), a.Properties.String())
 		a.RAMCostAdjustment = 0

core/pkg/opencost/allocation_json.go

@@ -23,6 +23,7 @@ type AllocationJSON struct {
 	CPUCoreHours                   *float64                        `json:"cpuCoreHours"`
 	CPUCost                        *float64                        `json:"cpuCost"`
 	CPUCostAdjustment              *float64                        `json:"cpuCostAdjustment"`
+	CPUCostIdle                    *float64                        `json:"cpuCostIdle"`
 	CPUEfficiency                  *float64                        `json:"cpuEfficiency"`
 	GPUCount                       *float64                        `json:"gpuCount"`
 	GPURequestAverage              *float64                        `json:"gpuRequestAverage"`
@@ -30,6 +31,7 @@ type AllocationJSON struct {
 	GPUHours                       *float64                        `json:"gpuHours"`
 	GPUCost                        *float64                        `json:"gpuCost"`
 	GPUCostAdjustment              *float64                        `json:"gpuCostAdjustment"`
+	GPUCostIdle                    *float64                        `json:"gpuCostIdle"`
 	GPUEfficiency                  *float64                        `json:"gpuEfficiency"`
 	NetworkTransferBytes           *float64                        `json:"networkTransferBytes"`
 	NetworkReceiveBytes            *float64                        `json:"networkReceiveBytes"`
@@ -51,6 +53,7 @@ type AllocationJSON struct {
 	RAMByteHours                   *float64                        `json:"ramByteHours"`
 	RAMCost                        *float64                        `json:"ramCost"`
 	RAMCostAdjustment              *float64                        `json:"ramCostAdjustment"`
+	RAMCostIdle                    *float64                        `json:"ramCostIdle"`
 	RAMEfficiency                  *float64                        `json:"ramEfficiency"`
 	ExternalCost                   *float64                        `json:"externalCost"`
 	SharedCost                     *float64                        `json:"sharedCost"`
@@ -78,6 +81,7 @@ func (aj *AllocationJSON) BuildFromAllocation(a *Allocation) {
 	aj.CPUCoreHours = formatFloat64ForResponse(a.CPUCoreHours)
 	aj.CPUCost = formatFloat64ForResponse(a.CPUCost)
 	aj.CPUCostAdjustment = formatFloat64ForResponse(a.CPUCostAdjustment)
+	aj.CPUCostIdle = formatFloat64ForResponse(a.CPUCostIdle)
 	aj.CPUEfficiency = formatFloat64ForResponse(a.CPUEfficiency())
 	aj.GPUCount = formatFloat64ForResponse(a.GPUs())
 	aj.GPURequestAverage = formatFloat64ForResponse(a.GPURequestAverage)
@@ -85,6 +89,7 @@ func (aj *AllocationJSON) BuildFromAllocation(a *Allocation) {
 	aj.GPUHours = formatFloat64ForResponse(a.GPUHours)
 	aj.GPUCost = formatFloat64ForResponse(a.GPUCost)
 	aj.GPUCostAdjustment = formatFloat64ForResponse(a.GPUCostAdjustment)
+	aj.GPUCostIdle = formatFloat64ForResponse(a.GPUCostIdle)
 	aj.GPUEfficiency = formatFloat64ForResponse(a.GPUEfficiency())
 	aj.NetworkTransferBytes = formatFloat64ForResponse(a.NetworkTransferBytes)
 	aj.NetworkReceiveBytes = formatFloat64ForResponse(a.NetworkReceiveBytes)
@@ -106,6 +111,7 @@ func (aj *AllocationJSON) BuildFromAllocation(a *Allocation) {
 	aj.RAMByteHours = formatFloat64ForResponse(a.RAMByteHours)
 	aj.RAMCost = formatFloat64ForResponse(a.RAMCost)
 	aj.RAMCostAdjustment = formatFloat64ForResponse(a.RAMCostAdjustment)
+	aj.RAMCostIdle = formatFloat64ForResponse(a.RAMCostIdle)
 	aj.RAMEfficiency = formatFloat64ForResponse(a.RAMEfficiency())
 	aj.SharedCost = formatFloat64ForResponse(a.SharedCost)
 	aj.ExternalCost = formatFloat64ForResponse(a.ExternalCost)

core/pkg/opencost/bingen.go

@@ -62,7 +62,7 @@ package opencost
 // @bingen:generate:LbAllocation
 // @bingen:end
 
-// @bingen:set[name=CloudCost,version=2]
+// @bingen:set[name=CloudCost,version=3]
 // @bingen:generate:CloudCost
 // @bingen:generate:CostMetric
 // @bingen:generate[stringtable]:CloudCostSet

core/pkg/opencost/cloudcost.go

@@ -104,8 +104,16 @@ func (cc *CloudCost) StringProperty(prop string) (string, error) {
 	switch prop {
 	case CloudCostInvoiceEntityIDProp:
 		return cc.Properties.InvoiceEntityID, nil
+	case CloudCostInvoiceEntityNameProp:
+		return cc.Properties.InvoiceEntityName, nil
 	case CloudCostAccountIDProp:
 		return cc.Properties.AccountID, nil
+	case CloudCostAccountNameProp:
+		return cc.Properties.AccountName, nil
+	case CloudCostRegionIDProp:
+		return cc.Properties.RegionID, nil
+	case CloudCostAvailabilityZoneProp:
+		return cc.Properties.AvailabilityZone, nil
 	case CloudCostProviderProp:
 		return cc.Properties.Provider, nil
 	case CloudCostProviderIDProp:

core/pkg/opencost/cloudcostmatcher.go

@@ -48,8 +48,16 @@ func cloudCostFieldMap(cc *CloudCost, identifier ast.Identifier) (string, error)
 	switch ccfilter.CloudCostField(identifier.Field.Name) {
 	case ccfilter.FieldInvoiceEntityID:
 		return cc.Properties.InvoiceEntityID, nil
+	case ccfilter.FieldInvoiceEntityName:
+		return cc.Properties.InvoiceEntityName, nil
 	case ccfilter.FieldAccountID:
 		return cc.Properties.AccountID, nil
+	case ccfilter.FieldAccountName:
+		return cc.Properties.AccountName, nil
+	case ccfilter.FieldRegionID:
+		return cc.Properties.RegionID, nil
+	case ccfilter.FieldAvailabilityZone:
+		return cc.Properties.AvailabilityZone, nil
 	case ccfilter.FieldProvider:
 		return cc.Properties.Provider, nil
 	case ccfilter.FieldProviderID:

core/pkg/opencost/cloudcostprops.go

@@ -28,14 +28,18 @@ func (apt *CloudCostProperty) GetLabel() string {
 }
 
 const (
-	CloudCostInvoiceEntityIDProp string = "invoiceEntityID"
-	CloudCostAccountIDProp       string = "accountID"
-	CloudCostProviderProp        string = "provider"
-	CloudCostProviderIDProp      string = "providerID"
-	CloudCostCategoryProp        string = "category"
-	CloudCostServiceProp         string = "service"
-	CloudCostLabelProp           string = "label"
-	CloudCostLabelSetProp        string = "labelSet"
+	CloudCostInvoiceEntityIDProp   string = "invoiceEntityID"
+	CloudCostInvoiceEntityNameProp string = "invoiceEntityName"
+	CloudCostAccountIDProp         string = "accountID"
+	CloudCostAccountNameProp       string = "accountName"
+	CloudCostRegionIDProp          string = "regionID"
+	CloudCostAvailabilityZoneProp  string = "availabilityZone"
+	CloudCostProviderProp          string = "provider"
+	CloudCostProviderIDProp        string = "providerID"
+	CloudCostCategoryProp          string = "category"
+	CloudCostServiceProp           string = "service"
+	CloudCostLabelProp             string = "label"
+	CloudCostLabelSetProp          string = "labelSet"
 )
 
 func ParseCloudProperties(props []string) ([]CloudCostProperty, error) {
@@ -61,8 +65,16 @@ func ParseCloudCostProperty(text string) (CloudCostProperty, error) {
 	switch strings.TrimSpace(strings.ToLower(text)) {
 	case "invoiceentityid":
 		return CloudCostProperty(CloudCostInvoiceEntityIDProp), nil
+	case "invoiceentityname":
+		return CloudCostProperty(CloudCostInvoiceEntityNameProp), nil
 	case "accountid":
 		return CloudCostProperty(CloudCostAccountIDProp), nil
+	case "accountname":
+		return CloudCostProperty(CloudCostAccountNameProp), nil
+	case "regionid":
+		return CloudCostProperty(CloudCostRegionIDProp), nil
+	case "availabilityzone":
+		return CloudCostProperty(CloudCostAvailabilityZoneProp), nil
 	case "provider":
 		return CloudCostProperty(CloudCostProviderProp), nil
 	case "providerid":
@@ -152,20 +164,28 @@ func (ccl CloudCostLabels) Intersection(that CloudCostLabels) CloudCostLabels {
 }
 
 type CloudCostProperties struct {
-	ProviderID      string          `json:"providerID,omitempty"`
-	Provider        string          `json:"provider,omitempty"`
-	AccountID       string          `json:"accountID,omitempty"`
-	InvoiceEntityID string          `json:"invoiceEntityID,omitempty"`
-	Service         string          `json:"service,omitempty"`
-	Category        string          `json:"category,omitempty"`
-	Labels          CloudCostLabels `json:"labels,omitempty"`
+	ProviderID        string          `json:"providerID,omitempty"`
+	Provider          string          `json:"provider,omitempty"`
+	AccountID         string          `json:"accountID,omitempty"`
+	AccountName       string          `json:"accountName,omitempty"` // @bingen:field[version=3]
+	InvoiceEntityID   string          `json:"invoiceEntityID,omitempty"`
+	InvoiceEntityName string          `json:"invoiceEntityName,omitempty"` // @bingen:field[version=3]
+	RegionID          string          `json:"regionID,omitempty"`          // @bingen:field[version=3]
+	AvailabilityZone  string          `json:"availabilityZone,omitempty"`  // @bingen:field[version=3]
+	Service           string          `json:"service,omitempty"`
+	Category          string          `json:"category,omitempty"`
+	Labels            CloudCostLabels `json:"labels,omitempty"`
 }
 
 func (ccp *CloudCostProperties) Equal(that *CloudCostProperties) bool {
 	return ccp.ProviderID == that.ProviderID &&
 		ccp.Provider == that.Provider &&
 		ccp.AccountID == that.AccountID &&
+		ccp.AccountName == that.AccountName &&
 		ccp.InvoiceEntityID == that.InvoiceEntityID &&
+		ccp.InvoiceEntityName == that.InvoiceEntityName &&
+		ccp.RegionID == that.RegionID &&
+		ccp.AvailabilityZone == that.AvailabilityZone &&
 		ccp.Service == that.Service &&
 		ccp.Category == that.Category &&
 		ccp.Labels.Equal(that.Labels)
@@ -173,13 +193,17 @@ func (ccp *CloudCostProperties) Equal(that *CloudCostProperties) bool {
 
 func (ccp *CloudCostProperties) Clone() *CloudCostProperties {
 	return &CloudCostProperties{
-		ProviderID:      ccp.ProviderID,
-		Provider:        ccp.Provider,
-		AccountID:       ccp.AccountID,
-		InvoiceEntityID: ccp.InvoiceEntityID,
-		Service:         ccp.Service,
-		Category:        ccp.Category,
-		Labels:          ccp.Labels.Clone(),
+		ProviderID:        ccp.ProviderID,
+		Provider:          ccp.Provider,
+		AccountID:         ccp.AccountID,
+		AccountName:       ccp.AccountName,
+		InvoiceEntityID:   ccp.InvoiceEntityID,
+		InvoiceEntityName: ccp.InvoiceEntityName,
+		RegionID:          ccp.RegionID,
+		AvailabilityZone:  ccp.AvailabilityZone,
+		Service:           ccp.Service,
+		Category:          ccp.Category,
+		Labels:            ccp.Labels.Clone(),
 	}
 }
 
@@ -206,9 +230,21 @@ func (ccp *CloudCostProperties) Intersection(that *CloudCostProperties) *CloudCo
 	if ccp.AccountID == that.AccountID {
 		intersectionCCP.AccountID = ccp.AccountID
 	}
+	if ccp.AccountName == that.AccountName {
+		intersectionCCP.AccountName = ccp.AccountName
+	}
 	if ccp.InvoiceEntityID == that.InvoiceEntityID {
 		intersectionCCP.InvoiceEntityID = ccp.InvoiceEntityID
 	}
+	if ccp.InvoiceEntityName == that.InvoiceEntityName {
+		intersectionCCP.InvoiceEntityName = ccp.InvoiceEntityName
+	}
+	if ccp.RegionID == that.RegionID {
+		intersectionCCP.RegionID = ccp.RegionID
+	}
+	if ccp.AvailabilityZone == that.AvailabilityZone {
+		intersectionCCP.AvailabilityZone = ccp.AvailabilityZone
+	}
 	if ccp.Service == that.Service {
 		intersectionCCP.Service = ccp.Service
 	}
@@ -220,15 +256,6 @@ func (ccp *CloudCostProperties) Intersection(that *CloudCostProperties) *CloudCo
 	return intersectionCCP
 }
 
-var cloudCostDefaultKeyProperties = []string{
-	CloudCostProviderProp,
-	CloudCostInvoiceEntityIDProp,
-	CloudCostAccountIDProp,
-	CloudCostCategoryProp,
-	CloudCostServiceProp,
-	CloudCostProviderIDProp,
-}
-
 // GenerateKey takes a list of properties and creates a "/" seperated key based on the values of the requested properties.
 // Invalid values are ignored with a warning. A nil input returns the default key, while an empty slice  returns the empty string
 func (ccp *CloudCostProperties) GenerateKey(props []string) string {
@@ -259,10 +286,26 @@ func (ccp *CloudCostProperties) GenerateKey(props []string) string {
 			if ccp.InvoiceEntityID != "" {
 				propVal = ccp.InvoiceEntityID
 			}
+		case prop == CloudCostInvoiceEntityNameProp:
+			if ccp.InvoiceEntityName != "" {
+				propVal = ccp.InvoiceEntityName
+			}
 		case prop == CloudCostAccountIDProp:
 			if ccp.AccountID != "" {
 				propVal = ccp.AccountID
 			}
+		case prop == CloudCostAccountNameProp:
+			if ccp.AccountName != "" {
+				propVal = ccp.AccountName
+			}
+		case prop == CloudCostRegionIDProp:
+			if ccp.RegionID != "" {
+				propVal = ccp.RegionID
+			}
+		case prop == CloudCostAvailabilityZoneProp:
+			if ccp.AvailabilityZone != "" {
+				propVal = ccp.AvailabilityZone
+			}
 		case prop == CloudCostServiceProp:
 			if ccp.Service != "" {
 				propVal = ccp.Service
@@ -298,7 +341,11 @@ func (ccp *CloudCostProperties) hashKey() string {
 	builder.WriteString(ccp.ProviderID)
 	builder.WriteString(ccp.Provider)
 	builder.WriteString(ccp.AccountID)
+	builder.WriteString(ccp.AccountName)
 	builder.WriteString(ccp.InvoiceEntityID)
+	builder.WriteString(ccp.InvoiceEntityName)
+	builder.WriteString(ccp.RegionID)
+	builder.WriteString(ccp.AvailabilityZone)
 	builder.WriteString(ccp.Service)
 	builder.WriteString(ccp.Category)
 

core/pkg/opencost/cloudcostprops_test.go

@@ -10,34 +10,46 @@ func TestCloudCostPropertiesIntersection(t *testing.T) {
 	}{
 		"When properties match between both CloudCostProperties": {
 			baseCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service1",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
 			},
 			intCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service1",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
 			},
 			expectedCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service1",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
@@ -45,34 +57,46 @@ func TestCloudCostPropertiesIntersection(t *testing.T) {
 		},
 		"When one of the properties differ in the two CloudCostProperties": {
 			baseCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service1",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
 			},
 			intCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service2",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service2",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
 			},
 			expectedCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
@@ -80,47 +104,108 @@ func TestCloudCostPropertiesIntersection(t *testing.T) {
 		},
 		"When two of the properties differ in the two CloudCostProperties": {
 			baseCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service1",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
 			},
 			intCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID2",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service2",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID2",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service2",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
 			},
 			expectedCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
 			},
 		},
+		"When all properties differ in the two CloudCostProperties": {
+			baseCCP: &CloudCostProperties{
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
+				Labels: map[string]string{
+					"key1": "value1",
+				},
+			},
+			intCCP: &CloudCostProperties{
+				Provider:          "CustomProvider2",
+				ProviderID:        "ProviderID2",
+				AccountID:         "WorkGroupID2",
+				AccountName:       "AccountName2",
+				InvoiceEntityID:   "InvoiceEntityID2",
+				InvoiceEntityName: "InvoiceEntityName2",
+				RegionID:          "RegionID2",
+				AvailabilityZone:  "AvailabilityZone2",
+				Service:           "Service2",
+				Category:          "Category2",
+				Labels: map[string]string{
+					"key2": "value2",
+				},
+			},
+			expectedCCP: &CloudCostProperties{
+				Provider:          "",
+				ProviderID:        "",
+				AccountID:         "",
+				AccountName:       "",
+				InvoiceEntityID:   "",
+				InvoiceEntityName: "",
+				RegionID:          "",
+				AvailabilityZone:  "",
+				Service:           "",
+				Category:          "",
+				Labels:            map[string]string{},
+			},
+		},
 		"When labels differ": {
 			baseCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service1",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 					"key2": "value2",
@@ -128,12 +213,16 @@ func TestCloudCostPropertiesIntersection(t *testing.T) {
 				},
 			},
 			intCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service1",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value2",
 					"key2": "value2",
@@ -141,12 +230,16 @@ func TestCloudCostPropertiesIntersection(t *testing.T) {
 				},
 			},
 			expectedCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service1",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key2": "value2",
 				},
@@ -176,45 +269,57 @@ func TestCloudCostProperties_hashKey(t *testing.T) {
 		},
 		"All props no labels": {
 			props: &CloudCostProperties{
-				ProviderID:      "providerid1",
-				Provider:        "provider1",
-				AccountID:       "workgroup1",
-				InvoiceEntityID: "billing1",
-				Service:         "service1",
-				Category:        "category1",
-				Labels:          map[string]string{},
-			},
-			want: "a19b7dddf0032572",
+				ProviderID:        "providerid1",
+				Provider:          "provider1",
+				AccountID:         "workgroup1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "billing1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "service1",
+				Category:          "category1",
+				Labels:            map[string]string{},
+			},
+			want: "d07ffd0bd6d5eaf1",
 		},
 		"All props": {
 			props: &CloudCostProperties{
-				ProviderID:      "providerid1",
-				Provider:        "provider1",
-				AccountID:       "workgroup1",
-				InvoiceEntityID: "billing1",
-				Service:         "service1",
-				Category:        "category1",
+				ProviderID:        "providerid1",
+				Provider:          "provider1",
+				AccountID:         "workgroup1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "billing1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "service1",
+				Category:          "category1",
 				Labels: map[string]string{
 					"label1": "value1",
 					"label2": "value2",
 				},
 			},
-			want: "9d54403e40ad4db6",
+			want: "318cb6294bf9e2d5",
 		},
 		"All props swap labels": {
 			props: &CloudCostProperties{
-				ProviderID:      "providerid1",
-				Provider:        "provider1",
-				AccountID:       "workgroup1",
-				InvoiceEntityID: "billing1",
-				Service:         "service1",
-				Category:        "category1",
+				ProviderID:        "providerid1",
+				Provider:          "provider1",
+				AccountID:         "workgroup1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "billing1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "service1",
+				Category:          "category1",
 				Labels: map[string]string{
 					"label2": "value2",
 					"label1": "value1",
 				},
 			},
-			want: "9d54403e40ad4db6",
+			want: "318cb6294bf9e2d5",
 		},
 	}
 	for name, tt := range tests {

core/pkg/opencost/opencost_codecs.go

@@ -13,12 +13,11 @@ package opencost
 
 import (
 	"fmt"
+	util "github.com/opencost/opencost/core/pkg/util"
 	"reflect"
 	"strings"
 	"sync"
 	"time"
-
-	util "github.com/opencost/opencost/core/pkg/util"
 )
 
 const (
@@ -44,7 +43,7 @@ const (
 	AllocationCodecVersion uint8 = 22
 
 	// CloudCostCodecVersion is used for any resources listed in the CloudCost version set
-	CloudCostCodecVersion uint8 = 2
+	CloudCostCodecVersion uint8 = 3
 )
 
 //--------------------------------------------------------------------------
@@ -3581,20 +3580,44 @@ func (target *CloudCostProperties) MarshalBinaryWithContext(ctx *EncodingContext
 		buff.WriteString(target.AccountID) // write string
 	}
 	if ctx.IsStringTable() {
-		d := ctx.Table.AddOrGet(target.InvoiceEntityID)
+		d := ctx.Table.AddOrGet(target.AccountName)
 		buff.WriteInt(d) // write table index
 	} else {
-		buff.WriteString(target.InvoiceEntityID) // write string
+		buff.WriteString(target.AccountName) // write string
 	}
 	if ctx.IsStringTable() {
-		e := ctx.Table.AddOrGet(target.Service)
+		e := ctx.Table.AddOrGet(target.InvoiceEntityID)
 		buff.WriteInt(e) // write table index
 	} else {
-		buff.WriteString(target.Service) // write string
+		buff.WriteString(target.InvoiceEntityID) // write string
 	}
 	if ctx.IsStringTable() {
-		f := ctx.Table.AddOrGet(target.Category)
+		f := ctx.Table.AddOrGet(target.InvoiceEntityName)
 		buff.WriteInt(f) // write table index
+	} else {
+		buff.WriteString(target.InvoiceEntityName) // write string
+	}
+	if ctx.IsStringTable() {
+		g := ctx.Table.AddOrGet(target.RegionID)
+		buff.WriteInt(g) // write table index
+	} else {
+		buff.WriteString(target.RegionID) // write string
+	}
+	if ctx.IsStringTable() {
+		h := ctx.Table.AddOrGet(target.AvailabilityZone)
+		buff.WriteInt(h) // write table index
+	} else {
+		buff.WriteString(target.AvailabilityZone) // write string
+	}
+	if ctx.IsStringTable() {
+		k := ctx.Table.AddOrGet(target.Service)
+		buff.WriteInt(k) // write table index
+	} else {
+		buff.WriteString(target.Service) // write string
+	}
+	if ctx.IsStringTable() {
+		l := ctx.Table.AddOrGet(target.Category)
+		buff.WriteInt(l) // write table index
 	} else {
 		buff.WriteString(target.Category) // write string
 	}
@@ -3608,14 +3631,14 @@ func (target *CloudCostProperties) MarshalBinaryWithContext(ctx *EncodingContext
 		buff.WriteInt(len(map[string]string(target.Labels))) // map length
 		for v, z := range map[string]string(target.Labels) {
 			if ctx.IsStringTable() {
-				g := ctx.Table.AddOrGet(v)
-				buff.WriteInt(g) // write table index
+				m := ctx.Table.AddOrGet(v)
+				buff.WriteInt(m) // write table index
 			} else {
 				buff.WriteString(v) // write string
 			}
 			if ctx.IsStringTable() {
-				h := ctx.Table.AddOrGet(z)
-				buff.WriteInt(h) // write table index
+				n := ctx.Table.AddOrGet(z)
+				buff.WriteInt(n) // write table index
 			} else {
 				buff.WriteString(z) // write string
 			}
@@ -3712,15 +3735,21 @@ func (target *CloudCostProperties) UnmarshalBinaryWithContext(ctx *DecodingConte
 	g := h
 	target.AccountID = g
 
-	var m string
-	if ctx.IsStringTable() {
-		n := buff.ReadInt() // read string index
-		m = ctx.Table[n]
+	// field version check
+	if uint8(3) <= version {
+		var m string
+		if ctx.IsStringTable() {
+			n := buff.ReadInt() // read string index
+			m = ctx.Table[n]
+		} else {
+			m = buff.ReadString() // read string
+		}
+		l := m
+		target.AccountName = l
+
 	} else {
-		m = buff.ReadString() // read string
+		target.AccountName = "" // default
 	}
-	l := m
-	target.InvoiceEntityID = l
 
 	var p string
 	if ctx.IsStringTable() {
@@ -3730,56 +3759,114 @@ func (target *CloudCostProperties) UnmarshalBinaryWithContext(ctx *DecodingConte
 		p = buff.ReadString() // read string
 	}
 	o := p
-	target.Service = o
+	target.InvoiceEntityID = o
 
-	var s string
+	// field version check
+	if uint8(3) <= version {
+		var s string
+		if ctx.IsStringTable() {
+			t := buff.ReadInt() // read string index
+			s = ctx.Table[t]
+		} else {
+			s = buff.ReadString() // read string
+		}
+		r := s
+		target.InvoiceEntityName = r
+
+	} else {
+		target.InvoiceEntityName = "" // default
+	}
+
+	// field version check
+	if uint8(3) <= version {
+		var w string
+		if ctx.IsStringTable() {
+			x := buff.ReadInt() // read string index
+			w = ctx.Table[x]
+		} else {
+			w = buff.ReadString() // read string
+		}
+		u := w
+		target.RegionID = u
+
+	} else {
+		target.RegionID = "" // default
+	}
+
+	// field version check
+	if uint8(3) <= version {
+		var aa string
+		if ctx.IsStringTable() {
+			bb := buff.ReadInt() // read string index
+			aa = ctx.Table[bb]
+		} else {
+			aa = buff.ReadString() // read string
+		}
+		y := aa
+		target.AvailabilityZone = y
+
+	} else {
+		target.AvailabilityZone = "" // default
+	}
+
+	var dd string
 	if ctx.IsStringTable() {
-		t := buff.ReadInt() // read string index
-		s = ctx.Table[t]
+		ee := buff.ReadInt() // read string index
+		dd = ctx.Table[ee]
 	} else {
-		s = buff.ReadString() // read string
+		dd = buff.ReadString() // read string
 	}
-	r := s
-	target.Category = r
+	cc := dd
+	target.Service = cc
+
+	var gg string
+	if ctx.IsStringTable() {
+		hh := buff.ReadInt() // read string index
+		gg = ctx.Table[hh]
+	} else {
+		gg = buff.ReadString() // read string
+	}
+	ff := gg
+	target.Category = ff
 
 	// --- [begin][read][alias](CloudCostLabels) ---
-	var u map[string]string
+	var kk map[string]string
 	if buff.ReadUInt8() == uint8(0) {
-		u = nil
+		kk = nil
 	} else {
 		// --- [begin][read][map](map[string]string) ---
-		x := buff.ReadInt() // map len
-		w := make(map[string]string, x)
-		for i := 0; i < x; i++ {
+		mm := buff.ReadInt() // map len
+		ll := make(map[string]string, mm)
+		for i := 0; i < mm; i++ {
 			var v string
-			var aa string
+			var oo string
 			if ctx.IsStringTable() {
-				bb := buff.ReadInt() // read string index
-				aa = ctx.Table[bb]
+				pp := buff.ReadInt() // read string index
+				oo = ctx.Table[pp]
 			} else {
-				aa = buff.ReadString() // read string
+				oo = buff.ReadString() // read string
 			}
-			y := aa
-			v = y
+			nn := oo
+			v = nn
 
 			var z string
-			var dd string
+			var rr string
 			if ctx.IsStringTable() {
-				ee := buff.ReadInt() // read string index
-				dd = ctx.Table[ee]
+				ss := buff.ReadInt() // read string index
+				rr = ctx.Table[ss]
 			} else {
-				dd = buff.ReadString() // read string
+				rr = buff.ReadString() // read string
 			}
-			cc := dd
-			z = cc
+			qq := rr
+			z = qq
 
-			w[v] = z
+			ll[v] = z
 		}
-		u = w
+		kk = ll
 		// --- [end][read][map](map[string]string) ---
 
 	}
-	target.Labels = CloudCostLabels(u)
+	target.Labels = CloudCostLabels(kk)
 	// --- [end][read][alias](CloudCostLabels) ---
 
 	return nil

core/pkg/opencost/summaryallocation.go

@@ -29,15 +29,18 @@ type SummaryAllocation struct {
 	CPUCoreRequestAverage  float64               `json:"cpuCoreRequestAverage"`
 	CPUCoreUsageAverage    float64               `json:"cpuCoreUsageAverage"`
 	CPUCost                float64               `json:"cpuCost"`
+	CPUCostIdle            float64               `json:"cpuCostIdle"`
 	GPURequestAverage      float64               `json:"gpuRequestAverage"`
 	GPUUsageAverage        float64               `json:"gpuUsageAverage"`
 	GPUCost                float64               `json:"gpuCost"`
+	GPUCostIdle            float64               `json:"gpuCostIdle"`
 	NetworkCost            float64               `json:"networkCost"`
 	LoadBalancerCost       float64               `json:"loadBalancerCost"`
 	PVCost                 float64               `json:"pvCost"`
 	RAMBytesRequestAverage float64               `json:"ramByteRequestAverage"`
 	RAMBytesUsageAverage   float64               `json:"ramByteUsageAverage"`
 	RAMCost                float64               `json:"ramCost"`
+	RAMCostIdle            float64               `json:"ramCostIdle"`
 	SharedCost             float64               `json:"sharedCost"`
 	ExternalCost           float64               `json:"externalCost"`
 	Share                  bool                  `json:"-"`

core/pkg/opencost/summaryallocation_json.go

@@ -15,15 +15,18 @@ type SummaryAllocationResponse struct {
 	CPUCoreRequestAverage  *float64  `json:"cpuCoreRequestAverage"`
 	CPUCoreUsageAverage    *float64  `json:"cpuCoreUsageAverage"`
 	CPUCost                *float64  `json:"cpuCost"`
+	CPUCostIdle            *float64  `json:"cpuCostIdle"`
 	GPURequestAverage      *float64  `json:"gpuRequestAverage"`
 	GPUUsageAverage        *float64  `json:"gpuUsageAverage"`
 	GPUCost                *float64  `json:"gpuCost"`
+	GPUCostIdle            *float64  `json:"gpuCostIdle"`
 	NetworkCost            *float64  `json:"networkCost"`
 	LoadBalancerCost       *float64  `json:"loadBalancerCost"`
 	PVCost                 *float64  `json:"pvCost"`
 	RAMBytesRequestAverage *float64  `json:"ramByteRequestAverage"`
 	RAMBytesUsageAverage   *float64  `json:"ramByteUsageAverage"`
 	RAMCost                *float64  `json:"ramCost"`
+	RAMCostIdle            *float64  `json:"ramCostIdle"`
 	SharedCost             *float64  `json:"sharedCost"`
 	ExternalCost           *float64  `json:"externalCost"`
 	TotalEfficiency        *float64  `json:"totalEfficiency"`
@@ -54,15 +57,18 @@ func (sa *SummaryAllocation) ToResponse() *SummaryAllocationResponse {
 		CPUCoreRequestAverage:  formatutil.Float64ToResponse(sa.CPUCoreRequestAverage),
 		CPUCoreUsageAverage:    formatutil.Float64ToResponse(sa.CPUCoreUsageAverage),
 		CPUCost:                formatutil.Float64ToResponse(sa.CPUCost),
+		CPUCostIdle:            formatutil.Float64ToResponse(sa.CPUCostIdle),
 		GPURequestAverage:      formatutil.Float64ToResponse(sa.GPURequestAverage),
 		GPUUsageAverage:        formatutil.Float64ToResponse(sa.GPUUsageAverage),
 		GPUCost:                formatutil.Float64ToResponse(sa.GPUCost),
+		GPUCostIdle:            formatutil.Float64ToResponse(sa.GPUCostIdle),
 		NetworkCost:            formatutil.Float64ToResponse(sa.NetworkCost),
 		LoadBalancerCost:       formatutil.Float64ToResponse(sa.LoadBalancerCost),
 		PVCost:                 formatutil.Float64ToResponse(sa.PVCost),
 		RAMBytesRequestAverage: formatutil.Float64ToResponse(sa.RAMBytesRequestAverage),
 		RAMBytesUsageAverage:   formatutil.Float64ToResponse(sa.RAMBytesUsageAverage),
 		RAMCost:                formatutil.Float64ToResponse(sa.RAMCost),
+		RAMCostIdle:            formatutil.Float64ToResponse(sa.RAMCostIdle),
 		SharedCost:             formatutil.Float64ToResponse(sa.SharedCost),
 		ExternalCost:           formatutil.Float64ToResponse(sa.ExternalCost),
 		TotalEfficiency:        formatutil.Float64ToResponse(efficiency),

core/pkg/opencost/window.go

@@ -956,7 +956,7 @@ func (w Window) getWeeklyWindow() Window {
 // getWeeklyWindows breaks up a window into weeks, with weeks starting on Sunday
 func (w Window) getWeeklyWindows() []Window {
 	wins := []Window{}
-	roundedWindow := w.getDailyWindow()
+	roundedWindow := w.getWeeklyWindow()
 
 	roundedStart := *roundedWindow.Start()
 	roundedEnd := *roundedWindow.End()

core/pkg/opencost/window_test.go

@@ -1192,3 +1192,118 @@ func TestBoundaryErrorIs(t *testing.T) {
 		t.Errorf("Multi wrap failure: %s != %s", baseError, multiWrapError)
 	}
 }
+func TestWindowGetWeeklyWindows(t *testing.T) {
+	testCases := []struct {
+		name     string
+		start    time.Time
+		end      time.Time
+		expected []Window
+	}{
+		{
+			name:  "Single week",
+			start: time.Date(2024, 9, 1, 0, 0, 0, 0, time.UTC),
+			end:   time.Date(2024, 9, 8, 0, 0, 0, 0, time.UTC),
+			expected: []Window{
+				NewClosedWindow(
+					time.Date(2024, 9, 1, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 9, 8, 0, 0, 0, 0, time.UTC),
+				),
+			},
+		},
+		{
+			name:  "Multiple weeks",
+			start: time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+			end:   time.Date(2024, 10, 20, 0, 0, 0, 0, time.UTC),
+			expected: []Window{
+				NewClosedWindow(
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 20, 0, 0, 0, 0, time.UTC),
+				),
+			},
+		},
+		{
+			name:  "Partial starting week",
+			start: time.Date(2024, 10, 1, 0, 0, 0, 0, time.UTC),
+			end:   time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+			expected: []Window{
+				NewClosedWindow(
+					time.Date(2024, 9, 29, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+				),
+			},
+		},
+		{
+			name:  "Partial ending week",
+			start: time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+			end:   time.Date(2024, 10, 7, 0, 0, 0, 0, time.UTC),
+			expected: []Window{
+				NewClosedWindow(
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+				),
+			},
+		},
+		{
+			name:  "multiple weeks, partial start and end",
+			start: time.Date(2024, 10, 1, 0, 0, 0, 0, time.UTC),
+			end:   time.Date(2024, 11, 21, 0, 0, 0, 0, time.UTC),
+			expected: []Window{
+				NewClosedWindow(
+					time.Date(2024, 9, 29, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 20, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 20, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 27, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 27, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 11, 3, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 11, 3, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 11, 10, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 11, 10, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 11, 17, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 11, 17, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 11, 24, 0, 0, 0, 0, time.UTC),
+				),
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			w := NewClosedWindow(tc.start, tc.end)
+			actual := w.getWeeklyWindows()
+			if len(actual) != len(tc.expected) {
+				t.Fatalf("expected %d windows, got %d", len(tc.expected), len(actual))
+			}
+			for i, expectedWindow := range tc.expected {
+				if !actual[i].Equal(expectedWindow) {
+					t.Errorf("expected window %s, got %s", expectedWindow, actual[i])
+				}
+			}
+		})
+	}
+}

core/pkg/util/worker/worker.go

@@ -9,6 +9,9 @@ import (
 	"github.com/opencost/opencost/core/pkg/collections"
 )
 
+// Runner is a function type that takes a single input and returns nothing.
+type Runner[T any] func(T)
+
 // Worker is a transformation function from input type T to output type U.
 type Worker[T any, U any] func(T) U
 
@@ -54,7 +57,7 @@ type queuedWorkerPool[T any, U any] struct {
 type ordered[T any, U any] struct {
 	workPool WorkerPool[T, U]
 	results  []U
-	wg       *sync.WaitGroup
+	wg       sync.WaitGroup
 	count    int
 }
 
@@ -129,7 +132,6 @@ func NewOrderedGroup[T any, U any](pool WorkerPool[T, U], size int) WorkGroup[T,
 	return &ordered[T, U]{
 		workPool: pool,
 		results:  make([]U, size),
-		wg:       new(sync.WaitGroup),
 		count:    0,
 	}
 }
@@ -166,6 +168,96 @@ func (ow *ordered[T, U]) Wait() []U {
 	return ow.results
 }
 
+// noResultGroup is a WorkGroup implementation which arbitrarily pushes inputs to
+// a runner pool to be executed concurrently. This group does not collect results.
+type noResultGroup[T any] struct {
+	workPool WorkerPool[T, struct{}]
+	wg       sync.WaitGroup
+}
+
+// NewNoResultGroup creates a new WorkGroup implementation for processing a group of inputs concurrently. This
+// work group implementation does not collect results, and therefore, requires a worker pool with a struct{} output.
+func NewNoResultGroup[T any](pool WorkerPool[T, struct{}]) WorkGroup[T, struct{}] {
+	return &noResultGroup[T]{
+		workPool: pool,
+	}
+}
+
+// Push adds a new input to the work group.
+func (ow *noResultGroup[T]) Push(input T) error {
+	onComplete := make(chan struct{})
+	err := ow.workPool.Run(input, onComplete)
+	if err != nil {
+		return err
+	}
+
+	ow.wg.Add(1)
+
+	go func() {
+		defer close(onComplete)
+		defer ow.wg.Done()
+
+		<-onComplete
+	}()
+
+	return nil
+}
+
+// Wait waits for all pending worker tasks to complete, then returns all the results.
+func (ow *noResultGroup[T]) Wait() []struct{} {
+	ow.wg.Wait()
+	return []struct{}{}
+}
+
+// collector is a WorkGroup implementation which collects non-nil results into the results slice
+// and ignores any nil results.
+type collector[T any, U any] struct {
+	workPool   WorkerPool[T, *U]
+	resultLock sync.Mutex
+	results    []*U
+	wg         sync.WaitGroup
+}
+
+// NewCollectionGroup creates a new WorkGroup implementation for processing a group of inputs concurrently. The
+// collection group implementation will collect all non-nil results into the output slice. Thus, the worker pool
+// parameter requires the output type to be a pointer.
+func NewCollectionGroup[T any, U any](pool WorkerPool[T, *U]) WorkGroup[T, *U] {
+	return &collector[T, U]{
+		workPool: pool,
+	}
+}
+
+// Push adds a new input to the work group.
+func (ow *collector[T, U]) Push(input T) error {
+	onComplete := make(chan *U)
+	err := ow.workPool.Run(input, onComplete)
+	if err != nil {
+		return err
+	}
+
+	ow.wg.Add(1)
+
+	go func() {
+		defer ow.wg.Done()
+		defer close(onComplete)
+
+		result := <-onComplete
+		if result != nil {
+			ow.resultLock.Lock()
+			ow.results = append(ow.results, result)
+			ow.resultLock.Unlock()
+		}
+	}()
+
+	return nil
+}
+
+// Wait waits for all pending worker tasks to complete, then returns all the results.
+func (ow *collector[T, U]) Wait() []*U {
+	ow.wg.Wait()
+	return ow.results
+}
+
 // these constraints protect against the possibility of unexpected output from runtime.NumCPU()
 const (
 	defaultMinWorkers = 4
@@ -190,10 +282,21 @@ func OptimalWorkerCountInRange(min int, max int) int {
 	return cores
 }
 
-// ConcurrentDo runs a pool of workers which concurrently call the provided worker func on each input to get ordered
-// output corresponding to the inputs
+// ConcurrentDo runs a pool of N workers which concurrently call the provided worker func on each
+// input to get ordered output corresponding to the inputs. The total number of workers is determined
+// by the total number of CPUs available, bound to a range from 4-16.
 func ConcurrentDo[T any, U any](worker Worker[T, U], inputs []T) []U {
-	workerPool := NewWorkerPool(OptimalWorkerCount(), worker)
+	return ConcurrentDoWith(OptimalWorkerCount(), worker, inputs)
+}
+
+// ConcurrentDoWith runs a pool of workers of the specified size which concurrently call the provided worker func
+// on each input to get ordered output corresponding to the inputs. Size inputs < 1 will automatically be set to 1.
+func ConcurrentDoWith[T any, U any](size int, worker Worker[T, U], inputs []T) []U {
+	if size < 1 {
+		size = 1
+	}
+
+	workerPool := NewWorkerPool(size, worker)
 	defer workerPool.Shutdown()
 
 	workGroup := NewOrderedGroup(workerPool, len(inputs))
@@ -203,3 +306,55 @@ func ConcurrentDo[T any, U any](worker Worker[T, U], inputs []T) []U {
 
 	return workGroup.Wait()
 }
+
+// ConcurrentCollect runs a pool of N workers which concurrently call the provided worker func on each
+// input to get a result slice of non-nil outputs. The total number of workers is determined
+// by the total number of CPUs available, bound to a range from 4-16.
+func ConcurrentCollect[T any, U any](workerFunc Worker[T, *U], inputs []T) []*U {
+	return ConcurrentCollectWith(OptimalWorkerCount(), workerFunc, inputs)
+}
+
+// ConcurrentCollectWith runs a pool of workers of the specified size which concurrently call the provided worker
+// func on each input to get a result slice of non-nil outputs. Size inputs < 1 will automatically be set to 1.
+func ConcurrentCollectWith[T any, U any](size int, workerFunc Worker[T, *U], inputs []T) []*U {
+	if size < 1 {
+		size = 1
+	}
+
+	workerPool := NewWorkerPool(size, workerFunc)
+	defer workerPool.Shutdown()
+
+	workGroup := NewCollectionGroup(workerPool)
+	for _, input := range inputs {
+		workGroup.Push(input)
+	}
+
+	return workGroup.Wait()
+}
+
+// ConcurrentRun runs a pool of N workers which concurrently call the provided runner func on each
+// input. The total number of workers is determined by the total number of CPUs available, bound to
+// a range from 4-16.
+func ConcurrentRun[T any](runner Runner[T], inputs []T) {
+	ConcurrentRunWith(OptimalWorkerCount(), runner, inputs)
+}
+
+// ConcurrentRunWith runs a pool of runners of the specified size which concurrently call the provided runner
+// func on each input. Size inputs < 1 will automatically be set to 1.
+func ConcurrentRunWith[T any](size int, runner Runner[T], inputs []T) {
+	if size < 1 {
+		size = 1
+	}
+
+	workerPool := NewWorkerPool(size, func(input T) (void struct{}) {
+		runner(input)
+		return
+	})
+
+	workGroup := NewNoResultGroup(workerPool)
+	for _, input := range inputs {
+		workGroup.Push(input)
+	}
+
+	workGroup.Wait()
+}

core/pkg/util/worker/worker_test.go

@@ -74,7 +74,6 @@ func TestWorkerPoolExactWorkers(t *testing.T) {
 	case <-time.After(5 * time.Second):
 		t.Errorf("Failed to Complete Run for %d jobs in 5s\n", workers)
 	}
-
 }
 
 func TestOrderedWorkGroup(t *testing.T) {
@@ -116,6 +115,38 @@ func TestOrderedWorkGroup(t *testing.T) {
 	// above assertion
 }
 
+func TestConcurrentRun(t *testing.T) {
+	const tasks = 50
+
+	var wg sync.WaitGroup
+	wg.Add(tasks)
+
+	// worker func logs start/finish for simulated work, returns input value
+	// for testing resulting group output
+	work := func(i int) {
+		defer wg.Done()
+
+		t.Logf("Starting Work: %d\n", i)
+		time.Sleep(time.Duration(rand.Intn(250)+250) * time.Millisecond)
+		t.Logf("Finished Work: %d\n", i)
+	}
+
+	// pre-build inputs
+	input := make([]int, tasks)
+	for i := 0; i < tasks; i++ {
+		input[i] = i + 1
+	}
+
+	// get results and verify they match the recorded inputs
+	ConcurrentRunWith(10, work, input)
+
+	select {
+	case <-waitChannelFor(&wg):
+	case <-time.After(5 * time.Second):
+		t.Errorf("Failed to Complete Run for %d jobs in 5s\n", tasks)
+	}
+}
+
 func TestConcurrentDoOrdered(t *testing.T) {
 	// Perform a similar test to the above ordered test, but use the helper func with pre-built inputs
 	const tasks = 50
@@ -147,3 +178,108 @@ func TestConcurrentDoOrdered(t *testing.T) {
 	// the result collection handles the ordering in the group, which is what we want to ensure in the
 	// above assertion
 }
+
+func TestConcurrentCollect(t *testing.T) {
+	type A struct {
+		Value int
+	}
+
+	type B struct {
+		Value int
+	}
+
+	// Perform a similar test to the above ordered test, but use the helper func with pre-built inputs
+	const tasks = 100
+	const expectedResults = 50
+
+	var inputs []*A
+	for i := 0; i < tasks; i++ {
+		inputs = append(inputs, &A{Value: i})
+	}
+
+	workerFunc := func(a *A) *B {
+		time.Sleep(time.Duration(rand.Intn(150)+100) * time.Millisecond)
+
+		if a.Value%2 == 0 {
+			return &B{Value: a.Value}
+		}
+
+		return nil
+	}
+
+	results := ConcurrentCollect(workerFunc, inputs)
+
+	if len(results) != expectedResults {
+		t.Errorf("Expected 50 results, got %d", len(results))
+	}
+
+	seen := map[int]bool{}
+	for _, result := range results {
+		if seen[result.Value] {
+			t.Errorf("Duplicate result: %d", result.Value)
+		}
+		seen[result.Value] = true
+
+		if result.Value%2 != 0 {
+			t.Errorf("Found odd value: %d", result.Value)
+		}
+	}
+}
+
+func TestConcurrentDoWithLessThanOne(t *testing.T) {
+	const tasks = 4
+
+	var wg sync.WaitGroup
+	wg.Add(tasks)
+
+	now := time.Now()
+
+	doIt := func(i int) int {
+		defer wg.Done()
+		time.Sleep(250 * time.Millisecond)
+		return i
+	}
+
+	results := ConcurrentDoWith(-1, doIt, []int{1, 2, 3, 4})
+
+	select {
+	case <-waitChannelFor(&wg):
+	case <-time.After(2 * time.Second):
+		t.Errorf("Failed to Complete Run for %d jobs in 2s\n", tasks)
+	}
+
+	if time.Since(now) > 1500*time.Millisecond {
+		t.Errorf("Expected to complete in 1.5s, took %dms", time.Since(now).Milliseconds())
+	}
+	for i := 1; i <= tasks; i++ {
+		if results[i-1] != i {
+			t.Errorf("Expected %d, got %d", i, results[i])
+		}
+	}
+}
+
+func TestConcurrentRunWithLessThanOne(t *testing.T) {
+	const tasks = 4
+
+	var wg sync.WaitGroup
+	wg.Add(tasks)
+
+	now := time.Now()
+
+	doIt := func(i int) {
+		defer wg.Done()
+		time.Sleep(250 * time.Millisecond)
+	}
+
+	ConcurrentRunWith(-1, doIt, []int{1, 2, 3, 4})
+
+	select {
+	case <-waitChannelFor(&wg):
+	case <-time.After(2 * time.Second):
+		t.Errorf("Failed to Complete Run for %d jobs in 2s\n", tasks)
+	}
+
+	if time.Since(now) > 1500*time.Millisecond {
+		t.Errorf("Expected to complete in 1.5s, took %dms", time.Since(now).Milliseconds())
+	}
+}

go.mod

@@ -65,6 +65,11 @@ require (
 	sigs.k8s.io/yaml v1.3.0
 )
 
+require (
+	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/sony/gobreaker v0.5.0 // indirect
+)
+
 require (
 	cloud.google.com/go v0.114.0 // indirect
 	cloud.google.com/go/auth v0.5.1 // indirect
@@ -148,6 +153,7 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/oklog/run v1.1.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.71.0
 	github.com/pelletier/go-toml v1.9.3 // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
@@ -178,7 +184,7 @@ require (
 	google.golang.org/genproto v0.0.0-20240528184218-531527333157 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect
-	google.golang.org/grpc v1.64.0 // indirect
+	google.golang.org/grpc v1.64.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -221,6 +221,8 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
+github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0=
 github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -460,6 +462,8 @@ github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE=
 github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk=
 github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A=
 github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU=
+github.com/oracle/oci-go-sdk/v65 v65.71.0 h1:eEnFD/CzcoqdAA0xu+EmK32kJL3jfV0oLYNWVzoKNyo=
+github.com/oracle/oci-go-sdk/v65 v65.71.0/go.mod h1:IBEV9l1qBzUpo7zgGaRUhbB05BVfcDGYRFBCPlTcPp0=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
@@ -508,6 +512,8 @@ github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9Nz
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg=
+github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
 github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY=
 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
@@ -523,6 +529,8 @@ github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -533,6 +541,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
@@ -755,6 +764,7 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
 golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -935,8 +945,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY=
-google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg=
+google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA=
+google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

pkg/cloud/alibaba/provider.go

@@ -67,7 +67,7 @@ var (
 )
 
 // Variable to keep track of instance families that fail in DescribePrice API due improper defaulting of systemDisk if the information is not available
-var alibabaDefaultToCloudEssd = []string{"g6e", "r6e", "r7", "g7", "g7a", "r7a"}
+var alibabaDefaultToCloudEssd = []string{"g6e", "r6e"}
 
 var alibabaRegions = []string{
 	"cn-qingdao",
@@ -980,9 +980,9 @@ func createDescribePriceACSRequest(i interface{}) (*requests.CommonRequest, erro
 				request.QueryParams["SystemDisk.PerformanceLevel"] = node.SystemDisk.PerformanceLevel
 			}
 		} else {
-			// When System Disk information is not available for instance family g6e, r7 and r6e the defaults in
-			// DescribePrice dont default rightly to cloud_essd for these instances.
-			if slices.Contains(alibabaDefaultToCloudEssd, node.InstanceTypeFamily) {
+			// When the system disk information is not available, and the instance family is g6e or r6e,
+			// or the instance generation is 6 or above, the default disk category in DescribePrice should be cloud_essd.
+			if slices.Contains(alibabaDefaultToCloudEssd, node.InstanceTypeFamily) || getInstanceFamilyGenerationFromType(node.InstanceType) > 6 {
 				request.QueryParams["SystemDisk.Category"] = ALIBABA_DISK_CLOUD_ESSD_CATEGORY
 			}
 		}
@@ -1148,6 +1148,26 @@ func getInstanceFamilyFromType(instanceType string) string {
 	return splitinstanceType[1]
 }
 
+// This function is used to obtain the generation of the instance family from the InstanceType,
+// because when the generation is higher than or equal to 7, the instance disk type will not support cloud_efficiency.
+// In such cases, when calling the DescribePrice interface, the system disk type will default to cloud_essd.
+func getInstanceFamilyGenerationFromType(instanceType string) int {
+	// FamilyName format: g7ne or g7 or r7 or r6e,
+	familyName := getInstanceFamilyFromType(instanceType)
+	re := regexp.MustCompile(`(\d+)`)
+	match := re.FindString(familyName)
+	if match != "" {
+		generation, err := strconv.Atoi(match)
+		if err != nil {
+			log.Errorf("unable to convert the generation of the instance type %s to integer", instanceType)
+		} else {
+			return generation
+		}
+	}
+	log.Warnf("unable to find the generation of the instance type %s,", instanceType)
+	return -1
+}
+
 // getInstanceIDFromProviderID returns the instance ID associated with the Node. A *v1.Node providerID in Alibaba cloud
 // is of <REGION-ID>.<INSTANCE-ID>. This function returns the Instance ID for the given ProviderID. if its unable to interpret
 // it defaults to empty string.

pkg/cloud/alibaba/provider_test.go

@@ -112,6 +112,20 @@ func TestProcessDescribePriceAndCreateAlibabaPricing(t *testing.T) {
 			},
 			expectedError: nil,
 		},
+		{
+			name: "test General Purpose Type g8a instance family",
+			teststruct: &SlimK8sNode{
+				InstanceType:       "ecs.g8a.8xlarge",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "33554432KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "cn-hangzhou.i-test-01c",
+				InstanceTypeFamily: "g8a",
+			},
+			expectedError: nil,
+		},
 		{
 			name: "test Enhanced General Purpose Type g6e instance family",
 			teststruct: &SlimK8sNode{
@@ -856,3 +870,108 @@ func TestDeterminePVRegion(t *testing.T) {
 	}
 
 }
+
+func TestGetInstanceFamilyGenerationFromType(t *testing.T) {
+	cases := []struct {
+		name                             string
+		instanceType                     string
+		expectedInstanceFamilyGeneration int
+	}{
+		{
+			name:                             "test if ecs.[instance-family].[different-type] work",
+			instanceType:                     "ecs.sn2ne.2xlarge",
+			expectedInstanceFamilyGeneration: 2,
+		},
+		{
+			name:                             "test if ecs.[instance-family].[different-type] work",
+			instanceType:                     "ecs.g7.large",
+			expectedInstanceFamilyGeneration: 7,
+		},
+		{
+			name:                             "test if random word gives you ALIBABA_UNKNOWN_INSTANCE_FAMILY_TYPE value ",
+			instanceType:                     "random.value",
+			expectedInstanceFamilyGeneration: -1,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			returnValue := getInstanceFamilyGenerationFromType(c.instanceType)
+			if returnValue != c.expectedInstanceFamilyGeneration {
+				t.Fatalf("Case name %s: expected instance family generation of type %d but got %d", c.name, c.expectedInstanceFamilyGeneration, returnValue)
+			}
+		})
+	}
+}
+
+func TestCreateDescribeNodePriceACSRequest(t *testing.T) {
+
+	cases := []struct {
+		name                 string
+		testStruct           interface{}
+		expectedError        error
+		expectedDiskCategory string
+	}{
+		{
+			// Test case for instance type ecs.g6.large
+			name: "test request parma when instance type is ecs.g6.large",
+			testStruct: &SlimK8sNode{
+				InstanceType:       "ecs.g6.large",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "16KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "Ali-XXX-node-01",
+				InstanceTypeFamily: "g6",
+			},
+			expectedError:        nil,
+			expectedDiskCategory: "",
+		},
+		{
+			// Test case for instance type ecs.g7.large
+			name: "test request parma when instance type is ecs.g7.large",
+			testStruct: &SlimK8sNode{
+				InstanceType:       "ecs.g7.large",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "16KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "Ali-XXX-node-02",
+				InstanceTypeFamily: "g7",
+			},
+			expectedError:        nil,
+			expectedDiskCategory: ALIBABA_DISK_CLOUD_ESSD_CATEGORY,
+		},
+		{
+			// Test case for instance type ecs.g7.large, this instance type is in 'alibabaDefaultToCloudEssd'
+			name: "test request parma when instance type is ecs.g6e.large",
+			testStruct: &SlimK8sNode{
+				InstanceType:       "ecs.g6e.large",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "16KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "Ali-XXX-node-03",
+				InstanceTypeFamily: "g6e",
+			},
+			expectedError:        nil,
+			expectedDiskCategory: ALIBABA_DISK_CLOUD_ESSD_CATEGORY,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			req, err := createDescribePriceACSRequest(c.testStruct)
+			t.Logf("Request Params SystemDisk.Category: %v", req.QueryParams["SystemDisk.Category"])
+			if err != nil && c.expectedError != nil {
+				t.Fatalf("Case name %s: Error converting to Alibaba cloud request", c.name)
+			}
+			if c.expectedDiskCategory != req.QueryParams["SystemDisk.Category"] {
+				t.Fatalf("Case name %s: Disk Category is not set correctly", c.name)
+			}
+		})
+	}
+}

pkg/cloud/aws/athenaintegration.go

@@ -14,6 +14,8 @@ import (
 )
 
 const LabelColumnPrefix = "resource_tags_user_"
+const AWSLabelColumnPrefix = "resource_tags_aws_"
+const AthenaResourceTagPrefix = "resource_tags_"
 
 // athenaDateLayout is the default AWS date format
 const AthenaDateLayout = "2006-01-02 15:04:05.000"
@@ -52,6 +54,7 @@ type AthenaQueryIndexes struct {
 	Query                  string
 	ColumnIndexes          map[string]int
 	TagColumns             []string
+	AWSTagColumns          []string
 	ListCostColumn         string
 	NetCostColumn          string
 	AmortizedNetCostColumn string
@@ -80,6 +83,8 @@ func (ai *AthenaIntegration) GetCloudCost(start, end time.Time) (*opencost.Cloud
 		"line_item_usage_account_id",
 		"line_item_product_code",
 		"line_item_usage_type",
+		"product_region_code",
+		"line_item_availability_zone",
 	}
 
 	// Create query indices
@@ -98,6 +103,10 @@ func (ai *AthenaIntegration) GetCloudCost(start, end time.Time) (*opencost.Cloud
 			groupByColumns = append(groupByColumns, quotedTag)
 			aqi.TagColumns = append(aqi.TagColumns, quotedTag)
 		}
+		if strings.HasPrefix(column, AWSLabelColumnPrefix) {
+			groupByColumns = append(groupByColumns, column)
+			aqi.AWSTagColumns = append(aqi.AWSTagColumns, column)
+		}
 	}
 	var selectColumns []string
 
@@ -333,7 +342,6 @@ func (ai *AthenaIntegration) RowToCloudCost(row types.Row, aqi AthenaQueryIndexe
 	// Iterate through the slice of tag columns, assigning
 	// values to the column names, minus the tag prefix.
 	labels := opencost.CloudCostLabels{}
-	labelValues := []string{}
 	for _, tagColumnName := range aqi.TagColumns {
 		// remove quotes
 		labelName := strings.TrimPrefix(tagColumnName, `"`)
@@ -343,7 +351,15 @@ func (ai *AthenaIntegration) RowToCloudCost(row types.Row, aqi AthenaQueryIndexe
 		value := GetAthenaRowValue(row, aqi.ColumnIndexes, tagColumnName)
 		if value != "" {
 			labels[labelName] = value
-			labelValues = append(labelValues, value)
+		}
+	}
+
+	for _, awsColumnName := range aqi.AWSTagColumns {
+		// partially remove prefix leaving "aws_"
+		labelName := strings.TrimPrefix(awsColumnName, AthenaResourceTagPrefix)
+		value := GetAthenaRowValue(row, aqi.ColumnIndexes, awsColumnName)
+		if value != "" {
+			labels[labelName] = value
 		}
 	}
 
@@ -353,6 +369,8 @@ func (ai *AthenaIntegration) RowToCloudCost(row types.Row, aqi AthenaQueryIndexe
 	providerID := GetAthenaRowValue(row, aqi.ColumnIndexes, "line_item_resource_id")
 	productCode := GetAthenaRowValue(row, aqi.ColumnIndexes, "line_item_product_code")
 	usageType := GetAthenaRowValue(row, aqi.ColumnIndexes, "line_item_usage_type")
+	regionCode := GetAthenaRowValue(row, aqi.ColumnIndexes, "product_region_code")
+	availabilityZone := GetAthenaRowValue(row, aqi.ColumnIndexes, "line_item_availability_zone")
 	isK8s, _ := strconv.ParseBool(GetAthenaRowValue(row, aqi.ColumnIndexes, aqi.IsK8sColumn))
 	k8sPct := 0.0
 	if isK8s {
@@ -396,13 +414,17 @@ func (ai *AthenaIntegration) RowToCloudCost(row types.Row, aqi AthenaQueryIndexe
 	}
 
 	properties := opencost.CloudCostProperties{
-		ProviderID:      providerID,
-		Provider:        opencost.AWSProvider,
-		AccountID:       accountID,
-		InvoiceEntityID: invoiceEntityID,
-		Service:         productCode,
-		Category:        category,
-		Labels:          labels,
+		ProviderID:        providerID,
+		Provider:          opencost.AWSProvider,
+		AccountID:         accountID,
+		AccountName:       accountID,
+		InvoiceEntityID:   invoiceEntityID,
+		InvoiceEntityName: invoiceEntityID,
+		RegionID:          regionCode,
+		AvailabilityZone:  availabilityZone,
+		Service:           productCode,
+		Category:          category,
+		Labels:            labels,
 	}
 
 	start, err := time.Parse(AthenaDateLayout, startStr)

pkg/cloud/aws/s3selectintegration.go

@@ -7,7 +7,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/opencost/opencost/core/pkg/log"
 	"github.com/opencost/opencost/core/pkg/opencost"
 )
@@ -15,13 +14,15 @@ import (
 const S3SelectDateLayout = "2006-01-02T15:04:05Z"
 
 // S3Object is aliased as "s" in queries
-const S3SelectAccountID = `s."bill/PayerAccountId"`
-
+const S3SelectBillPayerAccountID = `s."bill/PayerAccountId"`
+const S3SelectAccountID = `s."lineItem/UsageAccountId"`
 const S3SelectItemType = `s."lineItem/LineItemType"`
 const S3SelectStartDate = `s."lineItem/UsageStartDate"`
 const S3SelectProductCode = `s."lineItem/ProductCode"`
 const S3SelectResourceID = `s."lineItem/ResourceId"`
 const S3SelectUsageType = `s."lineItem/UsageType"`
+const S3SelectRegionCode = `s."product/regionCode"`
+const S3SelectAvailabilityZone = `s."lineItem/AvailabilityZone"`
 
 const S3SelectListCost = `s."lineItem/UnblendedCost"`
 const S3SelectNetCost = `s."lineItem/NetUnblendedCost"`
@@ -29,6 +30,12 @@ const S3SelectNetCost = `s."lineItem/NetUnblendedCost"`
 // These two may be used for Amortized<Net>Cost
 const S3SelectRICost = `s."reservation/EffectiveCost"`
 const S3SelectSPCost = `s."savingsPlan/SavingsPlanEffectiveCost"`
+const S3SelectNetRICost = `s."reservation/NetEffectiveCost"`
+const S3SelectNetSPCost = `s."savingsPlan/NetSavingsPlanEffectiveCost"`
+
+const S3SelectUserLabelPrefix = "resourceTags/user:"
+const S3SelectAWSLabelPrefix = "resourceTags/aws:"
+const S3SelectResourceTagsPrefix = "resourceTags/"
 
 type S3SelectIntegration struct {
 	S3SelectQuerier
@@ -44,15 +51,6 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 		opencost.NewWindow(&start, &end).String(),
 	)
 
-	// Set midnight yesterday as last point in time reconciliation data
-	// can be pulled from to ensure complete days of data
-	midnightYesterday := time.Now().In(
-		time.UTC,
-	).Truncate(time.Hour*24).AddDate(0, 0, -1)
-	if end.After(midnightYesterday) {
-		end = midnightYesterday
-	}
-
 	// ccsr to populate with cloudcosts.
 	ccsr, err := opencost.NewCloudCostSetRange(
 		start,
@@ -74,40 +72,71 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 		return nil, err
 	}
 	// Acquire headers
-	headers, err := s3si.GetHeaders(queryKeys, client)
+	headers, err := s3si.GetHeaders(queryKeys[0], client)
 	if err != nil {
 		return nil, err
 	}
-	// Exactly what it says on the tin. Though is there a set equivalent
-	// in Go? This seems like a good use case for that.
-	allColumns := map[string]bool{}
+
+	allColumns := map[string]struct{}{}
 	for _, header := range headers {
-		allColumns[header] = true
+		allColumns[header] = struct{}{}
 	}
 
 	formattedStart := start.Format("2006-01-02")
 	formattedEnd := end.Format("2006-01-02")
 	selectColumns := []string{
 		S3SelectStartDate,
+		S3SelectBillPayerAccountID,
 		S3SelectAccountID,
 		S3SelectResourceID,
 		S3SelectItemType,
 		S3SelectProductCode,
 		S3SelectUsageType,
+		S3SelectRegionCode,
+		S3SelectAvailabilityZone,
 		S3SelectListCost,
 	}
-	// OC equivalent to KCM env flags relevant at all?
+	_, checkNet := allColumns[S3SelectNetCost]
+	if checkNet {
+		selectColumns = append(selectColumns, S3SelectNetCost)
+	}
+
 	// Check for Reservation columns in CUR and query if available
-	checkReservations := allColumns[S3SelectRICost]
+	_, checkReservations := allColumns[S3SelectRICost]
 	if checkReservations {
 		selectColumns = append(selectColumns, S3SelectRICost)
 	}
+	_, checkNetReservations := allColumns[S3SelectNetRICost]
+	if checkNetReservations {
+		selectColumns = append(selectColumns, S3SelectNetRICost)
+	}
 
 	// Check for Savings Plan Columns in CUR and query if available
-	checkSavingsPlan := allColumns[S3SelectSPCost]
+	_, checkSavingsPlan := allColumns[S3SelectSPCost]
 	if checkSavingsPlan {
 		selectColumns = append(selectColumns, S3SelectSPCost)
 	}
+	_, checkNetSavingsPlan := allColumns[S3SelectNetSPCost]
+	if checkNetSavingsPlan {
+		selectColumns = append(selectColumns, S3SelectNetSPCost)
+	}
+
+	// Determine which columns are user-defined tags and add those to the list
+	// of columns to query.
+	labelColumns := []string{}
+	awsLabelColumns := []string{}
+	for column := range allColumns {
+		if strings.HasPrefix(column, S3SelectUserLabelPrefix) {
+			quotedTag := fmt.Sprintf(`s."%s"`, column)
+			selectColumns = append(selectColumns, quotedTag)
+			labelColumns = append(labelColumns, quotedTag)
+		}
+		if strings.HasPrefix(column, S3SelectAWSLabelPrefix) {
+			quotedTag := fmt.Sprintf(`s."%s"`, column)
+			selectColumns = append(selectColumns, quotedTag)
+			awsLabelColumns = append(awsLabelColumns, quotedTag)
+		}
+	}
 
 	// Build map of query columns to use for parsing query
 	columnIndexes := map[string]int{}
@@ -118,17 +147,8 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 	selectStr := strings.Join(selectColumns, ", ")
 	queryStr := `SELECT %s FROM s3object s
 	WHERE (CAST(s."lineItem/UsageStartDate" AS TIMESTAMP) BETWEEN CAST('%s' AS TIMESTAMP) AND CAST('%s' AS TIMESTAMP))
-	AND s."lineItem/ResourceId" <> ''
-	AND (
-		(
-			s."lineItem/ProductCode" = 'AmazonEC2' AND (
-				SUBSTRING(s."lineItem/ResourceId",1,2) = 'i-'
-				OR SUBSTRING(s."lineItem/ResourceId",1,4) = 'vol-'
-			)
-		)
-		OR s."lineItem/ProductCode" = 'AWSELB'
-       OR s."lineItem/ProductCode" = 'AmazonFSx'
-	)`
+	AND (s."lineItem/LineItemType" = 'Usage' OR s."lineItem/LineItemType" = 'DiscountedUsage' OR s."lineItem/LineItemType" = 'SavingsPlanCoveredUsage' OR s."lineItem/LineItemType" = 'EdpDiscount' OR s."lineItem/LineItemType" = 'PrivateRateDiscount')
+	`
 	query := fmt.Sprintf(queryStr, selectStr, formattedStart, formattedEnd)
 
 	processResults := func(reader *csv.Reader) error {
@@ -143,45 +163,107 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 			}
 
 			startStr := GetCSVRowValue(row, columnIndexes, S3SelectStartDate)
+			billPayerAccountID := GetCSVRowValue(row, columnIndexes, S3SelectBillPayerAccountID)
 			itemAccountID := GetCSVRowValue(row, columnIndexes, S3SelectAccountID)
 			itemProviderID := GetCSVRowValue(row, columnIndexes, S3SelectResourceID)
 			lineItemType := GetCSVRowValue(row, columnIndexes, S3SelectItemType)
 			itemProductCode := GetCSVRowValue(row, columnIndexes, S3SelectProductCode)
 			usageType := GetCSVRowValue(row, columnIndexes, S3SelectUsageType)
+			regionCode := GetCSVRowValue(row, columnIndexes, S3SelectRegionCode)
+			availabilityZone := GetCSVRowValue(row, columnIndexes, S3SelectAvailabilityZone)
+
+			// Iterate through the slice of tag columns, assigning
+			// values to the column names, minus the tag prefix.
+			labels := opencost.CloudCostLabels{}
+			for _, labelColumnName := range labelColumns {
+				// remove quotes
+				labelName := strings.TrimPrefix(labelColumnName, `s."`)
+				labelName = strings.TrimSuffix(labelName, `"`)
+				// remove prefix
+				labelName = strings.TrimPrefix(labelName, S3SelectUserLabelPrefix)
+				value := GetCSVRowValue(row, columnIndexes, labelColumnName)
+				if value != "" {
+					labels[labelName] = value
+				}
+			}
+			for _, awsLabelColumnName := range awsLabelColumns {
+				// remove quotes
+				labelName := strings.TrimPrefix(awsLabelColumnName, `s."`)
+				labelName = strings.TrimSuffix(labelName, `"`)
+				// partially remove prefix leaving "aws:"
+				labelName = strings.TrimPrefix(labelName, S3SelectResourceTagsPrefix)
+				value := GetCSVRowValue(row, columnIndexes, awsLabelColumnName)
+				if value != "" {
+					labels[labelName] = value
+				}
+			}
+
+			isKubernetes := 0.0
+			if itemProductCode == "AmazonEKS" || hasK8sLabel(labels) {
+				isKubernetes = 1.0
+			}
 
 			var (
-				amortizedCost float64
-				listCost      float64
-				netCost       float64
+				amortizedCost    float64
+				amortizedNetCost float64
+				listCost         float64
+				netCost          float64
 			)
 			// Get list and net costs
-			listCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectListCost)
-			if err != nil {
-				return err
+			if lineItemType != "EdpDiscount" && lineItemType != "PrivateRateDiscount" {
+				listCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectListCost)
+				if err != nil {
+					return err
+				}
 			}
-			netCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectNetCost)
-			if err != nil {
-				return err
+
+			// Get net cost if available
+			netCost = listCost
+			if checkNet {
+				netCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectNetCost)
+				if err != nil {
+					return err
+				}
 			}
 
 			// If there is a reservation_reservation_a_r_n on the line item use the awsRIPricingSUMColumn as cost
-			if checkReservations && lineItemType == "DiscountedUsage" {
-				amortizedCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectRICost)
-				if err != nil {
-					log.Errorf(err.Error())
-					continue
+			amortizedCost = listCost
+			amortizedNetCost = listCost
+			if lineItemType == "DiscountedUsage" {
+				if checkReservations {
+					amortizedCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectRICost)
+					if err != nil {
+						log.Errorf(err.Error())
+						continue
+					}
+					amortizedNetCost = amortizedCost
+				}
+				if checkNetReservations {
+					amortizedNetCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectNetRICost)
+					if err != nil {
+						log.Errorf(err.Error())
+						continue
+					}
 				}
 				// If there is a lineItemType of SavingsPlanCoveredUsage use the awsSPPricingSUMColumn
-			} else if checkSavingsPlan && lineItemType == "SavingsPlanCoveredUsage" {
-				amortizedCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectSPCost)
-				if err != nil {
-					log.Errorf(err.Error())
-					continue
+			} else if lineItemType == "SavingsPlanCoveredUsage" {
+				if checkSavingsPlan {
+					amortizedCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectSPCost)
+					if err != nil {
+						log.Errorf(err.Error())
+						continue
+					}
+					amortizedNetCost = amortizedCost
+				}
+				if checkNetSavingsPlan {
+					amortizedNetCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectNetSPCost)
+					if err != nil {
+						log.Errorf(err.Error())
+						continue
+					}
 				}
-			} else {
-				// Default to listCost
-				amortizedCost = listCost
 			}
+
 			category := SelectAWSCategory(itemProviderID, usageType, itemProductCode)
 			// Retrieve final stanza of product code for ProviderID
 			if itemProductCode == "AWSELB" || itemProductCode == "AmazonFSx" {
@@ -190,10 +272,16 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 
 			properties := opencost.CloudCostProperties{}
 			properties.Provider = opencost.AWSProvider
+			properties.InvoiceEntityID = billPayerAccountID
+			properties.InvoiceEntityName = billPayerAccountID
 			properties.AccountID = itemAccountID
+			properties.AccountName = itemAccountID
 			properties.Category = category
 			properties.Service = itemProductCode
 			properties.ProviderID = itemProviderID
+			properties.RegionID = regionCode
+			properties.AvailabilityZone = availabilityZone
+			properties.Labels = labels
 
 			itemStart, err := time.Parse(S3SelectDateLayout, startStr)
 			if err != nil {
@@ -211,19 +299,24 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 				Properties: &properties,
 				Window:     opencost.NewWindow(&itemStart, &itemEnd),
 				ListCost: opencost.CostMetric{
-					Cost: listCost,
+					Cost:              listCost,
+					KubernetesPercent: isKubernetes,
 				},
 				NetCost: opencost.CostMetric{
-					Cost: netCost,
+					Cost:              netCost,
+					KubernetesPercent: isKubernetes,
 				},
 				AmortizedNetCost: opencost.CostMetric{
-					Cost: amortizedCost,
+					Cost:              amortizedCost,
+					KubernetesPercent: isKubernetes,
 				},
 				AmortizedCost: opencost.CostMetric{
-					Cost: amortizedCost,
+					Cost:              amortizedNetCost,
+					KubernetesPercent: isKubernetes,
 				},
 				InvoicedCost: opencost.CostMetric{
-					Cost: netCost,
+					Cost:              netCost,
+					KubernetesPercent: isKubernetes,
 				},
 			}
 			ccsr.LoadCloudCost(cc)
@@ -237,25 +330,34 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 	return ccsr, nil
 }
 
-func (s3si *S3SelectIntegration) GetHeaders(queryKeys []string, client *s3.Client) ([]string, error) {
-	// Query to grab only header line from file
-	query := "SELECT * FROM S3OBJECT LIMIT 1"
-	var record []string
+const (
+	TagAWSEKSClusterName     = "aws:eks:cluster-name"
+	TagEKSClusterName        = "eks:cluster-name"
+	TagEKSCtlClusterName     = "alpha.eksctl.io/cluster-name"
+	TagKubernetesServiceName = "kubernetes.io/service-name"
+	TagKubernetesPVCName     = "kubernetes.io/created-for/pvc/name"
+	TagKubernetesPVName      = "kubernetes.io/created-for/pv/name"
+)
 
-	proccessheaders := func(reader *csv.Reader) error {
-		var err error
-		record, err = reader.Read()
-		if err != nil {
-			return err
-		}
-		return nil
+// hsK8sLabel checks if the labels contain a k8s label
+func hasK8sLabel(labels opencost.CloudCostLabels) bool {
+	if _, ok := labels[TagAWSEKSClusterName]; ok {
+		return true
 	}
-
-	// Use only the first query key with assumption that files share schema
-	err := s3si.Query(query, []string{queryKeys[0]}, client, proccessheaders)
-	if err != nil {
-		return nil, err
+	if _, ok := labels[TagEKSClusterName]; ok {
+		return true
 	}
-
-	return record, nil
+	if _, ok := labels[TagEKSCtlClusterName]; ok {
+		return true
+	}
+	if _, ok := labels[TagKubernetesServiceName]; ok {
+		return true
+	}
+	if _, ok := labels[TagKubernetesPVCName]; ok {
+		return true
+	}
+	if _, ok := labels[TagKubernetesPVName]; ok {
+		return true
+	}
+	return false
 }

pkg/cloud/aws/s3selectquerier.go

@@ -45,6 +45,18 @@ func (s3sq *S3SelectQuerier) Query(query string, queryKeys []string, cli *s3.Cli
 	return nil
 }
 
+func (s3sq *S3SelectQuerier) GetHeaders(queryKey string, cli *s3.Client) ([]string, error) {
+	reader, err := s3sq.fetchCSVReader("SELECT * FROM S3Object LIMIT 1", queryKey, cli, s3Types.FileHeaderInfoNone)
+	if err != nil {
+		return nil, err
+	}
+	record, err := reader.Read()
+	if err != nil {
+		return nil, err
+	}
+	return record, nil
+}
+
 // GetQueryKeys returns a list of s3 object names, where the there are 1 object for each month within the range between
 // start and end
 func (s3sq *S3SelectQuerier) GetQueryKeys(start, end time.Time, client *s3.Client) ([]string, error) {

pkg/cloud/azure/azurestorageintegration.go

@@ -34,13 +34,16 @@ func (asi *AzureStorageIntegration) GetCloudCost(start, end time.Time) (*opencos
 		// until we can revisit and spend the time to do the calculations correctly
 		cc := &opencost.CloudCost{
 			Properties: &opencost.CloudCostProperties{
-				ProviderID:      providerID,
-				Provider:        opencost.AzureProvider,
-				AccountID:       abv.SubscriptionID,
-				InvoiceEntityID: abv.InvoiceEntityID,
-				Service:         abv.Service,
-				Category:        SelectAzureCategory(abv.MeterCategory),
-				Labels:          abv.Tags,
+				ProviderID:        providerID,
+				Provider:          opencost.AzureProvider,
+				AccountID:         abv.SubscriptionID,
+				AccountName:       abv.SubscriptionName,
+				InvoiceEntityID:   abv.InvoiceEntityID,
+				InvoiceEntityName: abv.InvoiceEntityName,
+				RegionID:          abv.Region,
+				Service:           abv.Service,
+				Category:          SelectAzureCategory(abv.MeterCategory),
+				Labels:            abv.Tags,
 			},
 			Window: window,
 			AmortizedNetCost: opencost.CostMetric{

pkg/cloud/azure/billingexportparser.go

@@ -19,16 +19,19 @@ var groupRegex = regexp.MustCompile("(/[^/]+)")
 
 // BillingRowValues holder for Azure Billing Values
 type BillingRowValues struct {
-	Date            time.Time
-	MeterCategory   string
-	SubscriptionID  string
-	InvoiceEntityID string
-	InstanceID      string
-	Service         string
-	Tags            map[string]string
-	AdditionalInfo  map[string]any
-	Cost            float64
-	NetCost         float64
+	Date              time.Time
+	MeterCategory     string
+	SubscriptionID    string
+	SubscriptionName  string
+	InvoiceEntityID   string
+	InvoiceEntityName string
+	Region            string
+	InstanceID        string
+	Service           string
+	Tags              map[string]string
+	AdditionalInfo    map[string]any
+	Cost              float64
+	NetCost           float64
 }
 
 func (brv *BillingRowValues) IsCompute(category string) bool {
@@ -52,17 +55,20 @@ func (brv *BillingRowValues) IsCompute(category string) bool {
 
 // BillingExportParser holds indexes of relevent fields in Azure Billing CSV in addition to the correct data format
 type BillingExportParser struct {
-	Date            int
-	MeterCategory   int
-	InvoiceEntityID int
-	SubscriptionID  int
-	InstanceID      int
-	Service         int
-	Tags            int
-	AdditionalInfo  int
-	Cost            int
-	NetCost         int
-	DateFormat      string
+	Date              int
+	MeterCategory     int
+	InvoiceEntityID   int
+	InvoiceEntityName int
+	SubscriptionID    int
+	SubscriptionName  int
+	Region            int
+	InstanceID        int
+	Service           int
+	Tags              int
+	AdditionalInfo    int
+	Cost              int
+	NetCost           int
+	DateFormat        string
 }
 
 // match "SubscriptionGuid" in "Abonnement-GUID (SubscriptionGuid)"
@@ -106,6 +112,14 @@ func NewBillingParseSchema(headers []string) (*BillingExportParser, error) {
 		return nil, fmt.Errorf("NewBillingParseSchema: failed to find Subscription ID field")
 	}
 
+	// set Subscription Name
+	if i, ok := headerIndexes["subscriptionname"]; ok {
+		abp.SubscriptionName = i
+	} else {
+		// if no subscription name column use subscriptionID column
+		abp.SubscriptionName = abp.SubscriptionID
+	}
+
 	// Set Billing ID
 	if i, ok := headerIndexes["billingaccountid"]; ok {
 		abp.InvoiceEntityID = i
@@ -116,6 +130,25 @@ func NewBillingParseSchema(headers []string) (*BillingExportParser, error) {
 		abp.InvoiceEntityID = abp.SubscriptionID
 	}
 
+	// Set Billing Account Name
+	if i, ok := headerIndexes["billingaccountname"]; ok {
+		abp.InvoiceEntityName = i
+	} else {
+		// if no billing name column is present use billing ID index
+		abp.InvoiceEntityName = abp.InvoiceEntityID
+	}
+
+	// Set Region
+	if i, ok := headerIndexes["resourcelocation"]; ok {
+		abp.Region = i
+	} else if j, ok2 := headerIndexes["meterregion"]; ok2 {
+		abp.Region = j
+	} else if k, ok3 := headerIndexes["location"]; ok3 {
+		abp.Region = k
+	} else {
+		return nil, fmt.Errorf("NewBillingParseSchema: failed to find Region field")
+	}
+
 	// Set Instance ID
 	if i, ok := headerIndexes["instanceid"]; ok {
 		abp.InstanceID = i
@@ -237,16 +270,19 @@ func (bep *BillingExportParser) ParseRow(start, end time.Time, record []string)
 	}
 
 	return &BillingRowValues{
-		Date:            usageDate,
-		MeterCategory:   record[bep.MeterCategory],
-		SubscriptionID:  record[bep.SubscriptionID],
-		InvoiceEntityID: record[bep.InvoiceEntityID],
-		InstanceID:      record[bep.InstanceID],
-		Service:         record[bep.Service],
-		Tags:            tags,
-		AdditionalInfo:  additionalInfo,
-		Cost:            cost,
-		NetCost:         netCost,
+		Date:              usageDate,
+		MeterCategory:     record[bep.MeterCategory],
+		SubscriptionID:    record[bep.SubscriptionID],
+		SubscriptionName:  record[bep.SubscriptionName],
+		InvoiceEntityID:   record[bep.InvoiceEntityID],
+		InvoiceEntityName: record[bep.InvoiceEntityName],
+		Region:            record[bep.Region],
+		InstanceID:        record[bep.InstanceID],
+		Service:           record[bep.Service],
+		Tags:              tags,
+		AdditionalInfo:    additionalInfo,
+		Cost:              cost,
+		NetCost:           netCost,
 	}
 }
 

pkg/cloud/azure/provider.go

@@ -846,7 +846,12 @@ func (az *Azure) DownloadPricingData() error {
 	rateCardFilter := fmt.Sprintf("OfferDurableId eq '%s' and Currency eq '%s' and Locale eq 'en-US' and RegionInfo eq '%s'", config.AzureOfferDurableID, config.CurrencyCode, config.AzureBillingRegion)
 
 	log.Infof("Using ratecard query %s", rateCardFilter)
-	result, err := rcClient.Get(context.TODO(), rateCardFilter)
+	// rate-card client is old, it can hang indefinitely in some cases
+	// this happens on the main thread, so it may block the whole app
+	// there is can be a better way to set timeout for the client
+	ctx, cancel := context.WithTimeout(context.TODO(), 60*time.Second)
+	defer cancel()
+	result, err := rcClient.Get(ctx, rateCardFilter)
 	if err != nil {
 		log.Warnf("Error in pricing download query from API")
 		az.rateCardPricingError = err

pkg/cloud/azure/resources/billingexports/values/MissingBrackets.csv

@@ -1,2 +1,2 @@
-subscriptionid,billingaccountid,UsageDateTime,MeterCategory,costinbillingcurrency,paygcostinbillingcurrency,ConsumedService,InstanceId,Tags,AdditionalInfo
-11111111-12ab-34dc-56ef-123456abcdef,11111111-12ab-34dc-56ef-123456abcdef,2021-02-01,Virtual Machines,4,5,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss""","""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": ""aks-nodepool1-12345678-vmss_0"",  ""VCPUs"": 2"
+subscriptionid,billingaccountid,UsageDateTime,MeterCategory,costinbillingcurrency,paygcostinbillingcurrency,ConsumedService,InstanceId,Tags,AdditionalInfo,resourcelocation
+11111111-12ab-34dc-56ef-123456abcdef,11111111-12ab-34dc-56ef-123456abcdef,2021-02-01,Virtual Machines,4,5,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss""","""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": ""aks-nodepool1-12345678-vmss_0"",  ""VCPUs"": 2",""

pkg/cloud/azure/resources/billingexports/values/Template.csv

@@ -1,4 +1,4 @@
-subscriptionid,billingaccountid,UsageDateTime,MeterCategory,costinbillingcurrency,paygcostinbillingcurrency,ConsumedService,InstanceId,Tags,AdditionalInfo
+subscriptionid,billingaccountid,UsageDateTime,MeterCategory,costinbillingcurrency,paygcostinbillingcurrency,ConsumedService,InstanceId,Tags,AdditionalInfo,resourcelocation
 11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Load Balancer,0.075,0.075,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/loadBalancers/kubernetes,,
 11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Virtual Machines,3.504,3.504,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"{""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss"",""orchestrator"":""Kubernetes:1.15.7"",""poolName"":""nodepool1""}","{  ""UsageType"": ""ComputeHR"",  ""ImageType"": ""Canonical"",  ""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": ""aks-nodepool1-12345678-vmss_0"",  ""VMProperties"": ""Microsoft.AKS.Compute.AKS.Linux.Billing"",  ""VCPUs"": 2,  ""CPUs"": 0}"
 11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.0000045,0.0000045,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd03,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-pushgateway"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd03"",""created-by"":""kubernetes-azure-dd""}",

pkg/cloud/azure/resources/billingexports/values/VirtualMachine.csv

@@ -1,2 +1,2 @@
-subscriptionid,billingaccountid,UsageDateTime,MeterCategory,costinbillingcurrency,paygcostinbillingcurrency,ConsumedService,InstanceId,Tags,AdditionalInfo
-11111111-12ab-34dc-56ef-123456abcdef,11111111-12ab-34dc-56ef-123456billing,2021-02-01,Virtual Machines,4,5,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"{""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss""}","{ ""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": ""aks-nodepool1-12345678-vmss_0"",  ""VCPUs"": 2  }"
+subscriptionid,billingaccountid,UsageDateTime,MeterCategory,costinbillingcurrency,paygcostinbillingcurrency,ConsumedService,InstanceId,Tags,AdditionalInfo,resourcelocation
+11111111-12ab-34dc-56ef-123456abcdef,11111111-12ab-34dc-56ef-123456billing,2021-02-01,Virtual Machines,4,5,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"{""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss""}","{ ""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": ""aks-nodepool1-12345678-vmss_0"",  ""VCPUs"": 2  }",""

pkg/cloud/azure/storagebillingparser.go

@@ -69,7 +69,8 @@ func (asbp *AzureStorageBillingParser) ParseBillingData(start, end time.Time, re
 		for _, blob := range blobInfos {
 			blobName := *blob.Name
 
-			localFilePath := filepath.Join(localPath, filepath.Base(blobName))
+			// Use entire blob name to prevent collision with other files from previous months or other integrations (ex "part_0_0001.csv")
+			localFilePath := filepath.Join(localPath, strings.ReplaceAll(blobName, "/", "_"))
 
 			err := asbp.DownloadBlobToFile(localFilePath, blob, client, ctx)
 			if err != nil {
@@ -232,7 +233,7 @@ func (asbp *AzureStorageBillingParser) getMostRecentBlobs(start, end time.Time,
 		if err != nil {
 			return nil, fmt.Errorf("failed to retrieve manifest %w", err)
 		}
-		
+
 		var manifest manifestJson
 		err = json.Unmarshal(manifestBytes, &manifest)
 		if err != nil {
@@ -294,38 +295,3 @@ func (asbp *AzureStorageBillingParser) timeToMonthString(input time.Time) string
 	endOfMonth := input.AddDate(0, 1, -input.Day())
 	return startOfMonth.Format(format) + "-" + endOfMonth.Format(format)
 }
-
-// deleteFilesOlderThan7d recursively walks the directory specified and deletes
-// files which have not been modified in the last 7 days. Returns a list of
-// files deleted.
-func (asbp *AzureStorageBillingParser) deleteFilesOlderThan7d(localPath string) ([]string, error) {
-	duration := 7 * 24 * time.Hour
-	cleaned := []string{}
-	errs := []string{}
-
-	if _, err := os.Stat(localPath); err != nil {
-		return cleaned, nil // localPath does not exist
-	}
-
-	filepath.Walk(localPath, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			errs = append(errs, err.Error())
-			return err
-		}
-
-		if time.Since(info.ModTime()) > duration {
-			err := os.Remove(path)
-			if err != nil {
-				errs = append(errs, err.Error())
-			}
-			cleaned = append(cleaned, path)
-		}
-		return nil
-	})
-
-	if len(errs) == 0 {
-		return cleaned, nil
-	} else {
-		return cleaned, fmt.Errorf("deleteFilesOlderThan7d: %v", errs)
-	}
-}

pkg/cloud/azure/storageconnection.go

@@ -7,6 +7,8 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
+	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
@@ -17,6 +19,7 @@ import (
 // StorageConnection provides access to Azure Storage
 type StorageConnection struct {
 	StorageConfiguration
+	lock             sync.Mutex
 	ConnectionStatus cloud.ConnectionStatus
 }
 
@@ -82,6 +85,9 @@ func (sc *StorageConnection) StreamBlob(blobName string, client *azblob.Client)
 
 // DownloadBlobToFile downloads the Azure Billing CSV to a local file
 func (sc *StorageConnection) DownloadBlobToFile(localFilePath string, blob container.BlobItem, client *azblob.Client, ctx context.Context) error {
+	// Lock to prevent accessing a file which may not be fully downloaded
+	sc.lock.Lock()
+	defer sc.lock.Unlock()
 	blobName := *blob.Name
 	// Check if file already exists
 	if fileInfo, err := os.Stat(localFilePath); err == nil {
@@ -107,12 +113,59 @@ func (sc *StorageConnection) DownloadBlobToFile(localFilePath string, blob conta
 	defer fp.Close()
 
 	// Download newest Azure Billing CSV to disk
+
+	// Time out to prevent deadlock on download
+	timeoutCtx, cancel := context.WithTimeout(ctx, 30*time.Minute)
+	defer cancel()
+
 	log.Infof("CloudCost: Azure: DownloadBlobToFile: retrieving blob: %v", blobName)
-	filesize, err := client.DownloadFile(ctx, sc.Container, blobName, fp, nil)
+	filesize, err := client.DownloadFile(timeoutCtx, sc.Container, blobName, fp, nil)
 	if err != nil {
+		// Clean up file from failed download
+		err2 := os.Remove(localFilePath)
+		if err2 != nil {
+			log.Errorf("CloudCost: Azure: DownloadBlobToFile: failed to remove file %s after failed download %s", localFilePath, err2.Error())
+		}
 		return fmt.Errorf("CloudCost: Azure: DownloadBlobToFile: failed to download %w", err)
 	}
 	log.Infof("CloudCost: Azure: DownloadBlobToFile: retrieved %v of size %dMB", blobName, filesize/1024/1024)
 
 	return nil
 }
+
+// deleteFilesOlderThan7d recursively walks the directory specified and deletes
+// files which have not been modified in the last 7 days. Returns a list of
+// files deleted.
+func (sc *StorageConnection) deleteFilesOlderThan7d(localPath string) ([]string, error) {
+	sc.lock.Lock()
+	defer sc.lock.Unlock()
+	duration := 7 * 24 * time.Hour
+	cleaned := []string{}
+	errs := []string{}
+
+	if _, err := os.Stat(localPath); err != nil {
+		return cleaned, nil // localPath does not exist
+	}
+
+	filepath.Walk(localPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			errs = append(errs, err.Error())
+			return err
+		}
+
+		if time.Since(info.ModTime()) > duration {
+			err := os.Remove(path)
+			if err != nil {
+				errs = append(errs, err.Error())
+			}
+			cleaned = append(cleaned, path)
+		}
+		return nil
+	})
+
+	if len(errs) == 0 {
+		return cleaned, nil
+	} else {
+		return cleaned, fmt.Errorf("deleteFilesOlderThan7d: %v", errs)
+	}
+}

pkg/cloud/config/configurations.go

@@ -10,6 +10,7 @@ import (
 	"github.com/opencost/opencost/pkg/cloud/aws"
 	"github.com/opencost/opencost/pkg/cloud/azure"
 	"github.com/opencost/opencost/pkg/cloud/gcp"
+	"github.com/opencost/opencost/pkg/cloud/oracle"
 )
 
 // MultiCloudConfig struct is used to unmarshal cloud configs for each provider out of cloud-integration file
@@ -66,22 +67,31 @@ type Configurations struct {
 	GCP     *GCPConfigs     `json:"gcp,omitempty"`
 	Azure   *AzureConfigs   `json:"azure,omitempty"`
 	Alibaba *AlibabaConfigs `json:"alibaba,omitempty"`
+	OCI     *OCIConfigs     `json:"oci,omitempty"`
 }
 
 // UnmarshalJSON custom json unmarshalling to maintain support for MultiCloudConfig format
 func (c *Configurations) UnmarshalJSON(bytes []byte) error {
-	// Attempt to unmarshal into old config object
-	multiConfig := &MultiCloudConfig{}
-	err := json.Unmarshal(bytes, multiConfig)
-	// If unmarshal is successful, move values into config and return
-	if err == nil {
-		multiConfig.loadConfigurations(c)
-		return nil
-	}
+	// This has been tested for backwards compatability, and it works in both config formats.
+	// It also coincidentally works if you mix-and-match both the old format and the new
+	// format.
 	// Create inline type to gain access to default Unmarshalling
 	type ConfUnmarshaller *Configurations
 	var conf ConfUnmarshaller = c
-	return json.Unmarshal(bytes, conf)
+	err := json.Unmarshal(bytes, conf)
+	// If unmarshal is successful, return
+	if err == nil {
+		return nil
+	}
+
+	// Attempt to unmarshal into old config object
+	multiConfig := &MultiCloudConfig{}
+	err = json.Unmarshal(bytes, multiConfig)
+	if err != nil {
+		return err
+	}
+	multiConfig.loadConfigurations(c)
+	return nil
 }
 
 func (c *Configurations) Equals(that *Configurations) bool {
@@ -108,6 +118,10 @@ func (c *Configurations) Equals(that *Configurations) bool {
 		return false
 	}
 
+	if !c.OCI.Equals(that.OCI) {
+		return false
+	}
+
 	return true
 }
 
@@ -138,6 +152,11 @@ func (c *Configurations) Insert(keyedConfig cloud.Config) error {
 			c.Alibaba = &AlibabaConfigs{}
 		}
 		c.Alibaba.BOA = append(c.Alibaba.BOA, keyedConfig.(*alibaba.BOAConfiguration))
+	case *oracle.UsageApiConfiguration:
+		if c.OCI == nil {
+			c.OCI = &OCIConfigs{}
+		}
+		c.OCI.UsageAPI = append(c.OCI.UsageAPI, keyedConfig.(*oracle.UsageApiConfiguration))
 	default:
 		return fmt.Errorf("Configurations: Insert: failed to insert config of type: %T", keyedConfig)
 	}
@@ -174,6 +193,12 @@ func (c *Configurations) ToSlice() []cloud.KeyedConfig {
 		}
 	}
 
+	if c.OCI != nil {
+		for _, usageConfig := range c.OCI.UsageAPI {
+			keyedConfigs = append(keyedConfigs, usageConfig)
+		}
+	}
+
 	return keyedConfigs
 
 }
@@ -289,3 +314,28 @@ func (ac *AlibabaConfigs) Equals(that *AlibabaConfigs) bool {
 
 	return true
 }
+
+type OCIConfigs struct {
+	UsageAPI []*oracle.UsageApiConfiguration `json:"usageApi,omitempty"`
+}
+
+func (oc *OCIConfigs) Equals(that *OCIConfigs) bool {
+	if oc == nil && that == nil {
+		return true
+	}
+	if oc == nil || that == nil {
+		return false
+	}
+	// Check Usage API
+	if len(oc.UsageAPI) != len(that.UsageAPI) {
+		return false
+	}
+	for i, thisUsageAPI := range oc.UsageAPI {
+		thatUsageAPI := that.UsageAPI[i]
+		if !thisUsageAPI.Equals(thatUsageAPI) {
+			return false
+		}
+	}
+
+	return true
+}

pkg/cloud/config/statuses.go

@@ -9,6 +9,7 @@ import (
 	"github.com/opencost/opencost/pkg/cloud/aws"
 	"github.com/opencost/opencost/pkg/cloud/azure"
 	"github.com/opencost/opencost/pkg/cloud/gcp"
+	"github.com/opencost/opencost/pkg/cloud/oracle"
 )
 
 const (
@@ -16,6 +17,7 @@ const (
 	AthenaConfigType       = "athena"
 	BigQueryConfigType     = "bigquery"
 	AzureStorageConfigType = "azurestorage"
+	UsageApiConfigType     = "usageapi"
 )
 
 func ConfigTypeFromConfig(config cloud.KeyedConfig) (string, error) {
@@ -28,6 +30,8 @@ func ConfigTypeFromConfig(config cloud.KeyedConfig) (string, error) {
 		return BigQueryConfigType, nil
 	case *azure.StorageConfiguration:
 		return AzureStorageConfigType, nil
+	case *oracle.UsageApiConfiguration:
+		return UsageApiConfigType, nil
 	}
 	return "", fmt.Errorf("failed to config type for config with key: %s, type %T", config.Key(), config)
 }
@@ -114,6 +118,8 @@ func (s *Status) UnmarshalJSON(b []byte) error {
 		config = &gcp.BigQueryConfiguration{}
 	case AzureStorageConfigType:
 		config = &azure.StorageConfiguration{}
+	case UsageApiConfigType:
+		config = &oracle.UsageApiConfiguration{}
 	default:
 		return fmt.Errorf("Status: UnmarshalJSON: config type '%s' is not recognized", configType)
 	}

pkg/cloud/gcp/bigqueryintegration.go

@@ -20,6 +20,9 @@ const (
 	UsageDateColumnName          = "usage_date"
 	BillingAccountIDColumnName   = "billing_id"
 	ProjectIDColumnName          = "project_id"
+	ProjectNameColumnName        = "project_name"
+	RegionColumnName             = "region"
+	ZoneColumnName               = "zone"
 	ServiceDescriptionColumnName = "service"
 	SKUDescriptionColumnName     = "description"
 	LabelsColumnName             = "labels"
@@ -44,6 +47,9 @@ func (bqi *BigQueryIntegration) GetCloudCost(start time.Time, end time.Time) (*o
 		fmt.Sprintf("TIMESTAMP_TRUNC(usage_start_time, day) as %s", UsageDateColumnName),
 		fmt.Sprintf("billing_account_id as %s", BillingAccountIDColumnName),
 		fmt.Sprintf("project.id as %s", ProjectIDColumnName),
+		fmt.Sprintf("project.name as %s", ProjectNameColumnName),
+		fmt.Sprintf("location.region as %s", RegionColumnName),
+		fmt.Sprintf("location.zone as %s", ZoneColumnName),
 		fmt.Sprintf("service.description as %s", ServiceDescriptionColumnName),
 		fmt.Sprintf("sku.description as %s", SKUDescriptionColumnName),
 		fmt.Sprintf("resource.name as %s", ResourceNameColumnName),
@@ -58,6 +64,9 @@ func (bqi *BigQueryIntegration) GetCloudCost(start time.Time, end time.Time) (*o
 		UsageDateColumnName,
 		BillingAccountIDColumnName,
 		ProjectIDColumnName,
+		ProjectNameColumnName,
+		RegionColumnName,
+		ZoneColumnName,
 		ServiceDescriptionColumnName,
 		SKUDescriptionColumnName,
 		LabelsColumnName,
@@ -181,7 +190,7 @@ func (bqi *BigQueryIntegration) queryFlexibleCUDTotalCosts(start time.Time, end
 		  IFNULL(SUM((Select SUM(amount) FROM bd.credits)),0),
 		FROM %s
 		WHERE %s
-		GROUP BY usage_date, sku.description
+		GROUP BY usage_date
 	`
 
 	table := fmt.Sprintf(" `%s` bd ", bqi.GetBillingDataDataset())
@@ -214,7 +223,7 @@ func (bqi *BigQueryIntegration) queryFlexibleCUDTotalCredits(start time.Time, en
 	FROM %s
 	CROSS JOIN UNNEST(bd.credits) AS credits
 	WHERE %s
-	GROUP BY usage_date, credits.id
+	GROUP BY usage_date
 	`
 
 	table := fmt.Sprintf(" `%s` bd ", bqi.GetBillingDataDataset())

pkg/cloud/gcp/bigqueryintegration_test.go

@@ -13,7 +13,7 @@ import (
 func TestBigQueryIntegration_GetCloudCost(t *testing.T) {
 	bigQueryConfigPath := os.Getenv("BIGQUERY_CONFIGURATION")
 	if bigQueryConfigPath == "" {
-		t.Skip("skipping integration test, set environment variable ATHENA_CONFIGURATION")
+		t.Skip("skipping integration test, set environment variable BIGQUERY_CONFIGURATION\"")
 	}
 	bigQueryConfigBin, err := os.ReadFile(bigQueryConfigPath)
 	if err != nil {

pkg/cloud/gcp/bigqueryintegration_types.go

@@ -61,6 +61,8 @@ func (ccl *CloudCostLoader) Load(values []bigquery.Value, schema bigquery.Schema
 				invoiceEntityID = ""
 			}
 			properties.InvoiceEntityID = invoiceEntityID
+			// Use InvoiceEntityID as InvoiceEntityName
+			properties.InvoiceEntityName = invoiceEntityID
 		case ProjectIDColumnName:
 			accountID, ok := values[i].(string)
 			if !ok {
@@ -68,6 +70,27 @@ func (ccl *CloudCostLoader) Load(values []bigquery.Value, schema bigquery.Schema
 				accountID = ""
 			}
 			properties.AccountID = accountID
+		case ProjectNameColumnName:
+			accountName, ok := values[i].(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", ProjectNameColumnName, values[i])
+				accountName = ""
+			}
+			properties.AccountName = accountName
+		case RegionColumnName:
+			regionID, ok := values[i].(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", RegionColumnName, values[i])
+				regionID = ""
+			}
+			properties.RegionID = regionID
+		case ZoneColumnName:
+			zone, ok := values[i].(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", ZoneColumnName, values[i])
+				zone = ""
+			}
+			properties.AvailabilityZone = zone
 		case ServiceDescriptionColumnName:
 			service, ok := values[i].(string)
 			if !ok {

pkg/cloud/gcp/provider.go

@@ -495,7 +495,8 @@ func (gcp *GCP) GetOrphanedResources() ([]models.OrphanedResource, error) {
 				desc := map[string]string{}
 				if disk.Description != "" {
 					if err := json.Unmarshal([]byte(disk.Description), &desc); err != nil {
-						return nil, fmt.Errorf("error converting string to map: %s", err)
+						log.Errorf("ignoring orphaned disk %s, failed to convert disk description to map: %s", disk.Name, err)
+						continue
 					}
 				}
 

pkg/cloud/oracle/authorizer.go

@@ -0,0 +1,130 @@
+package oracle
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/oracle/oci-go-sdk/v65/common"
+)
+
+const RawConfigProviderAuthorizerType = "OCIRawConfigProvider"
+
+// Authorizer provides which is used in when creating clients in the OCI SDK
+type Authorizer interface {
+	cloud.Authorizer
+	CreateOCIConfig() (common.ConfigurationProvider, error)
+}
+
+// SelectAuthorizerByType is an implementation of AuthorizerSelectorFn and acts as a register for Authorizer types
+func SelectAuthorizerByType(typeStr string) (Authorizer, error) {
+	switch typeStr {
+	case RawConfigProviderAuthorizerType:
+		return &RawConfigProvider{}, nil
+	default:
+		return nil, fmt.Errorf("OCI: provider authorizer type '%s' is not valid", typeStr)
+	}
+}
+
+// RawConfigProvider holds OCI credentials and fulfils the common.ConfigurationProvider interface
+type RawConfigProvider struct {
+	TenancyID            string  `json:"tenancyID"`
+	UserID               string  `json:"userID"`
+	Region               string  `json:"region"`
+	Fingerprint          string  `json:"fingerprint"`
+	PrivateKey           string  `json:"privateKey"`
+	PrivateKeyPassphrase *string `json:"privateKeyPassphrase"`
+}
+
+// MarshalJSON custom json marshalling functions, sets properties as tagged in struct and sets the authorizer type property
+func (ak *RawConfigProvider) MarshalJSON() ([]byte, error) {
+	fmap := make(map[string]any, 6)
+	fmap[cloud.AuthorizerTypeProperty] = RawConfigProviderAuthorizerType
+	fmap["tenancyId"] = ak.TenancyID
+	fmap["userId"] = ak.UserID
+	fmap["region"] = ak.Region
+	fmap["fingerprint"] = ak.Fingerprint
+	fmap["privateKey"] = ak.PrivateKey
+	fmap["privateKeyPassphrase"] = ak.PrivateKeyPassphrase
+	return json.Marshal(fmap)
+}
+
+func (ak *RawConfigProvider) Validate() error {
+	if ak.TenancyID == "" {
+		return fmt.Errorf("RawConfigProvider: missing tenancy ID")
+	}
+	if ak.UserID == "" {
+		return fmt.Errorf("RawConfigProvider: missing user ID")
+	}
+	if ak.Fingerprint == "" {
+		return fmt.Errorf("RawConfigProvider: missing key fingerprint")
+	}
+	if ak.Region == "" {
+		return fmt.Errorf("RawConfigProvider: missing region")
+	}
+	if ak.PrivateKey == "" {
+		return fmt.Errorf("RawConfigProvider: missing private key")
+	}
+	if ak.PrivateKeyPassphrase != nil {
+		if *ak.PrivateKeyPassphrase == "" {
+			return fmt.Errorf("RawConfigProvider: missing private key passphrase")
+		}
+	}
+
+	return nil
+}
+
+func (ak *RawConfigProvider) Equals(config cloud.Config) bool {
+	if config == nil {
+		return false
+	}
+	thatConfig, ok := config.(*RawConfigProvider)
+	if !ok {
+		return false
+	}
+
+	if ak.TenancyID != thatConfig.TenancyID {
+		return false
+	}
+	if ak.UserID != thatConfig.UserID {
+		return false
+	}
+	if ak.Fingerprint != thatConfig.Fingerprint {
+		return false
+	}
+	if ak.Region != thatConfig.Region {
+		return false
+	}
+	if ak.PrivateKey != thatConfig.PrivateKey {
+		return false
+	}
+	if ak.PrivateKeyPassphrase == nil && thatConfig.PrivateKeyPassphrase != nil {
+		return false
+	}
+	if ak.PrivateKeyPassphrase != nil && thatConfig.PrivateKeyPassphrase == nil {
+		return false
+	}
+	if ak.PrivateKeyPassphrase != nil && thatConfig.PrivateKeyPassphrase != nil {
+		if *ak.PrivateKeyPassphrase != *thatConfig.PrivateKeyPassphrase {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (ak *RawConfigProvider) Sanitize() cloud.Config {
+	redacted := cloud.Redacted
+	return &RawConfigProvider{
+		TenancyID:            ak.TenancyID,
+		UserID:               ak.UserID,
+		Fingerprint:          ak.Fingerprint,
+		Region:               ak.Region,
+		PrivateKey:           cloud.Redacted,
+		PrivateKeyPassphrase: &redacted,
+	}
+}
+
+func (ak *RawConfigProvider) CreateOCIConfig() (common.ConfigurationProvider, error) {
+	return common.NewRawConfigurationProvider(ak.TenancyID, ak.UserID, ak.Region, ak.Fingerprint, ak.PrivateKey, ak.PrivateKeyPassphrase), nil
+}

pkg/cloud/oracle/partnumbers/shape_part_numbers.json

@@ -41,6 +41,12 @@
     "GPU": "B98415",
     "Disk": ""
   },
+  "BM.GPU.L40S.4": {
+    "OCPU": "",
+    "Memory": "",
+    "GPU": "B109479",
+    "Disk": ""
+  },
   "BM.GPU2.2": {
     "OCPU": "",
     "Memory": "",
@@ -185,6 +191,84 @@
     "GPU": "",
     "Disk": ""
   },
+  "PostgreSQL.VM.Standard.Flex.E4": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.16.256GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.2.32GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.32.512GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.4.64GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.64.1024GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.8.128GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex.16.256GB": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex.2.32GB": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex.32.512GB": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex.4.64GB": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex.8.128GB": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
   "VM.DenseIO.E4.Flex": {
     "OCPU": "B93121",
     "Memory": "B93122",
@@ -227,6 +311,12 @@
     "GPU": "",
     "Disk": ""
   },
+  "VM.DenseIO2.8": {
+    "OCPU": "B88516",
+    "Memory": "",
+    "GPU": "",
+    "Disk": ""
+  },
   "VM.GPU.A10.1": {
     "OCPU": "",
     "Memory": "",
@@ -275,6 +365,12 @@
     "GPU": "",
     "Disk": ""
   },
+  "VM.Standard.A2.Flex": {
+    "OCPU": "B109529",
+    "Memory": "B109530",
+    "GPU": "",
+    "Disk": ""
+  },
   "VM.Standard.E3.Flex": {
     "OCPU": "B92306",
     "Memory": "B92307",
@@ -335,4 +431,4 @@
     "GPU": "",
     "Disk": ""
   }
-}
+}

pkg/cloud/oracle/provider.go

@@ -256,7 +256,7 @@ func (o *Oracle) Regions() []string {
 		log.Debugf("Overriding Oracle regions with configured region list: %+v", regionOverrides)
 		return regionOverrides
 	}
-	return oracleRegions
+	return oracleRegions()
 }
 
 func (o *Oracle) PricingSourceSummary() interface{} {

pkg/cloud/oracle/provider_test.go

@@ -73,6 +73,11 @@ func TestGetPVKey(t *testing.T) {
 	assert.Equal(t, providerID, pvkey.ID())
 }
 
+func TestRegions(t *testing.T) {
+	regions := (&Oracle{}).Regions()
+	assert.Len(t, regions, 39)
+}
+
 func testNode(gpus int) *v1.Node {
 	capacity := map[v1.ResourceName]resource.Quantity{}
 	if gpus > 0 {

pkg/cloud/oracle/region.go

@@ -2,40 +2,46 @@ package oracle
 
 // Regions retrieved from https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm.
 // May also be listed using the OCI CLI, "oci iam region list"
-var oracleRegions = []string{
-	"eu-amsterdam-1",
-	"eu-stockholm-1",
-	"me-abudhabi-1",
-	"ap-mumbai-1",
-	"eu-paris-1",
-	"uk-cardiff-1",
-	"me-dubai-1",
-	"eu-frankfurt-1",
-	"sa-saopaulo-1",
-	"ap-hyderabad-1",
-	"us-ashburn-1",
-	"ap-seoul-1",
-	"me-jeddah-1",
-	"af-johannesburg-1",
-	"ap-osaka-1",
-	"uk-london-1",
-	"eu-milan-1",
-	"eu-madrid-1",
-	"ap-melbourne-1",
-	"eu-marseille-1",
-	"mx-monterrey-1",
-	"il-jerusalem-1",
-	"ap-tokyo-1",
-	"us-chicago-1",
-	"us-phoenix-1",
-	"mx-queretaro-1",
-	"sa-santiago-1",
-	"ap-singapore-1",
-	"us-sanjose-1",
-	"ap-sydney-1",
-	"sa-vinhedo-1",
-	"ap-chuncheon-1",
-	"ca-montreal-1",
-	"ca-toronto-1",
-	"eu-zurich-1",
+func oracleRegions() []string {
+	return []string{
+		"af-johannesburg-1",
+		"ap-chuncheon-1",
+		"ap-hyderabad-1",
+		"ap-melbourne-1",
+		"ap-mumbai-1",
+		"ap-osaka-1",
+		"ap-seoul-1",
+		"ap-singapore-1",
+		"ap-singapore-2",
+		"ap-sydney-1",
+		"ap-tokyo-1",
+		"ca-montreal-1",
+		"ca-toronto-1",
+		"eu-amsterdam-1",
+		"eu-frankfurt-1",
+		"eu-madrid-1",
+		"eu-marseille-1",
+		"eu-milan-1",
+		"eu-paris-1",
+		"eu-stockholm-1",
+		"eu-zurich-1",
+		"il-jerusalem-1",
+		"me-abudhabi-1",
+		"me-dubai-1",
+		"me-jeddah-1",
+		"me-riyadh-1",
+		"mx-monterrey-1",
+		"mx-queretaro-1",
+		"sa-bogota-1",
+		"sa-santiago-1",
+		"sa-saopaulo-1",
+		"sa-valparaiso-1",
+		"sa-vinhedo-1",
+		"uk-cardiff-1",
+		"uk-london-1",
+		"us-ashburn-1",
+		"us-chicago-1",
+		"us-phoenix-1",
+		"us-sanjose-1",
+	}
 }

pkg/cloud/oracle/region_test.go

@@ -0,0 +1,17 @@
+package oracle
+
+import (
+	"testing"
+
+	"github.com/oracle/oci-go-sdk/v65/common"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRegionValidation(t *testing.T) {
+	for _, r := range oracleRegions() {
+		// Use the OCI SDK to validate static regions.
+		region := common.StringToRegion(r)
+		_, err := region.RealmID()
+		assert.NoError(t, err)
+	}
+}

pkg/cloud/oracle/usageapiconfiguration.go

@@ -0,0 +1,131 @@
+package oracle
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/opencost/opencost/core/pkg/opencost"
+	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/oracle/oci-go-sdk/v65/usageapi"
+)
+
+type UsageApiConfiguration struct {
+	TenancyID  string     `json:"tenancyID"`
+	Region     string     `json:"region"`
+	Authorizer Authorizer `json:"authorizer"`
+}
+
+func (uac *UsageApiConfiguration) Validate() error {
+	// Validate Authorizer
+	if uac.Authorizer == nil {
+		return fmt.Errorf("UsageApiConfiguration: missing Authorizer")
+	}
+
+	err := uac.Authorizer.Validate()
+	if err != nil {
+		return fmt.Errorf("UsageApiConfiguration: %s", err)
+	}
+
+	// Validate base properties
+	if uac.TenancyID == "" {
+		return fmt.Errorf("UsageApiConfiguration: missing tenancyID")
+	}
+
+	if uac.Region == "" {
+		return fmt.Errorf("UsageApiConfiguration: missing region")
+	}
+
+	return nil
+}
+
+func (uac *UsageApiConfiguration) Equals(config cloud.Config) bool {
+	if config == nil {
+		return false
+	}
+	thatConfig, ok := config.(*UsageApiConfiguration)
+	if !ok {
+		return false
+	}
+
+	if uac.Authorizer != nil {
+		if !uac.Authorizer.Equals(thatConfig.Authorizer) {
+			return false
+		}
+	} else {
+		if thatConfig.Authorizer != nil {
+			return false
+		}
+	}
+
+	if uac.TenancyID != thatConfig.TenancyID {
+		return false
+	}
+
+	if uac.Region != thatConfig.Region {
+		return false
+	}
+
+	return true
+}
+
+func (uac *UsageApiConfiguration) Sanitize() cloud.Config {
+	return &UsageApiConfiguration{
+		TenancyID:  uac.TenancyID,
+		Region:     uac.Region,
+		Authorizer: uac.Authorizer.Sanitize().(Authorizer),
+	}
+}
+
+func (uac *UsageApiConfiguration) Key() string {
+	return uac.TenancyID
+}
+
+func (uac *UsageApiConfiguration) Provider() string {
+	return opencost.OracleProvider
+}
+
+func (uac *UsageApiConfiguration) GetUsageApiClient() (*usageapi.UsageapiClient, error) {
+	configProvider, err := uac.Authorizer.CreateOCIConfig()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create oci config: %s", err.Error())
+	}
+	client, err := usageapi.NewUsageapiClientWithConfigurationProvider(configProvider)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create usage api client: %s", err.Error())
+	}
+	return &client, nil
+}
+
+func (uac *UsageApiConfiguration) UnmarshalJSON(b []byte) error {
+	var f interface{}
+	err := json.Unmarshal(b, &f)
+	if err != nil {
+		return err
+	}
+
+	fmap := f.(map[string]interface{})
+
+	tenancyId, err := cloud.GetInterfaceValue[string](fmap, "tenancyID")
+	if err != nil {
+		return fmt.Errorf("UsageApiConfiguration: UnmarshalJSON: %w", err)
+	}
+	uac.TenancyID = tenancyId
+
+	region, err := cloud.GetInterfaceValue[string](fmap, "region")
+	if err != nil {
+		return fmt.Errorf("UsageApiConfiguration: UnmarshalJSON: %w", err)
+	}
+	uac.Region = region
+
+	authAny, ok := fmap["authorizer"]
+	if !ok {
+		return fmt.Errorf("UsageApiConfiguration: UnmarshalJSON: missing authorizer")
+	}
+	authorizer, err := cloud.AuthorizerFromInterface(authAny, SelectAuthorizerByType)
+	if err != nil {
+		return fmt.Errorf("UsageApiConfiguration: UnmarshalJSON: %w", err)
+	}
+	uac.Authorizer = authorizer
+
+	return nil
+}

pkg/cloud/oracle/usageapiconfiguration_test.go

@@ -0,0 +1,318 @@
+package oracle
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/opencost/opencost/core/pkg/log"
+	"github.com/opencost/opencost/core/pkg/util/json"
+	"github.com/opencost/opencost/pkg/cloud"
+)
+
+func TestUsageApiConfiguration_Validate(t *testing.T) {
+	testCases := map[string]struct {
+		config   UsageApiConfiguration
+		expected error
+	}{
+		"valid config OCI Key": {
+			config: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			expected: nil,
+		},
+		"invalid authorizer": {
+			config: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "",
+					PrivateKey:  "",
+				},
+			},
+			expected: fmt.Errorf("UsageApiConfiguration: RawConfigProvider: missing key fingerprint"),
+		},
+		"missing authorizer": {
+			config: UsageApiConfiguration{
+				TenancyID:  "tenancyID",
+				Region:     "region",
+				Authorizer: nil,
+			},
+			expected: fmt.Errorf("UsageApiConfiguration: missing Authorizer"),
+		},
+		"missing tenancyID": {
+			config: UsageApiConfiguration{
+				TenancyID: "",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			expected: fmt.Errorf("UsageApiConfiguration: missing tenancyID"),
+		},
+		"missing region": {
+			config: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			expected: fmt.Errorf("UsageApiConfiguration: missing region"),
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual := testCase.config.Validate()
+			actualString := "nil"
+			if actual != nil {
+				actualString = actual.Error()
+			}
+			expectedString := "nil"
+			if testCase.expected != nil {
+				expectedString = testCase.expected.Error()
+			}
+			if actualString != expectedString {
+				t.Errorf("errors do not match: Actual: '%s', Expected: '%s", actualString, expectedString)
+			}
+		})
+	}
+}
+
+func TestUsageApiConfiguration_Equals(t *testing.T) {
+	testCases := map[string]struct {
+		left     UsageApiConfiguration
+		right    cloud.Config
+		expected bool
+	}{
+		"matching config": {
+			left: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			right: &UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			expected: true,
+		},
+		"different configurer": {
+			left: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint2",
+					PrivateKey:  "key",
+				},
+			},
+			right: &UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			expected: false,
+		},
+		"missing both configurer": {
+			left: UsageApiConfiguration{
+				TenancyID:  "tenancyID",
+				Region:     "region",
+				Authorizer: nil,
+			},
+			right: &UsageApiConfiguration{
+				TenancyID:  "tenancyID",
+				Region:     "region",
+				Authorizer: nil,
+			},
+			expected: true,
+		},
+		"missing left configurer": {
+			left: UsageApiConfiguration{
+				TenancyID:  "tenancyID",
+				Region:     "region",
+				Authorizer: nil,
+			},
+			right: &UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			expected: false,
+		},
+		"missing right configurer": {
+			left: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			right: &UsageApiConfiguration{
+				TenancyID:  "tenancyID",
+				Region:     "region",
+				Authorizer: nil,
+			},
+			expected: false,
+		},
+		"different tenancyID": {
+			left: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			right: &UsageApiConfiguration{
+				TenancyID: "tenancyID2",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID2",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			expected: false,
+		},
+		"different region": {
+			left: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			right: &UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region2",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region2",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			expected: false,
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual := testCase.left.Equals(testCase.right)
+			if actual != testCase.expected {
+				t.Errorf("incorrect result: Actual: '%t', Expected: '%t", actual, testCase.expected)
+			}
+		})
+	}
+}
+
+func TestUsageApiConfiguration_JSON(t *testing.T) {
+	testCases := map[string]struct {
+		config UsageApiConfiguration
+	}{
+		"Empty Config": {
+			config: UsageApiConfiguration{},
+		},
+		"Nil Authorizer": {
+			config: UsageApiConfiguration{
+				TenancyID:  "tenancyID",
+				Region:     "region",
+				Authorizer: nil,
+			},
+		},
+		"RawConfigProviderAuthorizer": {
+			config: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region2",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+
+			// test JSON Marshalling
+			configJSON, err := json.Marshal(testCase.config)
+			if err != nil {
+				t.Errorf("failed to marshal configuration: %s", err.Error())
+			}
+			log.Info(string(configJSON))
+			unmarshalledConfig := &UsageApiConfiguration{}
+			err = json.Unmarshal(configJSON, unmarshalledConfig)
+			if err != nil {
+				t.Errorf("failed to unmarshal configuration: %s", err.Error())
+			}
+			if !testCase.config.Equals(unmarshalledConfig) {
+				t.Error("config does not equal unmarshalled config")
+			}
+		})
+	}
+}

pkg/cloud/oracle/usageapiintegration.go

@@ -0,0 +1,171 @@
+package oracle
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/opencost/opencost/core/pkg/opencost"
+	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/oracle/oci-go-sdk/v65/common"
+	"github.com/oracle/oci-go-sdk/v65/usageapi"
+)
+
+type UsageApiIntegration struct {
+	UsageApiConfiguration
+	ConnectionStatus cloud.ConnectionStatus
+}
+
+func (uai *UsageApiIntegration) GetCloudCost(start time.Time, end time.Time) (*opencost.CloudCostSetRange, error) {
+	client, err := uai.GetUsageApiClient()
+	if err != nil {
+		uai.ConnectionStatus = cloud.FailedConnection
+		return nil, fmt.Errorf("getting oracle usage api client: %s", err.Error())
+	}
+
+	req := usageapi.RequestSummarizedUsagesRequest{
+		RequestSummarizedUsagesDetails: usageapi.RequestSummarizedUsagesDetails{
+			Granularity:       usageapi.RequestSummarizedUsagesDetailsGranularityDaily,
+			GroupBy:           []string{"resourceId", "service", "subscriptionId", "tenantName"},
+			IsAggregateByTime: common.Bool(false),
+			TimeUsageStarted:  &common.SDKTime{Time: start},
+			TimeUsageEnded:    &common.SDKTime{Time: end},
+			QueryType:         usageapi.RequestSummarizedUsagesDetailsQueryTypeCost,
+			TenantId:          common.String(uai.TenancyID),
+		},
+		Limit: common.Int(500),
+	}
+
+	resp, err := client.RequestSummarizedUsages(context.Background(), req)
+	if err != nil {
+		uai.ConnectionStatus = cloud.FailedConnection
+		return nil, fmt.Errorf("failed to query usage: %w", err)
+	}
+
+	ccsr, err := opencost.NewCloudCostSetRange(start, end, opencost.AccumulateOptionDay, uai.Key())
+	if err != nil {
+		return nil, err
+	}
+
+	// Set status to missing data if query comes back empty and the status isn't already successful
+	if len(resp.Items) == 0 && uai.ConnectionStatus != cloud.SuccessfulConnection {
+		uai.ConnectionStatus = cloud.MissingData
+		return ccsr, nil
+	}
+
+	for _, item := range resp.Items {
+		resourceId := ""
+		if item.ResourceId != nil {
+			resourceId = *item.ResourceId
+		}
+
+		tenantName := ""
+		if item.TenantName != nil {
+			tenantName = *item.TenantName
+		}
+
+		subscriptionId := ""
+		if item.SubscriptionId != nil {
+			subscriptionId = *item.SubscriptionId
+		}
+
+		service := ""
+		if item.Service != nil {
+			service = *item.Service
+		}
+
+		category := SelectOCICategory(service)
+
+		// Iterate through the slice of tags, assigning
+		// keys and values to the map of labels
+		labels := opencost.CloudCostLabels{}
+		for _, tag := range item.Tags {
+			if tag.Key == nil || tag.Value == nil {
+				continue
+			}
+			labels[*tag.Key] = *tag.Value
+		}
+
+		properties := &opencost.CloudCostProperties{
+			ProviderID:      resourceId,
+			Provider:        opencost.OracleProvider,
+			AccountID:       uai.TenancyID,
+			AccountName:     tenantName,
+			InvoiceEntityID: subscriptionId,
+			RegionID:        uai.Region,
+			Service:         service,
+			Category:        category,
+			Labels:          labels,
+		}
+
+		winStart := item.TimeUsageStarted.Time
+		winEnd := start.AddDate(0, 0, 1)
+
+		listRate := 0.0
+		if item.ListRate != nil {
+			listRate = float64(*item.ListRate)
+		}
+
+		attrCostToParse := ""
+		if item.AttributedCost != nil {
+			attrCostToParse = *item.AttributedCost
+		}
+
+		attrCost, err := strconv.ParseFloat(attrCostToParse, 64)
+		if err != nil {
+			return nil, fmt.Errorf("unable to parse float '%s': %s", attrCostToParse, err.Error())
+		}
+
+		computedAmt := 0.0
+		if item.ComputedAmount != nil {
+			computedAmt = float64(*item.ComputedAmount)
+		}
+
+		cc := &opencost.CloudCost{
+			Properties: properties,
+			Window:     opencost.NewWindow(&winStart, &winEnd),
+			//todo: which returned costs go where?
+			ListCost: opencost.CostMetric{
+				Cost: listRate,
+			},
+			NetCost: opencost.CostMetric{
+				Cost: computedAmt,
+			},
+			AmortizedNetCost: opencost.CostMetric{
+				Cost: attrCost,
+			},
+			AmortizedCost: opencost.CostMetric{
+				Cost: attrCost,
+			},
+			InvoicedCost: opencost.CostMetric{
+				Cost: computedAmt,
+			},
+		}
+
+		ccsr.LoadCloudCost(cc)
+	}
+
+	uai.ConnectionStatus = cloud.SuccessfulConnection
+	return ccsr, nil
+}
+
+func (uai *UsageApiIntegration) GetStatus() cloud.ConnectionStatus {
+	// initialize status if it has not done so; this can happen if the integration is inactive
+	if uai.ConnectionStatus.String() == "" {
+		uai.ConnectionStatus = cloud.InitialStatus
+	}
+	return uai.ConnectionStatus
+}
+
+func SelectOCICategory(service string) string {
+	if service == "Compute" {
+		return opencost.ComputeCategory
+	} else if service == "Block Storage" || service == "Object Storage" {
+		return opencost.StorageCategory
+	} else if service == "Load Balancer" || service == "Virtual Cloud Network" {
+		return opencost.NetworkCategory
+	} else {
+		return opencost.OtherCategory
+	}
+}

pkg/cloud/oracle/usageapiintegration_test.go

@@ -0,0 +1,61 @@
+package oracle
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/opencost/opencost/core/pkg/util/timeutil"
+)
+
+func TestUsageAPIIntegration_GetCloudCost(t *testing.T) {
+	usageApiConfigPath := os.Getenv("USAGEAPI_CONFIGURATION")
+	if usageApiConfigPath == "" {
+		t.Skip("skipping integration test, set environment variable USAGEAPI_CONFIGURATION")
+	}
+	usageApiConfigBin, err := os.ReadFile(usageApiConfigPath)
+	if err != nil {
+		t.Fatalf("failed to read config file: %s", err.Error())
+	}
+	var usageApiConfig UsageApiConfiguration
+	err = json.Unmarshal(usageApiConfigBin, &usageApiConfig)
+	if err != nil {
+		t.Fatalf("failed to unmarshal config from JSON: %s", err.Error())
+	}
+	testCases := map[string]struct {
+		integration *UsageApiIntegration
+		start       time.Time
+		end         time.Time
+		expected    bool
+	}{
+		// No CUR data is expected within 2 days of now
+		"too_recent_window": {
+			integration: &UsageApiIntegration{
+				UsageApiConfiguration: usageApiConfig,
+			},
+			end:      time.Now(),
+			start:    time.Now().Add(-timeutil.Day),
+			expected: true,
+		},
+		// CUR data should be available
+		"last week window": {
+			integration: &UsageApiIntegration{
+				UsageApiConfiguration: usageApiConfig,
+			},
+			end:      time.Now().Add(-7 * timeutil.Day),
+			start:    time.Now().Add(-8 * timeutil.Day),
+			expected: false,
+		},
+	}
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual, err := testCase.integration.GetCloudCost(testCase.start, testCase.end)
+			if err != nil {
+				t.Errorf("Other error during testing %s", err)
+			} else if actual.IsEmpty() != testCase.expected {
+				t.Errorf("Incorrect result, actual emptiness: %t, expected: %t", actual.IsEmpty(), testCase.expected)
+			}
+		})
+	}
+}

pkg/cloud/provider/provider.go

@@ -2,9 +2,11 @@ package provider
 
 import (
 	"errors"
+	"fmt"
 	"net"
 	"net/http"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
@@ -327,6 +329,7 @@ var (
 	// gce://guestbook-227502/us-central1-a/gke-niko-n1-standard-2-wljla-8df8e58a-hfy7
 	//  => gke-niko-n1-standard-2-wljla-8df8e58a-hfy7
 	providerGCERegex = regexp.MustCompile("gce://[^/]*/[^/]*/([^/]+)")
+
 	// Capture "vol-0fc54c5e83b8d2b76" from "aws://us-east-2a/vol-0fc54c5e83b8d2b76"
 	persistentVolumeAWSRegex = regexp.MustCompile("aws:/[^/]*/[^/]*/([^/]+)")
 	// Capture "ad9d88195b52a47c89b5055120f28c58" from "ad9d88195b52a47c89b5055120f28c58-1037804914.us-east-2.elb.amazonaws.com"
@@ -373,3 +376,32 @@ func ParseLBID(id string) string {
 	// Return id for GCP Provider, Azure Provider, CSV Provider and Custom Provider
 	return id
 }
+
+// ParseLocalDiskID attempts to parse a ProviderID from the ProviderID of the node that the local disk is running on
+func ParseLocalDiskID(id string) string {
+	// Parse like node
+	id = ParseID(id)
+
+	if strings.HasPrefix(id, "azure://") {
+
+		// handle vmss ProviderID of type azure:///subscriptions/ae337b64-e7ba-3387-b043-187289efe4e3/resourceGroups/mc_test_eastus2/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userpool-12345678-vmss/virtualMachines/11
+		if strings.Contains(id, "virtualMachineScaleSets") {
+			split := strings.Split(id, "/virtualMachineScaleSets/")
+			// combine vmss name and number into a single string ending in a 6 character base 32 number
+			vmSplit := strings.Split(split[1], "/")
+			if len(vmSplit) != 3 {
+				return id
+			}
+			vmNum, err := strconv.ParseInt(vmSplit[2], 10, 64)
+			if err != nil {
+				return id
+			}
+
+			id = fmt.Sprintf("%s/disks/%s%06s", split[0], vmSplit[0], strconv.FormatInt(vmNum, 32))
+		}
+		id = strings.Replace(id, "/virtualMachines/", "/disks/", -1)
+		id = strings.ToLower(id)
+		return fmt.Sprintf("%s_osdisk", id)
+	}
+	return id
+}

pkg/cloud/provider/provider_test.go

@@ -0,0 +1,44 @@
+package provider
+
+import (
+	"testing"
+)
+
+func TestParseLocalDiskID(t *testing.T) {
+	tests := map[string]struct {
+		input string
+		want  string
+	}{
+		"empty string": {
+			input: "",
+			want:  "",
+		},
+		"generic string": {
+			input: "test",
+			want:  "test",
+		},
+		"AWS node provider id": {
+			input: "aws:///us-east-2a/i-0fea4fd46592d050b",
+			want:  "i-0fea4fd46592d050b",
+		},
+		"GCP node provider id": {
+			input: "gce://guestbook-11111/us-central1-a/gke-niko-n1-standard-2-wlkla-8d48e58a-hfy7",
+			want:  "gke-niko-n1-standard-2-wlkla-8d48e58a-hfy7",
+		},
+		"Azure vmss provider id": {
+			input: "azure:///subscriptions/ae337b64-e7ba-3387-b043-187289efe4e3/resourceGroups/mc_test_eastus2/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userpool-12345678-vmss/virtualMachines/11",
+			want:  "azure:///subscriptions/ae337b64-e7ba-3387-b043-187289efe4e3/resourcegroups/mc_test_eastus2/providers/microsoft.compute/disks/aks-userpool-12345678-vmss00000b_osdisk",
+		},
+		"Azure vm provider id": {
+			input: "azure:///subscriptions/ae337b64-e7ba-3387-b043-187289efe4e3/resourceGroups/mc_test_eastus2/providers/Microsoft.Compute/virtualMachines/master-0",
+			want:  "azure:///subscriptions/ae337b64-e7ba-3387-b043-187289efe4e3/resourcegroups/mc_test_eastus2/providers/microsoft.compute/disks/master-0_osdisk",
+		},
+	}
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			if got := ParseLocalDiskID(tt.input); got != tt.want {
+				t.Errorf("ParseLocalDiskID() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/cloudcost/integration.go

@@ -9,6 +9,7 @@ import (
 	"github.com/opencost/opencost/pkg/cloud/aws"
 	"github.com/opencost/opencost/pkg/cloud/azure"
 	"github.com/opencost/opencost/pkg/cloud/gcp"
+	"github.com/opencost/opencost/pkg/cloud/oracle"
 )
 
 // CloudCostIntegration is an interface for retrieving daily granularity CloudCost data for a given range
@@ -57,15 +58,24 @@ func GetIntegrationFromConfig(kc cloud.KeyedConfig) CloudCostIntegration {
 	case *azure.StorageConnection:
 		return &azure.AzureStorageIntegration{
 			AzureStorageBillingParser: azure.AzureStorageBillingParser{
-				StorageConnection: *keyedConfig,
+				StorageConnection: azure.StorageConnection{
+					StorageConfiguration: keyedConfig.StorageConfiguration},
 			},
 		}
 	case *azure.AzureStorageBillingParser:
 		return &azure.AzureStorageIntegration{
-			AzureStorageBillingParser: *keyedConfig,
+			AzureStorageBillingParser: azure.AzureStorageBillingParser{
+				StorageConnection: azure.StorageConnection{
+					StorageConfiguration: keyedConfig.StorageConfiguration},
+			},
 		}
 	case *azure.AzureStorageIntegration:
-		return keyedConfig
+		return &azure.AzureStorageIntegration{
+			AzureStorageBillingParser: azure.AzureStorageBillingParser{
+				StorageConnection: azure.StorageConnection{
+					StorageConfiguration: keyedConfig.StorageConfiguration},
+			},
+		}
 	// S3SelectIntegration
 	case *aws.S3Configuration:
 		return &aws.S3SelectIntegration{
@@ -90,6 +100,10 @@ func GetIntegrationFromConfig(kc cloud.KeyedConfig) CloudCostIntegration {
 	// Alibaba BOA Integration
 	case *alibaba.BOAConfiguration:
 		return nil
+	case *oracle.UsageApiConfiguration:
+		return &oracle.UsageApiIntegration{
+			UsageApiConfiguration: *keyedConfig,
+		}
 	default:
 		return nil
 	}

pkg/costmodel/aggregation.go

@@ -2284,7 +2284,9 @@ func (a *Accesses) ComputeAllocationHandler(w http.ResponseWriter, r *http.Reque
 	// include aggregated labels/annotations if true
 	includeAggregatedMetadata := qp.GetBool("includeAggregatedMetadata", false)
 
-	asr, err := a.Model.QueryAllocation(window, resolution, step, aggregateBy, includeIdle, idleByNode, includeProportionalAssetResourceCosts, includeAggregatedMetadata, sharedLoadBalancer, accumulateBy)
+	shareIdle := qp.GetBool("shareIdle", false)
+
+	asr, err := a.Model.QueryAllocation(window, resolution, step, aggregateBy, includeIdle, idleByNode, includeProportionalAssetResourceCosts, includeAggregatedMetadata, sharedLoadBalancer, accumulateBy, shareIdle)
 	if err != nil {
 		if strings.Contains(strings.ToLower(err.Error()), "bad request") {
 			WriteError(w, BadRequest(err.Error()))

pkg/costmodel/allocation.go

@@ -28,7 +28,7 @@ const (
 	queryFmtCPURequests                 = `avg(avg_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
 	queryFmtCPUUsageAvg                 = `avg(rate(container_cpu_usage_seconds_total{container!="", container_name!="POD", container!="POD", %s}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
 	queryFmtGPUsRequested               = `avg(avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
-	queryFmtGPUsUsageAvg                = `avg(avg_over_time(DCGM_FI_DEV_GPU_UTIL{container!=""}[%s])) by (container, pod, namespace, %s)`
+	queryFmtGPUsUsageAvg                = `avg(avg_over_time(DCGM_FI_PROF_GR_ENGINE_ACTIVE{container!=""}[%s])) by (container, pod, namespace, %s)`
 	queryFmtGPUsAllocated               = `avg(avg_over_time(container_gpu_allocation{container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
 	queryFmtNodeCostPerCPUHr            = `avg(avg_over_time(node_cpu_hourly_cost{%s}[%s])) by (node, %s, instance_type, provider_id)`
 	queryFmtNodeCostPerRAMGiBHr         = `avg(avg_over_time(node_ram_hourly_cost{%s}[%s])) by (node, %s, instance_type, provider_id)`

pkg/costmodel/allocation_helpers.go

@@ -615,7 +615,7 @@ func applyRAMBytesUsedMax(podMap map[podKey]*pod, resRAMBytesUsedMax []*prom.Que
 }
 
 func applyGPUUsageAvg(podMap map[podKey]*pod, resGPUUsageAvg []*prom.QueryResult, podUIDKeyMap map[podKey][]podKey) {
-	// Example PromQueryResult: {container="dcgmproftester12", namespace="gpu", pod="dcgmproftester3-deployment-fc89c8dd6-ph7z5"} 99
+	// Example PromQueryResult: {container="dcgmproftester12", namespace="gpu", pod="dcgmproftester3-deployment-fc89c8dd6-ph7z5"} 0.997307
 	for _, res := range resGPUUsageAvg {
 		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
 		if err != nil {
@@ -649,9 +649,8 @@ func applyGPUUsageAvg(podMap map[podKey]*pod, resGPUUsageAvg []*prom.QueryResult
 				thisPod.appendContainer(container)
 			}
 
-			// DCGM_FI_DEV_GPU_UTIL metric is a number 0-100. Scale down to a
-			// percentage so it is consistent with other fields.
-			thisPod.Allocations[container].GPUUsageAverage = res.Values[0].Value * 0.01
+			// DCGM_FI_PROF_GR_ENGINE_ACTIVE metric is a float between 0-1.
+			thisPod.Allocations[container].GPUUsageAverage = res.Values[0].Value
 		}
 	}
 }

pkg/costmodel/cluster.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/opencost/opencost/pkg/cloud/provider"
@@ -42,7 +43,16 @@ const (
 	queryNodes = `sum(avg(node_total_hourly_cost{%s}) by (node, %s)) * 730 %s`
 )
 
-const maxLocalDiskSize = 200 // AWS limits root disks to 100 Gi, and occasional metric errors in filesystem size should not contribute to large costs.
+const MAX_LOCAL_STORAGE_SIZE = 1024 * 1024 * 1024 * 1024
+
+// When ASSET_INCLUDE_LOCAL_DISK_COST is set to false, local storage
+// provisioned by sig-storage-local-static-provisioner is excluded
+// by checking if the volume is prefixed by "local-pv-".
+//
+// This is based on the sig-storage-local-static-provisioner implementation,
+// which creates all PVs with the "local-pv-" prefix. For reference, see:
+// https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner/blob/b6f465027bd059e92c0032c81dd1e1d90e35c909/pkg/discovery/discovery.go#L410-L417
+const SIG_STORAGE_LOCAL_PROVISIONER_PREFIX = "local-pv-"
 
 // Costs represents cumulative and monthly cluster costs over a given duration. Costs
 // are broken down by cores, memory, and storage.
@@ -142,7 +152,7 @@ type DiskIdentifier struct {
 	Name    string
 }
 
-func ClusterDisks(client prometheus.Client, provider models.Provider, start, end time.Time) (map[DiskIdentifier]*Disk, error) {
+func ClusterDisks(client prometheus.Client, cp models.Provider, start, end time.Time) (map[DiskIdentifier]*Disk, error) {
 	// Start from the time "end", querying backwards
 	t := end
 
@@ -209,11 +219,15 @@ func ClusterDisks(client prometheus.Client, provider models.Provider, start, end
 		hourlyToCumulative := float64(minsPerResolution) * (1.0 / 60.0)
 		costPerGBHr := 0.04 / 730.0
 
-		queryLocalStorageCost := fmt.Sprintf(`sum_over_time(sum(container_fs_limit_bytes{device!="tmpfs", id="/", %s}) by (instance, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
-		queryLocalStorageUsedCost := fmt.Sprintf(`sum_over_time(sum(container_fs_usage_bytes{device!="tmpfs", id="/", %s}) by (instance, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
-		queryLocalStorageUsedAvg := fmt.Sprintf(`avg(sum(avg_over_time(container_fs_usage_bytes{device!="tmpfs", id="/", %s}[%s])) by (instance, %s, job)) by (instance, %s)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel(), env.GetPromClusterLabel())
-		queryLocalStorageUsedMax := fmt.Sprintf(`max(sum(max_over_time(container_fs_usage_bytes{device!="tmpfs", id="/", %s}[%s])) by (instance, %s, job)) by (instance, %s)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel(), env.GetPromClusterLabel())
-		queryLocalStorageBytes := fmt.Sprintf(`avg_over_time(sum(container_fs_limit_bytes{device!="tmpfs", id="/", %s}) by (instance, %s)[%s:%dm])`, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
+		// container_fs metrics contains metrics for disks that are not local storage of the node. While not perfect to
+		// attempt to identify the correct device which is being used as local storage we first filter for devices mounted
+		// at paths `/dev/nvme.*` or `/dev/sda.*`. There still may be multiple devices mounted at paths matching the regex
+		// so later on we will select the device with the highest `container_fs_limit_bytes` per instance to create a local disk asset
+		queryLocalStorageCost := fmt.Sprintf(`sum_over_time(sum(container_fs_limit_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
+		queryLocalStorageUsedCost := fmt.Sprintf(`sum_over_time(sum(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
+		queryLocalStorageUsedAvg := fmt.Sprintf(`avg(sum(avg_over_time(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}[%s])) by (instance, device, %s, job)) by (instance, device, %s)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel(), env.GetPromClusterLabel())
+		queryLocalStorageUsedMax := fmt.Sprintf(`max(sum(max_over_time(container_fs_usage_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}[%s])) by (instance, device, %s, job)) by (instance, device, %s)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel(), env.GetPromClusterLabel())
+		queryLocalStorageBytes := fmt.Sprintf(`avg_over_time(sum(container_fs_limit_bytes{device=~"/dev/(nvme|sda).*", id="/", %s}) by (instance, device, %s)[%s:%dm])`, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
 		queryLocalActiveMins := fmt.Sprintf(`count(node_total_hourly_cost{%s}) by (%s, node)[%s:%dm]`, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
 
 		resChLocalStorageCost := ctx.QueryAtTime(queryLocalStorageCost, t)
@@ -273,9 +287,18 @@ func ClusterDisks(client prometheus.Client, provider models.Provider, start, end
 		diskMap[key].ClaimNamespace = claimNamespace
 	}
 
-	pvCosts(diskMap, resolution, resActiveMins, resPVSize, resPVCost, resPVUsedAvg, resPVUsedMax, resPVCInfo, provider, opencost.NewClosedWindow(start, end))
+	pvCosts(diskMap, resolution, resActiveMins, resPVSize, resPVCost, resPVUsedAvg, resPVUsedMax, resPVCInfo, cp, opencost.NewClosedWindow(start, end))
 
-	for _, result := range resLocalStorageCost {
+	type localStorage struct {
+		device string
+		disk   *Disk
+	}
+
+	localStorageDisks := map[DiskIdentifier]localStorage{}
+
+	// Start with local storage bytes so that the device with the largest size which has passed the
+	// query filters can be determined
+	for _, result := range resLocalStorageBytes {
 		cluster, err := result.GetString(env.GetPromClusterLabel())
 		if err != nil {
 			cluster = env.GetClusterID()
@@ -287,23 +310,37 @@ func ClusterDisks(client prometheus.Client, provider models.Provider, start, end
 			continue
 		}
 
-		cost := result.Values[0].Value
+		device, err := result.GetString("device")
+		if err != nil {
+			log.Warnf("ClusterDisks: local storage data missing device")
+			continue
+		}
+
+		bytes := result.Values[0].Value
+		// Ignore disks that are larger than the max size
+		if bytes > MAX_LOCAL_STORAGE_SIZE {
+			continue
+		}
+
 		key := DiskIdentifier{cluster, name}
-		if _, ok := diskMap[key]; !ok {
-			diskMap[key] = &Disk{
-				Cluster:   cluster,
-				Name:      name,
-				Breakdown: &ClusterCostsBreakdown{},
-				Local:     true,
+
+		// only keep the device with the most bytes per instance
+		if current, ok := localStorageDisks[key]; !ok || current.disk.Bytes < bytes {
+			localStorageDisks[key] = localStorage{
+				device: device,
+				disk: &Disk{
+					Cluster:      cluster,
+					Name:         name,
+					Breakdown:    &ClusterCostsBreakdown{},
+					Local:        true,
+					StorageClass: opencost.LocalStorageClass,
+					Bytes:        bytes,
+				},
 			}
 		}
-		diskMap[key].Cost += cost
-
-		//Assigning explicitly the storage class of local storage to local
-		diskMap[key].StorageClass = opencost.LocalStorageClass
 	}
 
-	for _, result := range resLocalStorageUsedCost {
+	for _, result := range resLocalStorageCost {
 		cluster, err := result.GetString(env.GetPromClusterLabel())
 		if err != nil {
 			cluster = env.GetClusterID()
@@ -311,24 +348,27 @@ func ClusterDisks(client prometheus.Client, provider models.Provider, start, end
 
 		name, err := result.GetString("instance")
 		if err != nil {
-			log.Warnf("ClusterDisks: local storage usage data missing instance")
+			log.Warnf("ClusterDisks: local storage data missing instance")
+			continue
+		}
+
+		device, err := result.GetString("device")
+		if err != nil {
+			log.Warnf("ClusterDisks: local storage data missing device")
 			continue
 		}
 
 		cost := result.Values[0].Value
 		key := DiskIdentifier{cluster, name}
-		if _, ok := diskMap[key]; !ok {
-			diskMap[key] = &Disk{
-				Cluster:   cluster,
-				Name:      name,
-				Breakdown: &ClusterCostsBreakdown{},
-				Local:     true,
-			}
+		ls, ok := localStorageDisks[key]
+		if !ok || ls.device != device {
+			continue
 		}
-		diskMap[key].Breakdown.System = cost / diskMap[key].Cost
+		ls.disk.Cost = cost
+
 	}
 
-	for _, result := range resLocalStorageUsedAvg {
+	for _, result := range resLocalStorageUsedCost {
 		cluster, err := result.GetString(env.GetPromClusterLabel())
 		if err != nil {
 			cluster = env.GetClusterID()
@@ -336,24 +376,26 @@ func ClusterDisks(client prometheus.Client, provider models.Provider, start, end
 
 		name, err := result.GetString("instance")
 		if err != nil {
-			log.Warnf("ClusterDisks: local storage data missing instance")
+			log.Warnf("ClusterDisks: local storage usage data missing instance")
 			continue
 		}
 
-		bytesAvg := result.Values[0].Value
+		device, err := result.GetString("device")
+		if err != nil {
+			log.Warnf("ClusterDisks: local storage data missing device")
+			continue
+		}
+
+		cost := result.Values[0].Value
 		key := DiskIdentifier{cluster, name}
-		if _, ok := diskMap[key]; !ok {
-			diskMap[key] = &Disk{
-				Cluster:   cluster,
-				Name:      name,
-				Breakdown: &ClusterCostsBreakdown{},
-				Local:     true,
-			}
+		ls, ok := localStorageDisks[key]
+		if !ok || ls.device != device {
+			continue
 		}
-		diskMap[key].BytesUsedAvgPtr = &bytesAvg
+		ls.disk.Breakdown.System = cost / ls.disk.Cost
 	}
 
-	for _, result := range resLocalStorageUsedMax {
+	for _, result := range resLocalStorageUsedAvg {
 		cluster, err := result.GetString(env.GetPromClusterLabel())
 		if err != nil {
 			cluster = env.GetClusterID()
@@ -365,20 +407,22 @@ func ClusterDisks(client prometheus.Client, provider models.Provider, start, end
 			continue
 		}
 
-		bytesMax := result.Values[0].Value
+		device, err := result.GetString("device")
+		if err != nil {
+			log.Warnf("ClusterDisks: local storage data missing device")
+			continue
+		}
+
+		bytesAvg := result.Values[0].Value
 		key := DiskIdentifier{cluster, name}
-		if _, ok := diskMap[key]; !ok {
-			diskMap[key] = &Disk{
-				Cluster:   cluster,
-				Name:      name,
-				Breakdown: &ClusterCostsBreakdown{},
-				Local:     true,
-			}
+		ls, ok := localStorageDisks[key]
+		if !ok || ls.device != device {
+			continue
 		}
-		diskMap[key].BytesUsedMaxPtr = &bytesMax
+		ls.disk.BytesUsedAvgPtr = &bytesAvg
 	}
 
-	for _, result := range resLocalStorageBytes {
+	for _, result := range resLocalStorageUsedMax {
 		cluster, err := result.GetString(env.GetPromClusterLabel())
 		if err != nil {
 			cluster = env.GetClusterID()
@@ -390,21 +434,19 @@ func ClusterDisks(client prometheus.Client, provider models.Provider, start, end
 			continue
 		}
 
-		bytes := result.Values[0].Value
-		key := DiskIdentifier{cluster, name}
-		if _, ok := diskMap[key]; !ok {
-			diskMap[key] = &Disk{
-				Cluster:   cluster,
-				Name:      name,
-				Breakdown: &ClusterCostsBreakdown{},
-				Local:     true,
-			}
+		device, err := result.GetString("device")
+		if err != nil {
+			log.Warnf("ClusterDisks: local storage data missing device")
+			continue
 		}
-		diskMap[key].Bytes = bytes
-		if bytes/1024/1024/1024 > maxLocalDiskSize {
-			log.DedupedWarningf(5, "Deleting large root disk/localstorage disk from analysis")
-			delete(diskMap, key)
+
+		bytesMax := result.Values[0].Value
+		key := DiskIdentifier{cluster, name}
+		ls, ok := localStorageDisks[key]
+		if !ok || ls.device != device {
+			continue
 		}
+		ls.disk.BytesUsedMaxPtr = &bytesMax
 	}
 
 	for _, result := range resLocalActiveMins {
@@ -419,12 +461,20 @@ func ClusterDisks(client prometheus.Client, provider models.Provider, start, end
 			continue
 		}
 
+		providerID, err := result.GetString("provider_id")
+		if err != nil {
+			log.DedupedWarningf(5, "ClusterDisks: local active mins data missing instance")
+			continue
+		}
+
 		key := DiskIdentifier{cluster, name}
-		if _, ok := diskMap[key]; !ok {
-			log.DedupedWarningf(5, "ClusterDisks: local active mins for unidentified disk or disk deleted from analysis")
+		ls, ok := localStorageDisks[key]
+		if !ok {
 			continue
 		}
 
+		ls.disk.ProviderID = provider.ParseLocalDiskID(providerID)
+
 		if len(result.Values) == 0 {
 			continue
 		}
@@ -435,9 +485,14 @@ func ClusterDisks(client prometheus.Client, provider models.Provider, start, end
 
 		// TODO niko/assets if mins >= threshold, interpolate for missing data?
 
-		diskMap[key].End = e
-		diskMap[key].Start = s
-		diskMap[key].Minutes = mins
+		ls.disk.End = e
+		ls.disk.Start = s
+		ls.disk.Minutes = mins
+	}
+
+	// move local storage disks to main disk map
+	for key, ls := range localStorageDisks {
+		diskMap[key] = ls.disk
 	}
 
 	var unTracedDiskLogData []DiskIdentifier
@@ -487,6 +542,10 @@ func ClusterDisks(client prometheus.Client, provider models.Provider, start, end
 		}
 	}
 
+	if !env.GetAssetIncludeLocalDiskCost() {
+		return filterOutLocalPVs(diskMap), nil
+	}
+
 	return diskMap, nil
 }
 
@@ -1503,11 +1562,13 @@ func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActi
 				log.Debugf("ClusterDisks: pv claim data missing volumename")
 				continue
 			}
+
 			thatClaimName, err := thatRes.GetString("persistentvolumeclaim")
 			if err != nil {
 				log.Debugf("ClusterDisks: pv claim data missing persistentvolumeclaim")
 				continue
 			}
+
 			thatClaimNamespace, err := thatRes.GetString("namespace")
 			if err != nil {
 				log.Debugf("ClusterDisks: pv claim data missing namespace")
@@ -1544,6 +1605,7 @@ func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActi
 			log.Debugf("ClusterDisks: pv usage data missing persistentvolumeclaim")
 			continue
 		}
+
 		claimNamespace, err := result.GetString("namespace")
 		if err != nil {
 			log.Debugf("ClusterDisks: pv usage data missing namespace")
@@ -1564,11 +1626,13 @@ func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActi
 				log.Debugf("ClusterDisks: pv claim data missing volumename")
 				continue
 			}
+
 			thatClaimName, err := thatRes.GetString("persistentvolumeclaim")
 			if err != nil {
 				log.Debugf("ClusterDisks: pv claim data missing persistentvolumeclaim")
 				continue
 			}
+
 			thatClaimNamespace, err := thatRes.GetString("namespace")
 			if err != nil {
 				log.Debugf("ClusterDisks: pv claim data missing namespace")
@@ -1594,3 +1658,22 @@ func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActi
 		diskMap[key].BytesUsedMaxPtr = &usage
 	}
 }
+
+// filterOutLocalPVs removes local Persistent Volumes (PVs) from the given disk map.
+// Local PVs are identified by the prefix "local-pv-" in their names, which is the
+// convention used by sig-storage-local-static-provisioner.
+//
+// Parameters:
+//   - diskMap: A map of DiskIdentifier to Disk pointers, representing all PVs.
+//
+// Returns:
+//   - A new map of DiskIdentifier to Disk pointers, containing only non-local PVs.
+func filterOutLocalPVs(diskMap map[DiskIdentifier]*Disk) map[DiskIdentifier]*Disk {
+	nonLocalPVDiskMap := map[DiskIdentifier]*Disk{}
+	for key, val := range diskMap {
+		if !strings.HasPrefix(key.Name, SIG_STORAGE_LOCAL_PROVISIONER_PREFIX) {
+			nonLocalPVDiskMap[key] = val
+		}
+	}
+	return nonLocalPVDiskMap
+}

pkg/costmodel/cluster_test.go

@@ -0,0 +1,54 @@
+package costmodel
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_filterOutLocalPVs(t *testing.T) {
+	testCases := []struct {
+		name     string
+		input    map[DiskIdentifier]*Disk
+		expected map[DiskIdentifier]*Disk
+	}{
+		{
+			name: "Filter out local PVs",
+			input: map[DiskIdentifier]*Disk{
+				{Cluster: "cluster1", Name: "pv1"}:              &Disk{Name: "pv1"},
+				{Cluster: "cluster1", Name: "local-pv-123"}:     &Disk{Name: "local-pv-123"},
+				{Cluster: "cluster2", Name: "pv2"}:              &Disk{Name: "pv2"},
+				{Cluster: "cluster2", Name: "local-pv-456"}:     &Disk{Name: "local-pv-456"},
+				{Cluster: "cluster3", Name: "not-local-pv-789"}: &Disk{Name: "not-local-pv-789"},
+			},
+			expected: map[DiskIdentifier]*Disk{
+				{Cluster: "cluster1", Name: "pv1"}:              &Disk{Name: "pv1"},
+				{Cluster: "cluster2", Name: "pv2"}:              &Disk{Name: "pv2"},
+				{Cluster: "cluster3", Name: "not-local-pv-789"}: &Disk{Name: "not-local-pv-789"},
+			},
+		},
+		{
+			name: "No local PVs to filter",
+			input: map[DiskIdentifier]*Disk{
+				{Cluster: "cluster1", Name: "pv1"}: &Disk{Name: "pv1"},
+				{Cluster: "cluster2", Name: "pv2"}: &Disk{Name: "pv2"},
+			},
+			expected: map[DiskIdentifier]*Disk{
+				{Cluster: "cluster1", Name: "pv1"}: &Disk{Name: "pv1"},
+				{Cluster: "cluster2", Name: "pv2"}: &Disk{Name: "pv2"},
+			},
+		},
+		{
+			name:     "Empty input",
+			input:    map[DiskIdentifier]*Disk{},
+			expected: map[DiskIdentifier]*Disk{},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := filterOutLocalPVs(tc.input)
+			assert.Equal(t, tc.expected, result)
+		})
+	}
+}

pkg/costmodel/costmodel.go

@@ -182,7 +182,7 @@ const (
 	queryGPURequestsStr = `avg(
 		label_replace(
 			label_replace(
-				sum_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!="", %s}[%s] %s), 
+				sum_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!="", %s}[%s] %s),
 				"container_name","$1","container","(.+)"
 			), "pod_name","$1","pod","(.+)"
 		)
@@ -988,15 +988,6 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 	nodeList := cm.Cache.GetAllNodes()
 	nodes := make(map[string]*costAnalyzerCloud.Node)
 
-	vgpuCount, err := getAllocatableVGPUs(cm.Cache)
-	if err != nil {
-		return nil, err
-	}
-	vgpuCoeff := 10.0
-	if vgpuCount > 0.0 {
-		vgpuCoeff = vgpuCount
-	}
-
 	pmd := &costAnalyzerCloud.PricingMatchMetadata{
 		TotalNodes:        0,
 		PricingTypeCounts: make(map[costAnalyzerCloud.PricingType]int),
@@ -1028,6 +1019,8 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 			pmd.PricingTypeCounts[cnode.PricingType] = 1
 		}
 
+		// newCnode builds upon cnode but populates/overrides certain fields.
+		// cnode was populated leveraging cloud provider public pricing APIs.
 		newCnode := *cnode
 		if newCnode.InstanceType == "" {
 			it, _ := util.GetInstanceType(n.Labels)
@@ -1070,48 +1063,24 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 
 		newCnode.RAMBytes = fmt.Sprintf("%f", ram)
 
-		// Azure does not seem to provide a GPU count in its pricing API. GKE supports attaching multiple GPUs
-		// So the k8s api will often report more accurate results for GPU count under status > capacity > nvidia.com/gpu than the cloud providers billing data
-		// not all providers are guaranteed to use this, so don't overwrite a Provider assignment if we can't find something under that capacity exists
-		gpuc := 0.0
-		q, ok := n.Status.Capacity["nvidia.com/gpu"]
-		_, hasReplicas := n.Labels["nvidia.com/gpu.replicas"]
-
-		if ok && !hasReplicas {
-			gpuCount := q.Value()
-			if gpuCount != 0 {
-				newCnode.GPU = fmt.Sprintf("%d", gpuCount)
-				newCnode.VGPU = newCnode.GPU
-				gpuc = float64(gpuCount)
-			}
-		} else if hasReplicas { // See https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/gpu-sharing.html
-			if q.Value() == 0 {
-				q = n.Status.Capacity["nvidia.com/gpu.shared"]
-			}
-			g, ok := n.Labels["nvidia.com/gpu.count"]
-			if ok {
-				newCnode.GPU = g
-			} else {
-				newCnode.GPU = fmt.Sprintf("%d", 0)
-			}
-			newCnode.VGPU = fmt.Sprintf("%d", q.Value())
+		gpuc, err := strconv.ParseFloat(newCnode.GPU, 64)
+		if err != nil {
+			gpuc = 0.0
+		}
 
-		} else if g, ok := n.Status.Capacity["k8s.amazonaws.com/vgpu"]; ok {
-			gpuCount := g.Value()
-			if gpuCount != 0 {
-				newCnode.GPU = fmt.Sprintf("%d", int(float64(gpuCount)/vgpuCoeff))
-				newCnode.VGPU = fmt.Sprintf("%d", gpuCount)
-				gpuc = float64(gpuCount) / vgpuCoeff
-			}
-		} else {
-			gpuc, err = strconv.ParseFloat(newCnode.GPU, 64)
-			if err != nil {
-				gpuc = 0.0
-			}
+		// The k8s API will often report more accurate results for GPU count
+		// than cloud provider public pricing APIs. If found, override the
+		// original value.
+		gpuOverride, vgpuOverride, err := getGPUCount(cm.Cache, n)
+		if err != nil {
+			log.Warnf("Unable to get GPUCount for node %s: %s", n.Name, err.Error())
 		}
-		if math.IsNaN(gpuc) {
-			log.Warnf("gpu count parsed as NaN. Setting to 0.")
-			gpuc = 0.0
+		if gpuOverride > 0 {
+			newCnode.GPU = fmt.Sprintf("%f", gpuOverride)
+			gpuc = gpuOverride
+		}
+		if vgpuOverride > 0 {
+			newCnode.VGPU = fmt.Sprintf("%f", vgpuOverride)
 		}
 
 		// Special case for SUSE rancher, since it won't behave with normal
@@ -2358,6 +2327,68 @@ func getStatefulSetsOfPod(pod v1.Pod) []string {
 	return []string{}
 }
 
+// getGPUCount reads the node's Status and Labels (via the k8s API) to identify
+// the number of GPUs and vGPUs are equipped on the node. If unable to identify
+// a GPU count, it will return -1.
+func getGPUCount(cache clustercache.ClusterCache, n *v1.Node) (float64, float64, error) {
+	g, hasGpu := n.Status.Capacity["nvidia.com/gpu"]
+	_, hasReplicas := n.Labels["nvidia.com/gpu.replicas"]
+
+	// Case 1: Standard NVIDIA GPU
+	if hasGpu && g.Value() != 0 && !hasReplicas {
+		return float64(g.Value()), float64(g.Value()), nil
+	}
+
+	// Case 2: NVIDIA GPU with GPU Feature Discovery (GFD) Pod enabled.
+	// Ref: https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/gpu-sharing.html#verifying-the-gpu-time-slicing-configuration
+	// Ref: https://github.com/NVIDIA/k8s-device-plugin/blob/d899752a424818428f744a946d32b132ea2c0cf1/internal/lm/resource_test.go#L44-L45
+	// Ref: https://github.com/NVIDIA/k8s-device-plugin/blob/d899752a424818428f744a946d32b132ea2c0cf1/internal/lm/resource_test.go#L103-L118
+	if hasReplicas {
+		resultGPU := 0.0
+		resultVGPU := 0.0
+
+		if c, ok := n.Labels["nvidia.com/gpu.count"]; ok {
+			var err error
+			resultGPU, err = strconv.ParseFloat(c, 64)
+			if err != nil {
+				return -1, -1, fmt.Errorf("could not parse label \"nvidia.com/gpu.count\": %v", err)
+			}
+		}
+
+		if s, ok := n.Status.Capacity["nvidia.com/gpu.shared"]; ok { // GFD configured `renameByDefault=true`
+			resultVGPU = float64(s.Value())
+		} else if g, ok := n.Status.Capacity["nvidia.com/gpu"]; ok { // GFD configured `renameByDefault=false`
+			resultVGPU = float64(g.Value())
+		} else {
+			resultVGPU = resultGPU
+		}
+
+		return resultGPU, resultVGPU, nil
+	}
+
+	// Case 3: AWS vGPU
+	if vgpu, ok := n.Status.Capacity["k8s.amazonaws.com/vgpu"]; ok {
+		vgpuCount, err := getAllocatableVGPUs(cache)
+		if err != nil {
+			return -1, -1, err
+		}
+
+		vgpuCoeff := 10.0
+		if vgpuCount > 0.0 {
+			vgpuCoeff = vgpuCount
+		}
+
+		if vgpu.Value() != 0 {
+			resultGPU := float64(vgpu.Value()) / vgpuCoeff
+			resultVGPU := float64(vgpu.Value())
+			return resultGPU, resultVGPU, nil
+		}
+	}
+
+	// No GPU found
+	return -1, -1, nil
+}
+
 func getAllocatableVGPUs(cache clustercache.ClusterCache) (float64, error) {
 	daemonsets := cache.GetAllDaemonSets()
 	vgpuCount := 0.0
@@ -2408,7 +2439,7 @@ func measureTimeAsync(start time.Time, threshold time.Duration, name string, ch
 	}
 }
 
-func (cm *CostModel) QueryAllocation(window opencost.Window, resolution, step time.Duration, aggregate []string, includeIdle, idleByNode, includeProportionalAssetResourceCosts, includeAggregatedMetadata, sharedLoadBalancer bool, accumulateBy opencost.AccumulateOption) (*opencost.AllocationSetRange, error) {
+func (cm *CostModel) QueryAllocation(window opencost.Window, resolution, step time.Duration, aggregate []string, includeIdle, idleByNode, includeProportionalAssetResourceCosts, includeAggregatedMetadata, sharedLoadBalancer bool, accumulateBy opencost.AccumulateOption, shareIdle bool) (*opencost.AllocationSetRange, error) {
 	// Validate window is legal
 	if window.IsOpen() || window.IsNegative() {
 		return nil, fmt.Errorf("illegal window: %s", window)
@@ -2479,10 +2510,18 @@ func (cm *CostModel) QueryAllocation(window opencost.Window, resolution, step ti
 	}
 
 	// Set aggregation options and aggregate
+	var shareIdleOpt string
+	if shareIdle {
+		shareIdleOpt = opencost.ShareWeighted
+	} else {
+		shareIdleOpt = opencost.ShareNone
+	}
+
 	opts := &opencost.AllocationAggregationOptions{
 		IncludeProportionalAssetResourceCosts: includeProportionalAssetResourceCosts,
 		IdleByNode:                            idleByNode,
 		IncludeAggregatedMetadata:             includeAggregatedMetadata,
+		ShareIdle:                             shareIdleOpt,
 	}
 
 	// Aggregate

pkg/costmodel/costmodel_test.go

@@ -4,8 +4,95 @@ import (
 	"testing"
 
 	"github.com/opencost/opencost/core/pkg/util"
+	"github.com/stretchr/testify/assert"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+func TestGetGPUCount(t *testing.T) {
+	tests := []struct {
+		name          string
+		node          *v1.Node
+		expectedGPU   float64
+		expectedVGPU  float64
+		expectedError bool
+	}{
+		{
+			name: "Standard NVIDIA GPU",
+			node: &v1.Node{
+				Status: v1.NodeStatus{
+					Capacity: v1.ResourceList{
+						"nvidia.com/gpu": resource.MustParse("2"),
+					},
+				},
+			},
+			expectedGPU:  2.0,
+			expectedVGPU: 2.0,
+		},
+		{
+			name: "NVIDIA GPU with GFD - renameByDefault=true",
+			node: &v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"nvidia.com/gpu.replicas": "4",
+						"nvidia.com/gpu.count":    "1",
+					},
+				},
+				Status: v1.NodeStatus{
+					Capacity: v1.ResourceList{
+						"nvidia.com/gpu.shared": resource.MustParse("4"),
+					},
+				},
+			},
+			expectedGPU:  1.0,
+			expectedVGPU: 4.0,
+		},
+		{
+			name: "NVIDIA GPU with GFD - renameByDefault=false",
+			node: &v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"nvidia.com/gpu.replicas": "4",
+						"nvidia.com/gpu.count":    "1",
+					},
+				},
+				Status: v1.NodeStatus{
+					Capacity: v1.ResourceList{
+						"nvidia.com/gpu": resource.MustParse("4"),
+					},
+				},
+			},
+			expectedGPU:  1.0,
+			expectedVGPU: 4.0,
+		},
+		{
+			name: "No GPU",
+			node: &v1.Node{
+				Status: v1.NodeStatus{
+					Capacity: v1.ResourceList{},
+				},
+			},
+			expectedGPU:  -1.0,
+			expectedVGPU: -1.0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gpu, vgpu, err := getGPUCount(nil, tt.node)
+
+			if tt.expectedError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.expectedGPU, gpu)
+				assert.Equal(t, tt.expectedVGPU, vgpu)
+			}
+		})
+	}
+}
+
 func Test_CostData_GetController_CronJob(t *testing.T) {
 	cases := []struct {
 		name string

spec/opencost-specv01.md

@@ -15,7 +15,7 @@ As Kubernetes adoption increases within an organization, these complexities beco
 
 ## Foundational definitions
 
-**Total Cluster Costs** represent all costs required to operate a Kubernetes cluster. **Cluster Assets Costs** are the portion of these costs that are related to directly observable entities within a cluster; these include expenses from nodes, persistent volumes, attached disks, load balancers, and network ingress/egress costs. From a financial accounting perspective, these are equivalent to the Cost of Goods Sold when measuring product costs. **Cluster Overhead Costs** measure the overhead required to operate all of the Assets of a cluster, e.g. Cluster Management Fees. These are the equivalent to Selling, General and Administrative Expenses (SG&A), or indirect costs, when viewed from a financial accounting perspective.
+**Total Cluster Costs** represent all costs required to operate a Kubernetes cluster. **Cluster Assets Costs** are the portion of these costs that are related to directly observable entities within a cluster; these include expenses from nodes, persistent volumes, attached disks, load balancers, and network ingress/egress costs. From a financial accounting perspective, these are equivalent to the Cost of Goods Sold when measuring product costs. **Cluster Overhead Costs** measure the overhead required to operate all of the Assets of a cluster, e.g. Cluster Management Fees. These are the equivalent of Selling, General, and Administrative Expenses (SG&A), or indirect costs when viewed from a financial accounting perspective.
 
 
 <table>
@@ -33,10 +33,8 @@ As Kubernetes adoption increases within an organization, these complexities beco
   </tr>
 </table>
 
-
 Cluster Asset Costs can be further segmented into **Resource Allocation Costs** and **Resource Usage Costs**. Resource Allocation Costs are expenses that accumulate based on the amount of time provisioned irrespective of usage (e.g. CPU hourly rate) whereas Resource Usage Costs accumulate on a per-unit basis (e.g. cost per byte egressed). Costs for an individual Asset are the summation of its Resource Allocation and Usage Costs, e.g. a Node’s cost is equal to CPU cost + GPU cost + RAM cost + Node Network Costs
 
-
 <table>
   <tr>
    <td><strong>Total Cluster Costs</strong>
@@ -65,19 +63,16 @@ Cluster Asset Costs can be further segmented into **Resource Allocation Costs**
   </tr>
 </table>
 
-
 The following chart shows these relationships:
 
 <img width="796" alt="image4" src="https://user-images.githubusercontent.com/453512/171577990-8f7c9a53-f5b1-4fbc-b2f6-75cd6ea67960.png"/>
 
-While billing models can differ by environment, below are common examples of segmentation by Allocation, Usage and Overhead Costs.
+While billing models can differ by environment, below are common examples of segmentation by Allocation, Usage, and Overhead Costs.
 
 <img width="292" alt="image1" src="https://user-images.githubusercontent.com/453512/171578190-d84dc3a7-1d20-4575-9bcc-2a5722de5eea.png"/>
 
-
 Once calculated, these Asset Costs can then be distributed to the tenants that consume them, where Workload Costs plus Idle Costs equals Asset Costs. **Workload costs** are expenses that can be directly attributed to a set of Kubernetes workloads, e.g. a container, pod, deployment, etc. **Cluster Idle Costs** are the portion of Resource Allocation Costs that are not allocated to any workload[^1].
 
-
 <table>
   <tr>
    <td><strong>Total Cluster Costs</strong>
@@ -104,22 +99,18 @@ The following chart shows these relationships:
 
 ## Cluster Asset Costs
 
-Cluster Assets are observable entities within a Kubernetes cluster that directly incur costs related to their resources. Asset Costs consist of Resource Allocation Costs and Resource Usage Costs. Every Asset conforming to this specification MUST include at least one cost component with Amount, Unit and Rate attributes as well as a TotalCost value.
+Cluster Assets are observable entities within a Kubernetes cluster that directly incur costs related to their resources. Asset Costs consist of Resource Allocation Costs and Resource Usage Costs. Every Asset conforming to this specification MUST include at least one cost component with Amount, Unit, and Rate attributes as well as a TotalCost value.
 
 Attributes for measured Resource Allocation Costs:
 
-
-
 * [float] Amount - the amount of resource reserved by the asset, e.g. 2 CPU cores
 * [float] Duration - time between the start and end of the allocation period measured in hours, e.g. 24 hours
 * [string] Unit - the amount’s unit of measurement, e.g. CPU cores
-* [float] HourlyRate - cost per one unit hour, e.g. $0.2 per CPU hourly rate
+* [float] HourlyRate - cost per one unit hour, e.g. $0.20 per CPU hourly rate
 * [float] Total Cost - defined as Amount * Duration * HourlyRate
 
 Attributes for measured Resource Usage Costs:
 
-
-
 * [float] Amount - the amount of resource used, e.g. 1GB of internet egress
 * [string] Unit - the amount’s unit of measurement, e.g. GB
 * [float] UnitRate - cost per unit, e.g $ per GB egressed
@@ -127,8 +118,6 @@ Attributes for measured Resource Usage Costs:
 
 Below are example inputs when measuring asset costs over a designated time window (e.g. 24 hours) with common billing models:
 
-
-
 * **Nodes**
     * CPU allocation costs
         * cores = avg_over_time(cpu) by (node) [cores]
@@ -163,7 +152,6 @@ Below are example inputs when measuring asset costs over a designated time windo
 
 Workloads are defined as entities to which Asset Costs are committed. Some resources solely have Usage Costs, but others have Allocation Costs independent of actual usage. Workload Costs should be understood as _max(request, usage)_ when Assets have Resource Allocation Costs, e.g. CPU or GPU. This formula effectively assigns costs that have been directly reserved or allocated by _kube-scheduler_. Workload Costs should be calculated at the lowest level possible, i.e. _container_ level[^2], and then they can be aggregated by any dimension.
 
-
 <table>
   <tr>
    <td>Resource Type
@@ -225,9 +213,7 @@ The following workload cost aggregations are supported in a complete implementat
 
 ## Shared Costs
 
-Shared Workload Costs, Cluster Idle Costs, and Overhead Costs are common examples of costs that organizations can optionally distribute amongst tenants. A common example would be system workload costs, e.g. kube-system pods, that benefit all tenants. Common methods for distributing these costs include the following:
-
-
+Shared Workload Costs, Cluster Idle Costs, and Overhead Costs are common examples of costs that organizations can optionally distribute amongst tenants. A common example would be system workload costs, e.g. _kube-system_ pods, that benefit all tenants. Common methods for distributing these costs include the following:
 
 1. Uniformly across other tenants
 2. Proportionate to a tenant's consumption of Cluster Asset costs
@@ -235,10 +221,9 @@ Shared Workload Costs, Cluster Idle Costs, and Overhead Costs are common example
 
 A full implementation of the spec should support various methods of distributing shared costs.
 
-
 ## Idle Costs
 
-Idle Costs can be calculated at both the Asset/Resource level as well as the Workload level. Asset Idle Costs represent the cost-weighted difference between Cluster Asset Costs and costs of resources being allocated or consumed. Idle Costs and then Idle Percentage can be calculated as follows:
+Idle Costs can be calculated at both the Asset/Resource level as well as the Workload level. Asset Idle Costs represent the cost-weighted difference between Cluster Asset Costs and the costs of resources being allocated or consumed. Idle Costs and then Idle Percentage can be calculated as follows:
 
 
 <table>
@@ -256,8 +241,6 @@ Idle Costs can be calculated at both the Asset/Resource level as well as the Wor
   </tr>
 </table>
 
-
-
 <table>
   <tr>
    <td><strong>Cluster </strong>
@@ -276,14 +259,12 @@ Idle Costs can be calculated at both the Asset/Resource level as well as the Wor
   </tr>
 </table>
 
-
-
 ##
 The following chart shows these relationships:
 ![image3](https://user-images.githubusercontent.com/453512/171579570-055bebe8-cc97-4129-9238-c4bcda8e123c.png)
 
 
-Asset Idle Cost can be calculated by individual assets, groups of assets, cluster(s), and by individual resources, e.g. CPU. Resources that are strictly billed on usage can be viewed to have 100% efficiency but should not be included when measuring idle percentage of a cluster.
+Asset Idle Cost can be calculated by individual assets, groups of assets, cluster(s), and by individual resources, e.g. CPU. Resources that are strictly billed on usage can be viewed to have 100% efficiency but should not be included when measuring the idle percentage of a cluster.
 
 Workload Idle Costs is a cost-weighted measurement of [requested](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container) resources that are unused. Workload Idle Costs can be calculated on any grouping of Kubernetes workloads, e.g. containers, pods, labels, annotations, namespaces, etc.
 
@@ -299,19 +280,19 @@ The state of a pod will affect the ability to assign costs and whether a resourc
 ## Glossary
 
 
-**Cluster Assets** – Observable entities within a Kubernetes cluster that directly incur costs related to their resources. Examples include nodes, persistent volumes, attached disks, load balancers.
+**Cluster Assets** – Observable entities within a Kubernetes cluster that directly incur costs related to their resources. Examples include nodes, persistent volumes, attached disks, and load balancers.
 
 
 **Container** - An instance of a container image. You may have multiple copies of the same image running at the same time. [More info](https://kubernetes.io/docs/concepts/containers/)
 
 
-**Image** - A template of a container which contains software (usually microservices) that needs to be run. [More info](https://kubernetes.io/docs/concepts/containers/images/)
+**Image** - A template of a container that contains software (usually microservices) that needs to be run. [More info](https://kubernetes.io/docs/concepts/containers/images/)
 
 
 **Server / Instance / Node / Node Pool** - A machine (possibly cloud or on-prem, physical or virtual) in this context used by Kubernetes [More info](https://kubernetes.io/docs/concepts/architecture/nodes/)
 
 
-**Pod** - A Kubernetes specific concept that consists of a group of containers. A pod is treated as a single block of resources that may be scheduled or scaled on a cluster. [More info](https://kubernetes.io/docs/concepts/workloads/pods/)
+**Pod** - A Kubernetes-specific concept that consists of a group of containers. A pod is treated as a single block of resources that may be scheduled or scaled on a cluster. [More info](https://kubernetes.io/docs/concepts/workloads/pods/)
 
 
 **Container Orchestration** - Manages the cluster of server instances and maintains the lifecycle of containers and pods. Scheduling is a function of the container orchestrator which schedules pods/containers to run on a server instance.
@@ -320,30 +301,30 @@ The state of a pod will affect the ability to assign costs and whether a resourc
 **Cluster** - A group of server instances
 
 
-**Namespace** - A Kubernetes concept which creates a ‘virtual’ cluster where pods/containers may be deployed and observed discreetly from other namespaces. [More info](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/)
+**Namespace** - A Kubernetes concept that creates a ‘virtual’ cluster where pods/containers may be deployed and observed discreetly from other namespaces. [More info](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/)
 
 
-**Pod Labels** - Key / Value pairs which may be used to identify objects that are meaningful to the user. There is no semantic meaning to the core of the system. Labels are typically used where a grouping of multiple namespaces need to be associated with a workload. [More info](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)
+**Pod Labels** - Key / Value pairs that may be used to identify objects that are meaningful to the user. There is no semantic meaning to the core of the system. Labels are typically used where a grouping of multiple namespaces need to be associated with a workload. [More info](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)
 
 
 ## Appendix A
 
-Various cloud providers supply an hourly resource cost directly in their user billing model.The OpenCost model recommends utilizing the fully Amortized Net Cost for each resource as an input when this is the case. When explicit RAM, CPU or GPU prices are not provided by a cloud provider, the OpenCost model needs to derive these values. The recommendation is to use a scalable ratio of CPU, GPU, RAM and other price inputs. These default values should be based on the marginal resource rates of the provider by family.
+Various cloud providers supply an hourly resource cost directly in their user billing model. The OpenCost model recommends utilizing the fully Amortized Net Cost for each resource as an input when this is the case. When explicit RAM, CPU or GPU prices are not provided by a cloud provider, the OpenCost model needs to derive these values. The recommendation is to use a scalable ratio of CPU, GPU, RAM, and other price inputs. These default values should be based on the marginal resource rates of the provider by family.
 
 One approach for calculating is to ensure the sum of each component is equal to the total price of the Asset (e.g. node) based on billing rates from your provider. When the sum of resources (e.g. RAM/CPU/GPU) cost is greater (or less) than the price of the node, then the ratio between the input prices is held constant but the total value is adjusted.
 
-As an example, you have provisioned a node with 1 GPU, 1 CPU and 1 GB of RAM that costs $35/mo. If your base GPU price is $30, base CPU price is $30, and RAM GB price is $10, based on the average marginal costs across instances in this family class, then these inputs will be normalized to $15 for GPU, $15 for CPU and $5 for RAM so that the sum equals the cost of the node. Note that the price of a GPU, as well as the price of a CPU remain 3x the price of a Gb of RAM.
+As an example, you have provisioned a node with 1 GPU, 1 CPU, and 1 GB of RAM that costs $35/mo. If your base GPU price is $30, base CPU price is $30, and RAM GB price is $10, based on the average marginal costs across instances in this family class, then these inputs will be normalized to $15 for GPU, $15 for CPU and $5 for RAM so that the sum equals the cost of the node. Note that the price of a GPU, as well as the price of a CPU remain 3x the price of a GB of RAM.
 
 
 ## Appendix B
 
-Sampling Kubernetes resources is recommended with the following metrics / datasources:
+Sampling Kubernetes resources is recommended with the following metrics/data sources:
 
 
 
-* container_cpu_usage_seconds_total – sample from cAdvisor
+* container_cpu/usage_seconds_total – sample from cAdvisor
 * container_memory_working_set_bytes –  sampled from cAdvisor
-* gpu_usage – sampled via chipset specific metrics
+* gpu_usage – sampled via chipset-specific metrics
 * cpu_requested – sampled from kube API
 * ram_requested – sampled from kube API
 * gpu_requested – sampled from kube API
@@ -357,7 +338,7 @@ Working examples of OpenCost data to come!
 ## Notes
 
 [^1]:
-     Resource **usage** costs cannot be part of idle cost because they are always used, the corresponding resource never "sits idle."
+     Resource **usage** costs cannot be part of the idle cost because they are always used, the corresponding resource never "sits idle."
 
 [^2]:
      This is because containers are the smallest identifiable unit of "thing that uses resources." For example, the lowest level of reliable CPU usage information is usually a container.