Merge branch 'develop' into mattray-patch-2

.github/workflows/build-and-publish-release.yml

@@ -57,12 +57,11 @@ jobs:
         id: branch
         run: |
           VERSION_NUMBER=${{ steps.version_number.outputs.RELEASE_VERSION }}
-          echo "BRANCH_NAME=v${VERSION_NUMBER%.*}" >> $GITHUB_ENV
+          echo "BRANCH_NAME=v${VERSION_NUMBER%.*}" >> $GITHUB_OUTPUT
 
       - name: Checkout Repo
         uses: actions/checkout@v4
         with:
-          repository: 'opencost/opencost'
           ref: '${{ steps.branch.outputs.BRANCH_NAME }}'
           path: ./opencost
 
@@ -85,9 +84,9 @@ jobs:
       - name: Set OpenCost Image Tags
         id: tags
         run: |
-          echo "IMAGE_TAG=ghcr.io/opencost/opencost:${{ steps.sha.outputs.OC_SHORTHASH }}" >> $GITHUB_OUTPUT
-          echo "IMAGE_TAG_LATEST=ghcr.io/opencost/opencost:latest" >> $GITHUB_OUTPUT
-          echo "IMAGE_TAG_VERSION=ghcr.io/opencost/opencost:${{ steps.version_number.outputs.RELEASE_VERSION }}" >> $GITHUB_OUTPUT
+          echo "IMAGE_TAG=ghcr.io/${{ github.repository_owner }}/opencost:${{ steps.sha.outputs.OC_SHORTHASH }}" >> $GITHUB_OUTPUT
+          echo "IMAGE_TAG_LATEST=ghcr.io/${{ github.repository_owner }}/opencost:latest" >> $GITHUB_OUTPUT
+          echo "IMAGE_TAG_VERSION=ghcr.io/${{ github.repository_owner }}/opencost:${{ steps.version_number.outputs.RELEASE_VERSION }}" >> $GITHUB_OUTPUT
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3

.github/workflows/build-test.yaml

@@ -87,9 +87,9 @@ jobs:
       - name: Upload code coverage
         uses: actions/upload-artifact@v4
         with:
-          name: oc-code-coverage
+          name: code-coverage
           path: |
            coverage.out
            pr_num.txt
            base.txt
-           head.txt
+           head.txt

.github/workflows/sonar.yaml

@@ -1,4 +1,4 @@
-name: Sonar Code Coverage Upload
+name: Sonar
 on:
   workflow_run:
     workflows: ["Build/Test"]
@@ -8,43 +8,74 @@ jobs:
     name: Sonar
     runs-on: ubuntu-latest
     if: github.event.workflow_run.conclusion == 'success'
+    permissions:
+      checks: write
+      contents: read
+      actions: read
     steps:
+      - uses: LouisBrunner/checks-action@v2.0.0
+        if: always()
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          name: Quality Gate
+          status: in_progress
+          sha: ${{ github.event.workflow_run.head_sha }}
       - uses: actions/checkout@v4
         with:
           repository: ${{ github.event.workflow_run.head_repository.full_name }}
           ref: ${{ github.event.workflow_run.head_branch }}
           fetch-depth: 0
-      - name: 'Download code coverage'
-        uses: actions/github-script@v7
+      - name: Download coverage artifacts
+        uses: actions/download-artifact@v4
         with:
-          script: |
-            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               run_id: context.payload.workflow_run.id,
-            });
-            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
-              return artifact.name == "oc-code-coverage"
-            })[0];
-            let download = await github.rest.actions.downloadArtifact({
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               artifact_id: matchArtifact.id,
-               archive_format: 'zip',
-            });
-            let fs = require('fs');
-            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/oc-code-coverage.zip`, Buffer.from(download.data));
-      - name: 'Unzip code coverage'
-        run: unzip oc-code-coverage.zip -d coverage
-      - name: set env vars 
+          name: code-coverage
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ github.token }}
+          path: pr-artifact
+      - name: Validate Coverage Vars
+        id: validate-vars
+        if: github.event.workflow_run.head_branch != 'develop'
+        shell: bash
+        run: | 
+          # check the PR number
+          pr_content=$(cat pr-artifact/pr_num.txt | tr -d '\n' | tr -d ' ')
+
+          # Check if the content matches a single number
+          if [[ "$pr_content" =~ ^[0-9]+$ ]]; then
+            echo "The file 'pr_num.txt' contains a single number: $pr_content"
+          else
+            echo "The file 'pr_num.txt' does not contain a single number."
+            exit 1
+          fi
+
+          base_content=$(cat pr-artifact/base.txt | tr -d '\n' | tr -d ' ')
+          if git check-ref-format --allow-onelevel "$base_content"; then
+            echo "The file 'base.txt' contains a valid git ref: $base_content"
+          else
+            echo "The file 'base.txt' does not contain a valid git ref: $base_content"
+            exit 1
+          fi
+
+          head_content=$(cat pr-artifact/head.txt | tr -d '\n' | tr -d ' ')
+          if git check-ref-format --allow-onelevel "$head_content"; then
+            echo "The file 'head.txt' contains a valid git ref: $head_content"
+          else
+            echo "The file 'head.txt' does not contain a valid git ref: $head_content"
+            exit 1
+          fi
+
+      - name: set vars 
+        id: set-vars
         run: | 
-          echo "SONAR_PR_NUM=$(cat coverage/pr_num.txt)" >> $GITHUB_ENV
-          echo "SONAR_BASE=$(cat coverage/base.txt)" >> $GITHUB_ENV
-          echo "SONAR_HEAD=$(cat coverage/head.txt)" >> $GITHUB_ENV
+          echo "SONAR_PR_NUM=$(cat pr-artifact/pr_num.txt | tr -d '\n' | tr -d ' ')" >> $GITHUB_OUTPUT
+          echo "SONAR_BASE=$(cat pr-artifact/base.txt | tr -d '\n' | tr -d ' ')" >> $GITHUB_OUTPUT
+          echo "SONAR_HEAD=$(cat pr-artifact/head.txt | tr -d '\n' | tr -d ' ')" >> $GITHUB_OUTPUT
+          # move coverage file to root where sonar properties file is expecting it
+          cp pr-artifact/coverage.out coverage.out
       # on develop branch, only run a baseline scan
       - name: SonarCloud Scan (Baseline)
         uses: sonarsource/sonarcloud-github-action@master
-        if: env.SONAR_HEAD == 'develop'
+        if: github.event.workflow_run.head_branch == 'develop'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
@@ -54,18 +85,48 @@ jobs:
             -Dsonar.projectKey=opencost_opencost
             -Dsonar.organization=opencost
             -Dsonar.branch.name=develop
-            -Dsonar.branch.target=develop
       - name: SonarCloud Scan (PR)
         uses: sonarsource/sonarcloud-github-action@master
-        if: env.SONAR_HEAD != 'develop'
+        if: github.event.workflow_run.head_branch != 'develop'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         with:
           args: >
             -Dsonar.scm.revision=${{ github.event.workflow_run.head_sha }}
-            -Dsonar.pullrequest.key=${{ env.SONAR_PR_NUM }}
-            -Dsonar.pullrequest.branch=${{ env.SONAR_HEAD }}
-            -Dsonar.pullrequest.base=${{ env.SONAR_BASE }}
+            -Dsonar.pullrequest.key=${{ steps.set-vars.outputs.SONAR_PR_NUM }}
+            -Dsonar.pullrequest.branch="${{ steps.set-vars.outputs.SONAR_HEAD }}"
+            -Dsonar.pullrequest.base="${{ steps.set-vars.outputs.SONAR_BASE }}"
             -Dsonar.projectKey=opencost_opencost
-            -Dsonar.organization=opencost
+            -Dsonar.organization=opencost
+      - name: SonarQube Quality Gate check
+        id: sonarqube-quality-gate-check
+        continue-on-error: true
+        uses: sonarsource/sonarqube-quality-gate-action@master
+        # fail step after specific time.
+        timeout-minutes: 5
+        env:
+             SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+             SONAR_HOST_URL: "https://sonarcloud.io"
+      - uses: LouisBrunner/checks-action@v2.0.0
+        id: fail-quality-gate
+        if: steps.sonarqube-quality-gate-check.outputs.quality-gate-status != 'PASSED'
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          name: Quality Gate
+          status: completed
+          conclusion: failure
+          sha: ${{ github.event.workflow_run.head_sha }}
+          output: |
+            {"summary":"Failed - see https://sonarcloud.io/summary/new_code?id=opencost_opencostl&pullRequest=${{ steps.set-vars.outputs.SONAR_PR_NUM }}","text_description":"Quality Gate failed. Check the [SonarCloud Dashboard](https://sonarcloud.io/dashboard?id=opencost_opencost&pullRequest=${{ steps.set-vars.outputs.SONAR_PR_NUM }}) for more details."}
+      - uses: LouisBrunner/checks-action@v2.0.0
+        id: pass-quality-gate
+        if: steps.sonarqube-quality-gate-check.outputs.quality-gate-status == 'PASSED'
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          name: Quality Gate
+          status: completed
+          conclusion: success
+          sha: ${{ github.event.workflow_run.head_sha }}
+          output: |
+            {"summary":"Passed","text_description":"Quality Gate passed. Check the [SonarCloud Dashboard](https://sonarcloud.io/dashboard?id=opencost_opencost&pullRequest=${{ steps.set-vars.outputs.SONAR_PR_NUM }}) for more details."} 

.gitignore

@@ -9,6 +9,8 @@ cmd/costmodel/costmodel-tilt
 
 pkg/cloud/azureorphan_test.go
 
+pkg/cloud/oracle/cloud-integration.json
+
 # VS Code
 .vscode
 

MAINTAINERS.md

@@ -8,9 +8,8 @@ Official list of [OpenCost Maintainers](https://github.com/orgs/opencost/teams/o
 | --------------- | --------- | ----------- | ----------- |
 | Ajay Tripathy | @AjayTripathy | Kubecost | <Ajay@kubecost.com> |
 | Alex Meijer | @ameijer | Kubecost | <ameijer@kubecost.com> |
-| Artur Khantimirov | @r2k1 | Microsoft | |
+| Artur Khantimirov | @r2k1 | Microsoft | <akhantimirov@microsoft.com> |
 | Matt Bolt | @​mbolt35 | Kubecost | <matt@kubecost.com> |
-| Matt Ray | @mattray | Kubecost | <mattray@kubecost.com> |
 | Niko Kovacevic | @nikovacevic | Kubecost | <niko@kubecost.com> |
 | Sean Holcomb | @Sean-Holcomb | Kubecost | <Sean@kubecost.com> |
 | Thomas Evans | @teevans | Kubecost | <thomas@kubecost.com> |
@@ -20,4 +19,5 @@ We would like to acknowledge previous committers and their huge contributions to
 
 | Maintainer | GitHub ID | Affiliation | Email |
 | --------------- | --------- | ----------- | ----------- |
-| Michael Dresser | @michaelmdresser | Kubecost | <michaelmdresser@gmail.com> |
+| Michael Dresser | @michaelmdresser | Kubecost (former) | <michaelmdresser@gmail.com> |
+| Matt Ray | @mattray | Kubecost (former) | <mattray@kubecost.com> |

README.md

@@ -1,7 +1,7 @@
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/6219/badge)](https://www.bestpractices.dev/projects/6219)
 
-<img src="./opencost-header.png"/>
+![](./opencost-header.png)
 
 # OpenCost — your favorite open source cost monitoring tool for Kubernetes and cloud spend
 

core/go.mod

@@ -1,6 +1,6 @@
 module github.com/opencost/opencost/core
 
-go 1.21.0
+go 1.22.0
 
 require (
 	github.com/davecgh/go-spew v1.1.1
@@ -16,7 +16,7 @@ require (
 	golang.org/x/sync v0.6.0
 	golang.org/x/text v0.14.0
 	google.golang.org/grpc v1.62.0
-	google.golang.org/protobuf v1.32.0
+	google.golang.org/protobuf v1.33.0
 	k8s.io/api v0.25.3
 	k8s.io/apimachinery v0.25.3
 )
@@ -47,8 +47,8 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
-	golang.org/x/net v0.21.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

core/go.sum

@@ -386,8 +386,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -463,8 +463,8 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -641,8 +641,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
-google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=

core/pkg/filter/cloudcost/fields.go

@@ -8,11 +8,15 @@ import (
 type CloudCostField string
 
 const (
-	FieldInvoiceEntityID CloudCostField = CloudCostField(fieldstrings.FieldInvoiceEntityID)
-	FieldAccountID       CloudCostField = CloudCostField(fieldstrings.FieldAccountID)
-	FieldProvider        CloudCostField = CloudCostField(fieldstrings.FieldProvider)
-	FieldProviderID      CloudCostField = CloudCostField(fieldstrings.FieldProviderID)
-	FieldCategory        CloudCostField = CloudCostField(fieldstrings.FieldCategory)
-	FieldService         CloudCostField = CloudCostField(fieldstrings.FieldService)
-	FieldLabel           CloudCostField = CloudCostField(fieldstrings.FieldLabel)
+	FieldInvoiceEntityID   CloudCostField = CloudCostField(fieldstrings.FieldInvoiceEntityID)
+	FieldInvoiceEntityName CloudCostField = CloudCostField(fieldstrings.FieldInvoiceEntityName)
+	FieldAccountID         CloudCostField = CloudCostField(fieldstrings.FieldAccountID)
+	FieldAccountName       CloudCostField = CloudCostField(fieldstrings.FieldAccountName)
+	FieldRegionID          CloudCostField = CloudCostField(fieldstrings.FieldRegionID)
+	FieldAvailabilityZone  CloudCostField = CloudCostField(fieldstrings.FieldAvailabilityZone)
+	FieldProvider          CloudCostField = CloudCostField(fieldstrings.FieldProvider)
+	FieldProviderID        CloudCostField = CloudCostField(fieldstrings.FieldProviderID)
+	FieldCategory          CloudCostField = CloudCostField(fieldstrings.FieldCategory)
+	FieldService           CloudCostField = CloudCostField(fieldstrings.FieldService)
+	FieldLabel             CloudCostField = CloudCostField(fieldstrings.FieldLabel)
 )

core/pkg/filter/cloudcost/parser.go

@@ -6,7 +6,11 @@ import "github.com/opencost/opencost/core/pkg/filter/ast"
 // valid left-hand comparators
 var cloudCostFilterFields []*ast.Field = []*ast.Field{
 	ast.NewField(FieldInvoiceEntityID),
+	ast.NewField(FieldInvoiceEntityName),
 	ast.NewField(FieldAccountID),
+	ast.NewField(FieldAccountName),
+	ast.NewField(FieldRegionID),
+	ast.NewField(FieldAvailabilityZone),
 	ast.NewField(FieldProvider),
 	ast.NewField(FieldProviderID),
 	ast.NewField(FieldCategory),

core/pkg/filter/cloudcost/parser_test.go

@@ -0,0 +1,76 @@
+package cloudcost
+
+import (
+	"testing"
+)
+
+func TestNewCloudCostFilterParserParse(t *testing.T) {
+	parser := NewCloudCostFilterParser()
+	testCases := map[string]struct {
+		input       string
+		expectError bool
+	}{
+		"Empty": {
+			input:       ``,
+			expectError: false,
+		},
+		"InvoiceEntityID": {
+			input:       `invoiceEntityID: "123"`,
+			expectError: false,
+		},
+		"InvoiceEntityName": {
+			input:       `invoiceEntityName: "foo"`,
+			expectError: false,
+		},
+		"AccountID": {
+			input:       `accountID: "123"`,
+			expectError: false,
+		},
+		"AccountName": {
+			input:       `accountName: "foo"`,
+			expectError: false,
+		},
+		"RegionID": {
+			input:       `regionID: "us-west-1"`,
+			expectError: false,
+		},
+		"AvailabilityZone": {
+			input:       `availabilityZone: "us-west-1a"`,
+			expectError: false,
+		},
+		"Provider": {
+			input:       `provider: "aws"`,
+			expectError: false,
+		},
+		"ProviderID": {
+			input:       `providerID: "i-123"`,
+			expectError: false,
+		},
+		"Category": {
+			input:       `category: "compute"`,
+			expectError: false,
+		},
+		"Service": {
+			input:       `service: "ec2"`,
+			expectError: false,
+		},
+		"Label": {
+			input:       `label[foo]:"bar"`,
+			expectError: false,
+		},
+		"InvalidField": {
+			input:       `foo: "bar"`,
+			expectError: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			_, err := parser.Parse(tc.input)
+			if (err != nil) != tc.expectError {
+				t.Errorf("expected error: %v, got: %v", tc.expectError, err)
+			}
+		})
+	}
+
+}

core/pkg/filter/fieldstrings/fieldstrings.go

@@ -24,8 +24,12 @@ const (
 	FieldAccount    string = "account"
 	FieldService    string = "service"
 
-	FieldInvoiceEntityID string = "invoiceEntityID"
-	FieldAccountID       string = "accountID"
+	FieldInvoiceEntityID   string = "invoiceEntityID"
+	FieldInvoiceEntityName string = "invoiceEntityName"
+	FieldAccountID         string = "accountID"
+	FieldAccountName       string = "accountName"
+	FieldRegionID          string = "regionID"
+	FieldAvailabilityZone  string = "availabilityZone"
 
 	AliasDepartment  string = "department"
 	AliasEnvironment string = "environment"

core/pkg/log/log.go

@@ -49,6 +49,14 @@ func InitLogging(showLogLevelSetMessage bool) {
 
 }
 
+func GetLogger() *zerolog.Logger {
+	return &log.Logger
+}
+
+func SetLogger(l *zerolog.Logger) {
+	log.Logger = *l
+}
+
 func GetLogLevel() string {
 	return zerolog.GlobalLevel().String()
 }

core/pkg/log/log_test.go

@@ -0,0 +1,64 @@
+package log
+
+import (
+	"bytes"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/rs/zerolog"
+)
+
+func TestGetLogger(t *testing.T) {
+	initialLogger := GetLogger()
+	if initialLogger == nil {
+		t.Error("GetLogger() returned nil")
+	}
+
+	secondLogger := GetLogger()
+	if initialLogger != secondLogger {
+		t.Error("GetLogger() returned different loggers on subsequent calls")
+	}
+}
+
+func TestSetLogger(t *testing.T) {
+	var buf bytes.Buffer
+	newLogger := zerolog.New(&buf).With().Str("test", "value").Logger()
+	SetLogger(&newLogger)
+
+	// Log a message using the global logger
+	Infof("Test message")
+
+	// Parse the logged message
+	loggedData := parseLogMessage(t, buf.String())
+
+	// Check if the "test" field is present in the logged message
+	if value, exists := loggedData["test"]; !exists || value != "value" {
+		t.Error("SetLogger() did not set the logger with expected context")
+	}
+}
+
+func TestLoggerConsistency(t *testing.T) {
+	var buf bytes.Buffer
+	newLogger := zerolog.New(&buf).With().Str("test", "consistency").Logger()
+	SetLogger(&newLogger)
+
+	// Log a message using the global logger
+	Infof("Consistency test message")
+
+	// Parse the logged message
+	loggedData := parseLogMessage(t, buf.String())
+
+	// Check if the "test" field is present in the logged message
+	if value, exists := loggedData["test"]; !exists || value != "consistency" {
+		t.Error("Logger inconsistency: Updated logger does not have expected context")
+	}
+}
+
+func parseLogMessage(t *testing.T, logMessage string) map[string]interface{} {
+	var loggedData map[string]interface{}
+	if err := json.Unmarshal([]byte(strings.TrimSpace(logMessage)), &loggedData); err != nil {
+		t.Fatalf("Failed to parse logged message: %v", err)
+	}
+	return loggedData
+}

core/pkg/opencost/allocation.go

@@ -935,7 +935,7 @@ func (a *Allocation) CPUEfficiency() float64 {
 		return a.CPUCoreUsageAverage / a.CPUCoreRequestAverage
 	}
 
-	if a.CPUCoreUsageAverage == 0.0 || a.CPUCost == 0.0 {
+	if a.CPUCoreUsageAverage == 0.0 || a.CPUTotalCost() == 0.0 {
 		return 0.0
 	}
 
@@ -954,7 +954,7 @@ func (a *Allocation) RAMEfficiency() float64 {
 		return a.RAMBytesUsageAverage / a.RAMBytesRequestAverage
 	}
 
-	if a.RAMBytesUsageAverage == 0.0 || a.RAMCost == 0.0 {
+	if a.RAMBytesUsageAverage == 0.0 || a.RAMTotalCost() == 0.0 {
 		return 0.0
 	}
 
@@ -973,7 +973,7 @@ func (a *Allocation) GPUEfficiency() float64 {
 		return a.GPUUsageAverage / a.GPURequestAverage
 	}
 
-	if a.GPUUsageAverage == 0.0 || a.GPUCost == 0.0 {
+	if a.GPUUsageAverage == 0.0 || a.GPUTotalCost() == 0.0 {
 		return 0.0
 	}
 

core/pkg/opencost/bingen.go

@@ -62,7 +62,7 @@ package opencost
 // @bingen:generate:LbAllocation
 // @bingen:end
 
-// @bingen:set[name=CloudCost,version=2]
+// @bingen:set[name=CloudCost,version=3]
 // @bingen:generate:CloudCost
 // @bingen:generate:CostMetric
 // @bingen:generate[stringtable]:CloudCostSet

core/pkg/opencost/cloudcost.go

@@ -104,8 +104,16 @@ func (cc *CloudCost) StringProperty(prop string) (string, error) {
 	switch prop {
 	case CloudCostInvoiceEntityIDProp:
 		return cc.Properties.InvoiceEntityID, nil
+	case CloudCostInvoiceEntityNameProp:
+		return cc.Properties.InvoiceEntityName, nil
 	case CloudCostAccountIDProp:
 		return cc.Properties.AccountID, nil
+	case CloudCostAccountNameProp:
+		return cc.Properties.AccountName, nil
+	case CloudCostRegionIDProp:
+		return cc.Properties.RegionID, nil
+	case CloudCostAvailabilityZoneProp:
+		return cc.Properties.AvailabilityZone, nil
 	case CloudCostProviderProp:
 		return cc.Properties.Provider, nil
 	case CloudCostProviderIDProp:

core/pkg/opencost/cloudcostmatcher.go

@@ -48,8 +48,16 @@ func cloudCostFieldMap(cc *CloudCost, identifier ast.Identifier) (string, error)
 	switch ccfilter.CloudCostField(identifier.Field.Name) {
 	case ccfilter.FieldInvoiceEntityID:
 		return cc.Properties.InvoiceEntityID, nil
+	case ccfilter.FieldInvoiceEntityName:
+		return cc.Properties.InvoiceEntityName, nil
 	case ccfilter.FieldAccountID:
 		return cc.Properties.AccountID, nil
+	case ccfilter.FieldAccountName:
+		return cc.Properties.AccountName, nil
+	case ccfilter.FieldRegionID:
+		return cc.Properties.RegionID, nil
+	case ccfilter.FieldAvailabilityZone:
+		return cc.Properties.AvailabilityZone, nil
 	case ccfilter.FieldProvider:
 		return cc.Properties.Provider, nil
 	case ccfilter.FieldProviderID:

core/pkg/opencost/cloudcostprops.go

@@ -28,14 +28,18 @@ func (apt *CloudCostProperty) GetLabel() string {
 }
 
 const (
-	CloudCostInvoiceEntityIDProp string = "invoiceEntityID"
-	CloudCostAccountIDProp       string = "accountID"
-	CloudCostProviderProp        string = "provider"
-	CloudCostProviderIDProp      string = "providerID"
-	CloudCostCategoryProp        string = "category"
-	CloudCostServiceProp         string = "service"
-	CloudCostLabelProp           string = "label"
-	CloudCostLabelSetProp        string = "labelSet"
+	CloudCostInvoiceEntityIDProp   string = "invoiceEntityID"
+	CloudCostInvoiceEntityNameProp string = "invoiceEntityName"
+	CloudCostAccountIDProp         string = "accountID"
+	CloudCostAccountNameProp       string = "accountName"
+	CloudCostRegionIDProp          string = "regionID"
+	CloudCostAvailabilityZoneProp  string = "availabilityZone"
+	CloudCostProviderProp          string = "provider"
+	CloudCostProviderIDProp        string = "providerID"
+	CloudCostCategoryProp          string = "category"
+	CloudCostServiceProp           string = "service"
+	CloudCostLabelProp             string = "label"
+	CloudCostLabelSetProp          string = "labelSet"
 )
 
 func ParseCloudProperties(props []string) ([]CloudCostProperty, error) {
@@ -61,8 +65,16 @@ func ParseCloudCostProperty(text string) (CloudCostProperty, error) {
 	switch strings.TrimSpace(strings.ToLower(text)) {
 	case "invoiceentityid":
 		return CloudCostProperty(CloudCostInvoiceEntityIDProp), nil
+	case "invoiceentityname":
+		return CloudCostProperty(CloudCostInvoiceEntityNameProp), nil
 	case "accountid":
 		return CloudCostProperty(CloudCostAccountIDProp), nil
+	case "accountname":
+		return CloudCostProperty(CloudCostAccountNameProp), nil
+	case "regionid":
+		return CloudCostProperty(CloudCostRegionIDProp), nil
+	case "availabilityzone":
+		return CloudCostProperty(CloudCostAvailabilityZoneProp), nil
 	case "provider":
 		return CloudCostProperty(CloudCostProviderProp), nil
 	case "providerid":
@@ -152,20 +164,28 @@ func (ccl CloudCostLabels) Intersection(that CloudCostLabels) CloudCostLabels {
 }
 
 type CloudCostProperties struct {
-	ProviderID      string          `json:"providerID,omitempty"`
-	Provider        string          `json:"provider,omitempty"`
-	AccountID       string          `json:"accountID,omitempty"`
-	InvoiceEntityID string          `json:"invoiceEntityID,omitempty"`
-	Service         string          `json:"service,omitempty"`
-	Category        string          `json:"category,omitempty"`
-	Labels          CloudCostLabels `json:"labels,omitempty"`
+	ProviderID        string          `json:"providerID,omitempty"`
+	Provider          string          `json:"provider,omitempty"`
+	AccountID         string          `json:"accountID,omitempty"`
+	AccountName       string          `json:"accountName,omitempty"` // @bingen:field[version=3]
+	InvoiceEntityID   string          `json:"invoiceEntityID,omitempty"`
+	InvoiceEntityName string          `json:"invoiceEntityName,omitempty"` // @bingen:field[version=3]
+	RegionID          string          `json:"regionID,omitempty"`          // @bingen:field[version=3]
+	AvailabilityZone  string          `json:"availabilityZone,omitempty"`  // @bingen:field[version=3]
+	Service           string          `json:"service,omitempty"`
+	Category          string          `json:"category,omitempty"`
+	Labels            CloudCostLabels `json:"labels,omitempty"`
 }
 
 func (ccp *CloudCostProperties) Equal(that *CloudCostProperties) bool {
 	return ccp.ProviderID == that.ProviderID &&
 		ccp.Provider == that.Provider &&
 		ccp.AccountID == that.AccountID &&
+		ccp.AccountName == that.AccountName &&
 		ccp.InvoiceEntityID == that.InvoiceEntityID &&
+		ccp.InvoiceEntityName == that.InvoiceEntityName &&
+		ccp.RegionID == that.RegionID &&
+		ccp.AvailabilityZone == that.AvailabilityZone &&
 		ccp.Service == that.Service &&
 		ccp.Category == that.Category &&
 		ccp.Labels.Equal(that.Labels)
@@ -173,13 +193,17 @@ func (ccp *CloudCostProperties) Equal(that *CloudCostProperties) bool {
 
 func (ccp *CloudCostProperties) Clone() *CloudCostProperties {
 	return &CloudCostProperties{
-		ProviderID:      ccp.ProviderID,
-		Provider:        ccp.Provider,
-		AccountID:       ccp.AccountID,
-		InvoiceEntityID: ccp.InvoiceEntityID,
-		Service:         ccp.Service,
-		Category:        ccp.Category,
-		Labels:          ccp.Labels.Clone(),
+		ProviderID:        ccp.ProviderID,
+		Provider:          ccp.Provider,
+		AccountID:         ccp.AccountID,
+		AccountName:       ccp.AccountName,
+		InvoiceEntityID:   ccp.InvoiceEntityID,
+		InvoiceEntityName: ccp.InvoiceEntityName,
+		RegionID:          ccp.RegionID,
+		AvailabilityZone:  ccp.AvailabilityZone,
+		Service:           ccp.Service,
+		Category:          ccp.Category,
+		Labels:            ccp.Labels.Clone(),
 	}
 }
 
@@ -206,9 +230,21 @@ func (ccp *CloudCostProperties) Intersection(that *CloudCostProperties) *CloudCo
 	if ccp.AccountID == that.AccountID {
 		intersectionCCP.AccountID = ccp.AccountID
 	}
+	if ccp.AccountName == that.AccountName {
+		intersectionCCP.AccountName = ccp.AccountName
+	}
 	if ccp.InvoiceEntityID == that.InvoiceEntityID {
 		intersectionCCP.InvoiceEntityID = ccp.InvoiceEntityID
 	}
+	if ccp.InvoiceEntityName == that.InvoiceEntityName {
+		intersectionCCP.InvoiceEntityName = ccp.InvoiceEntityName
+	}
+	if ccp.RegionID == that.RegionID {
+		intersectionCCP.RegionID = ccp.RegionID
+	}
+	if ccp.AvailabilityZone == that.AvailabilityZone {
+		intersectionCCP.AvailabilityZone = ccp.AvailabilityZone
+	}
 	if ccp.Service == that.Service {
 		intersectionCCP.Service = ccp.Service
 	}
@@ -220,15 +256,6 @@ func (ccp *CloudCostProperties) Intersection(that *CloudCostProperties) *CloudCo
 	return intersectionCCP
 }
 
-var cloudCostDefaultKeyProperties = []string{
-	CloudCostProviderProp,
-	CloudCostInvoiceEntityIDProp,
-	CloudCostAccountIDProp,
-	CloudCostCategoryProp,
-	CloudCostServiceProp,
-	CloudCostProviderIDProp,
-}
-
 // GenerateKey takes a list of properties and creates a "/" seperated key based on the values of the requested properties.
 // Invalid values are ignored with a warning. A nil input returns the default key, while an empty slice  returns the empty string
 func (ccp *CloudCostProperties) GenerateKey(props []string) string {
@@ -259,10 +286,26 @@ func (ccp *CloudCostProperties) GenerateKey(props []string) string {
 			if ccp.InvoiceEntityID != "" {
 				propVal = ccp.InvoiceEntityID
 			}
+		case prop == CloudCostInvoiceEntityNameProp:
+			if ccp.InvoiceEntityName != "" {
+				propVal = ccp.InvoiceEntityName
+			}
 		case prop == CloudCostAccountIDProp:
 			if ccp.AccountID != "" {
 				propVal = ccp.AccountID
 			}
+		case prop == CloudCostAccountNameProp:
+			if ccp.AccountName != "" {
+				propVal = ccp.AccountName
+			}
+		case prop == CloudCostRegionIDProp:
+			if ccp.RegionID != "" {
+				propVal = ccp.RegionID
+			}
+		case prop == CloudCostAvailabilityZoneProp:
+			if ccp.AvailabilityZone != "" {
+				propVal = ccp.AvailabilityZone
+			}
 		case prop == CloudCostServiceProp:
 			if ccp.Service != "" {
 				propVal = ccp.Service
@@ -298,7 +341,11 @@ func (ccp *CloudCostProperties) hashKey() string {
 	builder.WriteString(ccp.ProviderID)
 	builder.WriteString(ccp.Provider)
 	builder.WriteString(ccp.AccountID)
+	builder.WriteString(ccp.AccountName)
 	builder.WriteString(ccp.InvoiceEntityID)
+	builder.WriteString(ccp.InvoiceEntityName)
+	builder.WriteString(ccp.RegionID)
+	builder.WriteString(ccp.AvailabilityZone)
 	builder.WriteString(ccp.Service)
 	builder.WriteString(ccp.Category)
 

core/pkg/opencost/cloudcostprops_test.go

@@ -10,34 +10,46 @@ func TestCloudCostPropertiesIntersection(t *testing.T) {
 	}{
 		"When properties match between both CloudCostProperties": {
 			baseCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service1",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
 			},
 			intCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service1",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
 			},
 			expectedCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service1",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
@@ -45,34 +57,46 @@ func TestCloudCostPropertiesIntersection(t *testing.T) {
 		},
 		"When one of the properties differ in the two CloudCostProperties": {
 			baseCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service1",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
 			},
 			intCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service2",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service2",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
 			},
 			expectedCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
@@ -80,47 +104,108 @@ func TestCloudCostPropertiesIntersection(t *testing.T) {
 		},
 		"When two of the properties differ in the two CloudCostProperties": {
 			baseCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service1",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
 			},
 			intCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID2",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service2",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID2",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service2",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
 			},
 			expectedCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 				},
 			},
 		},
+		"When all properties differ in the two CloudCostProperties": {
+			baseCCP: &CloudCostProperties{
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
+				Labels: map[string]string{
+					"key1": "value1",
+				},
+			},
+			intCCP: &CloudCostProperties{
+				Provider:          "CustomProvider2",
+				ProviderID:        "ProviderID2",
+				AccountID:         "WorkGroupID2",
+				AccountName:       "AccountName2",
+				InvoiceEntityID:   "InvoiceEntityID2",
+				InvoiceEntityName: "InvoiceEntityName2",
+				RegionID:          "RegionID2",
+				AvailabilityZone:  "AvailabilityZone2",
+				Service:           "Service2",
+				Category:          "Category2",
+				Labels: map[string]string{
+					"key2": "value2",
+				},
+			},
+			expectedCCP: &CloudCostProperties{
+				Provider:          "",
+				ProviderID:        "",
+				AccountID:         "",
+				AccountName:       "",
+				InvoiceEntityID:   "",
+				InvoiceEntityName: "",
+				RegionID:          "",
+				AvailabilityZone:  "",
+				Service:           "",
+				Category:          "",
+				Labels:            map[string]string{},
+			},
+		},
 		"When labels differ": {
 			baseCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service1",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value1",
 					"key2": "value2",
@@ -128,12 +213,16 @@ func TestCloudCostPropertiesIntersection(t *testing.T) {
 				},
 			},
 			intCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service1",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key1": "value2",
 					"key2": "value2",
@@ -141,12 +230,16 @@ func TestCloudCostPropertiesIntersection(t *testing.T) {
 				},
 			},
 			expectedCCP: &CloudCostProperties{
-				Provider:        "CustomProvider",
-				ProviderID:      "ProviderID1",
-				AccountID:       "WorkGroupID1",
-				InvoiceEntityID: "InvoiceEntityID1",
-				Service:         "Service1",
-				Category:        "Category1",
+				Provider:          "CustomProvider",
+				ProviderID:        "ProviderID1",
+				AccountID:         "WorkGroupID1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "InvoiceEntityID1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "Service1",
+				Category:          "Category1",
 				Labels: map[string]string{
 					"key2": "value2",
 				},
@@ -176,45 +269,57 @@ func TestCloudCostProperties_hashKey(t *testing.T) {
 		},
 		"All props no labels": {
 			props: &CloudCostProperties{
-				ProviderID:      "providerid1",
-				Provider:        "provider1",
-				AccountID:       "workgroup1",
-				InvoiceEntityID: "billing1",
-				Service:         "service1",
-				Category:        "category1",
-				Labels:          map[string]string{},
-			},
-			want: "a19b7dddf0032572",
+				ProviderID:        "providerid1",
+				Provider:          "provider1",
+				AccountID:         "workgroup1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "billing1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "service1",
+				Category:          "category1",
+				Labels:            map[string]string{},
+			},
+			want: "d07ffd0bd6d5eaf1",
 		},
 		"All props": {
 			props: &CloudCostProperties{
-				ProviderID:      "providerid1",
-				Provider:        "provider1",
-				AccountID:       "workgroup1",
-				InvoiceEntityID: "billing1",
-				Service:         "service1",
-				Category:        "category1",
+				ProviderID:        "providerid1",
+				Provider:          "provider1",
+				AccountID:         "workgroup1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "billing1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "service1",
+				Category:          "category1",
 				Labels: map[string]string{
 					"label1": "value1",
 					"label2": "value2",
 				},
 			},
-			want: "9d54403e40ad4db6",
+			want: "318cb6294bf9e2d5",
 		},
 		"All props swap labels": {
 			props: &CloudCostProperties{
-				ProviderID:      "providerid1",
-				Provider:        "provider1",
-				AccountID:       "workgroup1",
-				InvoiceEntityID: "billing1",
-				Service:         "service1",
-				Category:        "category1",
+				ProviderID:        "providerid1",
+				Provider:          "provider1",
+				AccountID:         "workgroup1",
+				AccountName:       "AccountName1",
+				InvoiceEntityID:   "billing1",
+				InvoiceEntityName: "InvoiceEntityName1",
+				RegionID:          "RegionID1",
+				AvailabilityZone:  "AvailabilityZone1",
+				Service:           "service1",
+				Category:          "category1",
 				Labels: map[string]string{
 					"label2": "value2",
 					"label1": "value1",
 				},
 			},
-			want: "9d54403e40ad4db6",
+			want: "318cb6294bf9e2d5",
 		},
 	}
 	for name, tt := range tests {

core/pkg/opencost/opencost_codecs.go

@@ -13,12 +13,11 @@ package opencost
 
 import (
 	"fmt"
+	util "github.com/opencost/opencost/core/pkg/util"
 	"reflect"
 	"strings"
 	"sync"
 	"time"
-
-	util "github.com/opencost/opencost/core/pkg/util"
 )
 
 const (
@@ -44,7 +43,7 @@ const (
 	AllocationCodecVersion uint8 = 22
 
 	// CloudCostCodecVersion is used for any resources listed in the CloudCost version set
-	CloudCostCodecVersion uint8 = 2
+	CloudCostCodecVersion uint8 = 3
 )
 
 //--------------------------------------------------------------------------
@@ -3581,20 +3580,44 @@ func (target *CloudCostProperties) MarshalBinaryWithContext(ctx *EncodingContext
 		buff.WriteString(target.AccountID) // write string
 	}
 	if ctx.IsStringTable() {
-		d := ctx.Table.AddOrGet(target.InvoiceEntityID)
+		d := ctx.Table.AddOrGet(target.AccountName)
 		buff.WriteInt(d) // write table index
 	} else {
-		buff.WriteString(target.InvoiceEntityID) // write string
+		buff.WriteString(target.AccountName) // write string
 	}
 	if ctx.IsStringTable() {
-		e := ctx.Table.AddOrGet(target.Service)
+		e := ctx.Table.AddOrGet(target.InvoiceEntityID)
 		buff.WriteInt(e) // write table index
 	} else {
-		buff.WriteString(target.Service) // write string
+		buff.WriteString(target.InvoiceEntityID) // write string
 	}
 	if ctx.IsStringTable() {
-		f := ctx.Table.AddOrGet(target.Category)
+		f := ctx.Table.AddOrGet(target.InvoiceEntityName)
 		buff.WriteInt(f) // write table index
+	} else {
+		buff.WriteString(target.InvoiceEntityName) // write string
+	}
+	if ctx.IsStringTable() {
+		g := ctx.Table.AddOrGet(target.RegionID)
+		buff.WriteInt(g) // write table index
+	} else {
+		buff.WriteString(target.RegionID) // write string
+	}
+	if ctx.IsStringTable() {
+		h := ctx.Table.AddOrGet(target.AvailabilityZone)
+		buff.WriteInt(h) // write table index
+	} else {
+		buff.WriteString(target.AvailabilityZone) // write string
+	}
+	if ctx.IsStringTable() {
+		k := ctx.Table.AddOrGet(target.Service)
+		buff.WriteInt(k) // write table index
+	} else {
+		buff.WriteString(target.Service) // write string
+	}
+	if ctx.IsStringTable() {
+		l := ctx.Table.AddOrGet(target.Category)
+		buff.WriteInt(l) // write table index
 	} else {
 		buff.WriteString(target.Category) // write string
 	}
@@ -3608,14 +3631,14 @@ func (target *CloudCostProperties) MarshalBinaryWithContext(ctx *EncodingContext
 		buff.WriteInt(len(map[string]string(target.Labels))) // map length
 		for v, z := range map[string]string(target.Labels) {
 			if ctx.IsStringTable() {
-				g := ctx.Table.AddOrGet(v)
-				buff.WriteInt(g) // write table index
+				m := ctx.Table.AddOrGet(v)
+				buff.WriteInt(m) // write table index
 			} else {
 				buff.WriteString(v) // write string
 			}
 			if ctx.IsStringTable() {
-				h := ctx.Table.AddOrGet(z)
-				buff.WriteInt(h) // write table index
+				n := ctx.Table.AddOrGet(z)
+				buff.WriteInt(n) // write table index
 			} else {
 				buff.WriteString(z) // write string
 			}
@@ -3712,15 +3735,21 @@ func (target *CloudCostProperties) UnmarshalBinaryWithContext(ctx *DecodingConte
 	g := h
 	target.AccountID = g
 
-	var m string
-	if ctx.IsStringTable() {
-		n := buff.ReadInt() // read string index
-		m = ctx.Table[n]
+	// field version check
+	if uint8(3) <= version {
+		var m string
+		if ctx.IsStringTable() {
+			n := buff.ReadInt() // read string index
+			m = ctx.Table[n]
+		} else {
+			m = buff.ReadString() // read string
+		}
+		l := m
+		target.AccountName = l
+
 	} else {
-		m = buff.ReadString() // read string
+		target.AccountName = "" // default
 	}
-	l := m
-	target.InvoiceEntityID = l
 
 	var p string
 	if ctx.IsStringTable() {
@@ -3730,56 +3759,114 @@ func (target *CloudCostProperties) UnmarshalBinaryWithContext(ctx *DecodingConte
 		p = buff.ReadString() // read string
 	}
 	o := p
-	target.Service = o
+	target.InvoiceEntityID = o
 
-	var s string
+	// field version check
+	if uint8(3) <= version {
+		var s string
+		if ctx.IsStringTable() {
+			t := buff.ReadInt() // read string index
+			s = ctx.Table[t]
+		} else {
+			s = buff.ReadString() // read string
+		}
+		r := s
+		target.InvoiceEntityName = r
+
+	} else {
+		target.InvoiceEntityName = "" // default
+	}
+
+	// field version check
+	if uint8(3) <= version {
+		var w string
+		if ctx.IsStringTable() {
+			x := buff.ReadInt() // read string index
+			w = ctx.Table[x]
+		} else {
+			w = buff.ReadString() // read string
+		}
+		u := w
+		target.RegionID = u
+
+	} else {
+		target.RegionID = "" // default
+	}
+
+	// field version check
+	if uint8(3) <= version {
+		var aa string
+		if ctx.IsStringTable() {
+			bb := buff.ReadInt() // read string index
+			aa = ctx.Table[bb]
+		} else {
+			aa = buff.ReadString() // read string
+		}
+		y := aa
+		target.AvailabilityZone = y
+
+	} else {
+		target.AvailabilityZone = "" // default
+	}
+
+	var dd string
 	if ctx.IsStringTable() {
-		t := buff.ReadInt() // read string index
-		s = ctx.Table[t]
+		ee := buff.ReadInt() // read string index
+		dd = ctx.Table[ee]
 	} else {
-		s = buff.ReadString() // read string
+		dd = buff.ReadString() // read string
 	}
-	r := s
-	target.Category = r
+	cc := dd
+	target.Service = cc
+
+	var gg string
+	if ctx.IsStringTable() {
+		hh := buff.ReadInt() // read string index
+		gg = ctx.Table[hh]
+	} else {
+		gg = buff.ReadString() // read string
+	}
+	ff := gg
+	target.Category = ff
 
 	// --- [begin][read][alias](CloudCostLabels) ---
-	var u map[string]string
+	var kk map[string]string
 	if buff.ReadUInt8() == uint8(0) {
-		u = nil
+		kk = nil
 	} else {
 		// --- [begin][read][map](map[string]string) ---
-		x := buff.ReadInt() // map len
-		w := make(map[string]string, x)
-		for i := 0; i < x; i++ {
+		mm := buff.ReadInt() // map len
+		ll := make(map[string]string, mm)
+		for i := 0; i < mm; i++ {
 			var v string
-			var aa string
+			var oo string
 			if ctx.IsStringTable() {
-				bb := buff.ReadInt() // read string index
-				aa = ctx.Table[bb]
+				pp := buff.ReadInt() // read string index
+				oo = ctx.Table[pp]
 			} else {
-				aa = buff.ReadString() // read string
+				oo = buff.ReadString() // read string
 			}
-			y := aa
-			v = y
+			nn := oo
+			v = nn
 
 			var z string
-			var dd string
+			var rr string
 			if ctx.IsStringTable() {
-				ee := buff.ReadInt() // read string index
-				dd = ctx.Table[ee]
+				ss := buff.ReadInt() // read string index
+				rr = ctx.Table[ss]
 			} else {
-				dd = buff.ReadString() // read string
+				rr = buff.ReadString() // read string
 			}
-			cc := dd
-			z = cc
+			qq := rr
+			z = qq
 
-			w[v] = z
+			ll[v] = z
 		}
-		u = w
+		kk = ll
 		// --- [end][read][map](map[string]string) ---
 
 	}
-	target.Labels = CloudCostLabels(u)
+	target.Labels = CloudCostLabels(kk)
 	// --- [end][read][alias](CloudCostLabels) ---
 
 	return nil

core/pkg/opencost/window.go

@@ -956,7 +956,7 @@ func (w Window) getWeeklyWindow() Window {
 // getWeeklyWindows breaks up a window into weeks, with weeks starting on Sunday
 func (w Window) getWeeklyWindows() []Window {
 	wins := []Window{}
-	roundedWindow := w.getDailyWindow()
+	roundedWindow := w.getWeeklyWindow()
 
 	roundedStart := *roundedWindow.Start()
 	roundedEnd := *roundedWindow.End()

core/pkg/opencost/window_test.go

@@ -1192,3 +1192,118 @@ func TestBoundaryErrorIs(t *testing.T) {
 		t.Errorf("Multi wrap failure: %s != %s", baseError, multiWrapError)
 	}
 }
+func TestWindowGetWeeklyWindows(t *testing.T) {
+	testCases := []struct {
+		name     string
+		start    time.Time
+		end      time.Time
+		expected []Window
+	}{
+		{
+			name:  "Single week",
+			start: time.Date(2024, 9, 1, 0, 0, 0, 0, time.UTC),
+			end:   time.Date(2024, 9, 8, 0, 0, 0, 0, time.UTC),
+			expected: []Window{
+				NewClosedWindow(
+					time.Date(2024, 9, 1, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 9, 8, 0, 0, 0, 0, time.UTC),
+				),
+			},
+		},
+		{
+			name:  "Multiple weeks",
+			start: time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+			end:   time.Date(2024, 10, 20, 0, 0, 0, 0, time.UTC),
+			expected: []Window{
+				NewClosedWindow(
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 20, 0, 0, 0, 0, time.UTC),
+				),
+			},
+		},
+		{
+			name:  "Partial starting week",
+			start: time.Date(2024, 10, 1, 0, 0, 0, 0, time.UTC),
+			end:   time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+			expected: []Window{
+				NewClosedWindow(
+					time.Date(2024, 9, 29, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+				),
+			},
+		},
+		{
+			name:  "Partial ending week",
+			start: time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+			end:   time.Date(2024, 10, 7, 0, 0, 0, 0, time.UTC),
+			expected: []Window{
+				NewClosedWindow(
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+				),
+			},
+		},
+		{
+			name:  "multiple weeks, partial start and end",
+			start: time.Date(2024, 10, 1, 0, 0, 0, 0, time.UTC),
+			end:   time.Date(2024, 11, 21, 0, 0, 0, 0, time.UTC),
+			expected: []Window{
+				NewClosedWindow(
+					time.Date(2024, 9, 29, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 13, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 20, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 20, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 10, 27, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 10, 27, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 11, 3, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 11, 3, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 11, 10, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 11, 10, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 11, 17, 0, 0, 0, 0, time.UTC),
+				),
+				NewClosedWindow(
+					time.Date(2024, 11, 17, 0, 0, 0, 0, time.UTC),
+					time.Date(2024, 11, 24, 0, 0, 0, 0, time.UTC),
+				),
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			w := NewClosedWindow(tc.start, tc.end)
+			actual := w.getWeeklyWindows()
+			if len(actual) != len(tc.expected) {
+				t.Fatalf("expected %d windows, got %d", len(tc.expected), len(actual))
+			}
+			for i, expectedWindow := range tc.expected {
+				if !actual[i].Equal(expectedWindow) {
+					t.Errorf("expected window %s, got %s", expectedWindow, actual[i])
+				}
+			}
+		})
+	}
+}

core/pkg/util/worker/worker.go

@@ -9,6 +9,9 @@ import (
 	"github.com/opencost/opencost/core/pkg/collections"
 )
 
+// Runner is a function type that takes a single input and returns nothing.
+type Runner[T any] func(T)
+
 // Worker is a transformation function from input type T to output type U.
 type Worker[T any, U any] func(T) U
 
@@ -54,7 +57,7 @@ type queuedWorkerPool[T any, U any] struct {
 type ordered[T any, U any] struct {
 	workPool WorkerPool[T, U]
 	results  []U
-	wg       *sync.WaitGroup
+	wg       sync.WaitGroup
 	count    int
 }
 
@@ -129,7 +132,6 @@ func NewOrderedGroup[T any, U any](pool WorkerPool[T, U], size int) WorkGroup[T,
 	return &ordered[T, U]{
 		workPool: pool,
 		results:  make([]U, size),
-		wg:       new(sync.WaitGroup),
 		count:    0,
 	}
 }
@@ -166,6 +168,96 @@ func (ow *ordered[T, U]) Wait() []U {
 	return ow.results
 }
 
+// noResultGroup is a WorkGroup implementation which arbitrarily pushes inputs to
+// a runner pool to be executed concurrently. This group does not collect results.
+type noResultGroup[T any] struct {
+	workPool WorkerPool[T, struct{}]
+	wg       sync.WaitGroup
+}
+
+// NewNoResultGroup creates a new WorkGroup implementation for processing a group of inputs concurrently. This
+// work group implementation does not collect results, and therefore, requires a worker pool with a struct{} output.
+func NewNoResultGroup[T any](pool WorkerPool[T, struct{}]) WorkGroup[T, struct{}] {
+	return &noResultGroup[T]{
+		workPool: pool,
+	}
+}
+
+// Push adds a new input to the work group.
+func (ow *noResultGroup[T]) Push(input T) error {
+	onComplete := make(chan struct{})
+	err := ow.workPool.Run(input, onComplete)
+	if err != nil {
+		return err
+	}
+
+	ow.wg.Add(1)
+
+	go func() {
+		defer close(onComplete)
+		defer ow.wg.Done()
+
+		<-onComplete
+	}()
+
+	return nil
+}
+
+// Wait waits for all pending worker tasks to complete, then returns all the results.
+func (ow *noResultGroup[T]) Wait() []struct{} {
+	ow.wg.Wait()
+	return []struct{}{}
+}
+
+// collector is a WorkGroup implementation which collects non-nil results into the results slice
+// and ignores any nil results.
+type collector[T any, U any] struct {
+	workPool   WorkerPool[T, *U]
+	resultLock sync.Mutex
+	results    []*U
+	wg         sync.WaitGroup
+}
+
+// NewCollectionGroup creates a new WorkGroup implementation for processing a group of inputs concurrently. The
+// collection group implementation will collect all non-nil results into the output slice. Thus, the worker pool
+// parameter requires the output type to be a pointer.
+func NewCollectionGroup[T any, U any](pool WorkerPool[T, *U]) WorkGroup[T, *U] {
+	return &collector[T, U]{
+		workPool: pool,
+	}
+}
+
+// Push adds a new input to the work group.
+func (ow *collector[T, U]) Push(input T) error {
+	onComplete := make(chan *U)
+	err := ow.workPool.Run(input, onComplete)
+	if err != nil {
+		return err
+	}
+
+	ow.wg.Add(1)
+
+	go func() {
+		defer ow.wg.Done()
+		defer close(onComplete)
+
+		result := <-onComplete
+		if result != nil {
+			ow.resultLock.Lock()
+			ow.results = append(ow.results, result)
+			ow.resultLock.Unlock()
+		}
+	}()
+
+	return nil
+}
+
+// Wait waits for all pending worker tasks to complete, then returns all the results.
+func (ow *collector[T, U]) Wait() []*U {
+	ow.wg.Wait()
+	return ow.results
+}
+
 // these constraints protect against the possibility of unexpected output from runtime.NumCPU()
 const (
 	defaultMinWorkers = 4
@@ -190,10 +282,21 @@ func OptimalWorkerCountInRange(min int, max int) int {
 	return cores
 }
 
-// ConcurrentDo runs a pool of workers which concurrently call the provided worker func on each input to get ordered
-// output corresponding to the inputs
+// ConcurrentDo runs a pool of N workers which concurrently call the provided worker func on each
+// input to get ordered output corresponding to the inputs. The total number of workers is determined
+// by the total number of CPUs available, bound to a range from 4-16.
 func ConcurrentDo[T any, U any](worker Worker[T, U], inputs []T) []U {
-	workerPool := NewWorkerPool(OptimalWorkerCount(), worker)
+	return ConcurrentDoWith(OptimalWorkerCount(), worker, inputs)
+}
+
+// ConcurrentDoWith runs a pool of workers of the specified size which concurrently call the provided worker func
+// on each input to get ordered output corresponding to the inputs. Size inputs < 1 will automatically be set to 1.
+func ConcurrentDoWith[T any, U any](size int, worker Worker[T, U], inputs []T) []U {
+	if size < 1 {
+		size = 1
+	}
+
+	workerPool := NewWorkerPool(size, worker)
 	defer workerPool.Shutdown()
 
 	workGroup := NewOrderedGroup(workerPool, len(inputs))
@@ -203,3 +306,55 @@ func ConcurrentDo[T any, U any](worker Worker[T, U], inputs []T) []U {
 
 	return workGroup.Wait()
 }
+
+// ConcurrentCollect runs a pool of N workers which concurrently call the provided worker func on each
+// input to get a result slice of non-nil outputs. The total number of workers is determined
+// by the total number of CPUs available, bound to a range from 4-16.
+func ConcurrentCollect[T any, U any](workerFunc Worker[T, *U], inputs []T) []*U {
+	return ConcurrentCollectWith(OptimalWorkerCount(), workerFunc, inputs)
+}
+
+// ConcurrentCollectWith runs a pool of workers of the specified size which concurrently call the provided worker
+// func on each input to get a result slice of non-nil outputs. Size inputs < 1 will automatically be set to 1.
+func ConcurrentCollectWith[T any, U any](size int, workerFunc Worker[T, *U], inputs []T) []*U {
+	if size < 1 {
+		size = 1
+	}
+
+	workerPool := NewWorkerPool(size, workerFunc)
+	defer workerPool.Shutdown()
+
+	workGroup := NewCollectionGroup(workerPool)
+	for _, input := range inputs {
+		workGroup.Push(input)
+	}
+
+	return workGroup.Wait()
+}
+
+// ConcurrentRun runs a pool of N workers which concurrently call the provided runner func on each
+// input. The total number of workers is determined by the total number of CPUs available, bound to
+// a range from 4-16.
+func ConcurrentRun[T any](runner Runner[T], inputs []T) {
+	ConcurrentRunWith(OptimalWorkerCount(), runner, inputs)
+}
+
+// ConcurrentRunWith runs a pool of runners of the specified size which concurrently call the provided runner
+// func on each input. Size inputs < 1 will automatically be set to 1.
+func ConcurrentRunWith[T any](size int, runner Runner[T], inputs []T) {
+	if size < 1 {
+		size = 1
+	}
+
+	workerPool := NewWorkerPool(size, func(input T) (void struct{}) {
+		runner(input)
+		return
+	})
+
+	workGroup := NewNoResultGroup(workerPool)
+	for _, input := range inputs {
+		workGroup.Push(input)
+	}
+
+	workGroup.Wait()
+}

core/pkg/util/worker/worker_test.go

@@ -74,7 +74,6 @@ func TestWorkerPoolExactWorkers(t *testing.T) {
 	case <-time.After(5 * time.Second):
 		t.Errorf("Failed to Complete Run for %d jobs in 5s\n", workers)
 	}
-
 }
 
 func TestOrderedWorkGroup(t *testing.T) {
@@ -116,6 +115,38 @@ func TestOrderedWorkGroup(t *testing.T) {
 	// above assertion
 }
 
+func TestConcurrentRun(t *testing.T) {
+	const tasks = 50
+
+	var wg sync.WaitGroup
+	wg.Add(tasks)
+
+	// worker func logs start/finish for simulated work, returns input value
+	// for testing resulting group output
+	work := func(i int) {
+		defer wg.Done()
+
+		t.Logf("Starting Work: %d\n", i)
+		time.Sleep(time.Duration(rand.Intn(250)+250) * time.Millisecond)
+		t.Logf("Finished Work: %d\n", i)
+	}
+
+	// pre-build inputs
+	input := make([]int, tasks)
+	for i := 0; i < tasks; i++ {
+		input[i] = i + 1
+	}
+
+	// get results and verify they match the recorded inputs
+	ConcurrentRunWith(10, work, input)
+
+	select {
+	case <-waitChannelFor(&wg):
+	case <-time.After(5 * time.Second):
+		t.Errorf("Failed to Complete Run for %d jobs in 5s\n", tasks)
+	}
+}
+
 func TestConcurrentDoOrdered(t *testing.T) {
 	// Perform a similar test to the above ordered test, but use the helper func with pre-built inputs
 	const tasks = 50
@@ -147,3 +178,108 @@ func TestConcurrentDoOrdered(t *testing.T) {
 	// the result collection handles the ordering in the group, which is what we want to ensure in the
 	// above assertion
 }
+
+func TestConcurrentCollect(t *testing.T) {
+	type A struct {
+		Value int
+	}
+
+	type B struct {
+		Value int
+	}
+
+	// Perform a similar test to the above ordered test, but use the helper func with pre-built inputs
+	const tasks = 100
+	const expectedResults = 50
+
+	var inputs []*A
+	for i := 0; i < tasks; i++ {
+		inputs = append(inputs, &A{Value: i})
+	}
+
+	workerFunc := func(a *A) *B {
+		time.Sleep(time.Duration(rand.Intn(150)+100) * time.Millisecond)
+
+		if a.Value%2 == 0 {
+			return &B{Value: a.Value}
+		}
+
+		return nil
+	}
+
+	results := ConcurrentCollect(workerFunc, inputs)
+
+	if len(results) != expectedResults {
+		t.Errorf("Expected 50 results, got %d", len(results))
+	}
+
+	seen := map[int]bool{}
+	for _, result := range results {
+		if seen[result.Value] {
+			t.Errorf("Duplicate result: %d", result.Value)
+		}
+		seen[result.Value] = true
+
+		if result.Value%2 != 0 {
+			t.Errorf("Found odd value: %d", result.Value)
+		}
+	}
+}
+
+func TestConcurrentDoWithLessThanOne(t *testing.T) {
+	const tasks = 4
+
+	var wg sync.WaitGroup
+	wg.Add(tasks)
+
+	now := time.Now()
+
+	doIt := func(i int) int {
+		defer wg.Done()
+		time.Sleep(250 * time.Millisecond)
+		return i
+	}
+
+	results := ConcurrentDoWith(-1, doIt, []int{1, 2, 3, 4})
+
+	select {
+	case <-waitChannelFor(&wg):
+	case <-time.After(2 * time.Second):
+		t.Errorf("Failed to Complete Run for %d jobs in 2s\n", tasks)
+	}
+
+	if time.Since(now) > 1500*time.Millisecond {
+		t.Errorf("Expected to complete in 1.5s, took %dms", time.Since(now).Milliseconds())
+	}
+	for i := 1; i <= tasks; i++ {
+		if results[i-1] != i {
+			t.Errorf("Expected %d, got %d", i, results[i])
+		}
+	}
+}
+
+func TestConcurrentRunWithLessThanOne(t *testing.T) {
+	const tasks = 4
+
+	var wg sync.WaitGroup
+	wg.Add(tasks)
+
+	now := time.Now()
+
+	doIt := func(i int) {
+		defer wg.Done()
+		time.Sleep(250 * time.Millisecond)
+	}
+
+	ConcurrentRunWith(-1, doIt, []int{1, 2, 3, 4})
+
+	select {
+	case <-waitChannelFor(&wg):
+	case <-time.After(2 * time.Second):
+		t.Errorf("Failed to Complete Run for %d jobs in 2s\n", tasks)
+	}
+
+	if time.Since(now) > 1500*time.Millisecond {
+		t.Errorf("Expected to complete in 1.5s, took %dms", time.Since(now).Milliseconds())
+	}
+}

go.mod

@@ -65,6 +65,11 @@ require (
 	sigs.k8s.io/yaml v1.3.0
 )
 
+require (
+	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/sony/gobreaker v0.5.0 // indirect
+)
+
 require (
 	cloud.google.com/go v0.114.0 // indirect
 	cloud.google.com/go/auth v0.5.1 // indirect
@@ -148,6 +153,7 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/oklog/run v1.1.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.71.0
 	github.com/pelletier/go-toml v1.9.3 // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
@@ -178,7 +184,7 @@ require (
 	google.golang.org/genproto v0.0.0-20240528184218-531527333157 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect
-	google.golang.org/grpc v1.64.0 // indirect
+	google.golang.org/grpc v1.64.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -221,6 +221,8 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
+github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0=
 github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -460,6 +462,8 @@ github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE=
 github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk=
 github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A=
 github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU=
+github.com/oracle/oci-go-sdk/v65 v65.71.0 h1:eEnFD/CzcoqdAA0xu+EmK32kJL3jfV0oLYNWVzoKNyo=
+github.com/oracle/oci-go-sdk/v65 v65.71.0/go.mod h1:IBEV9l1qBzUpo7zgGaRUhbB05BVfcDGYRFBCPlTcPp0=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
@@ -508,6 +512,8 @@ github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9Nz
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg=
+github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
 github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY=
 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
@@ -523,6 +529,8 @@ github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -533,6 +541,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
@@ -755,6 +764,7 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
 golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -935,8 +945,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY=
-google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg=
+google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA=
+google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

pkg/cloud/alibaba/provider.go

@@ -67,7 +67,7 @@ var (
 )
 
 // Variable to keep track of instance families that fail in DescribePrice API due improper defaulting of systemDisk if the information is not available
-var alibabaDefaultToCloudEssd = []string{"g6e", "r6e", "r7", "g7", "g7a", "r7a"}
+var alibabaDefaultToCloudEssd = []string{"g6e", "r6e"}
 
 var alibabaRegions = []string{
 	"cn-qingdao",
@@ -980,9 +980,9 @@ func createDescribePriceACSRequest(i interface{}) (*requests.CommonRequest, erro
 				request.QueryParams["SystemDisk.PerformanceLevel"] = node.SystemDisk.PerformanceLevel
 			}
 		} else {
-			// When System Disk information is not available for instance family g6e, r7 and r6e the defaults in
-			// DescribePrice dont default rightly to cloud_essd for these instances.
-			if slices.Contains(alibabaDefaultToCloudEssd, node.InstanceTypeFamily) {
+			// When the system disk information is not available, and the instance family is g6e or r6e,
+			// or the instance generation is 6 or above, the default disk category in DescribePrice should be cloud_essd.
+			if slices.Contains(alibabaDefaultToCloudEssd, node.InstanceTypeFamily) || getInstanceFamilyGenerationFromType(node.InstanceType) > 6 {
 				request.QueryParams["SystemDisk.Category"] = ALIBABA_DISK_CLOUD_ESSD_CATEGORY
 			}
 		}
@@ -1148,6 +1148,26 @@ func getInstanceFamilyFromType(instanceType string) string {
 	return splitinstanceType[1]
 }
 
+// This function is used to obtain the generation of the instance family from the InstanceType,
+// because when the generation is higher than or equal to 7, the instance disk type will not support cloud_efficiency.
+// In such cases, when calling the DescribePrice interface, the system disk type will default to cloud_essd.
+func getInstanceFamilyGenerationFromType(instanceType string) int {
+	// FamilyName format: g7ne or g7 or r7 or r6e,
+	familyName := getInstanceFamilyFromType(instanceType)
+	re := regexp.MustCompile(`(\d+)`)
+	match := re.FindString(familyName)
+	if match != "" {
+		generation, err := strconv.Atoi(match)
+		if err != nil {
+			log.Errorf("unable to convert the generation of the instance type %s to integer", instanceType)
+		} else {
+			return generation
+		}
+	}
+	log.Warnf("unable to find the generation of the instance type %s,", instanceType)
+	return -1
+}
+
 // getInstanceIDFromProviderID returns the instance ID associated with the Node. A *v1.Node providerID in Alibaba cloud
 // is of <REGION-ID>.<INSTANCE-ID>. This function returns the Instance ID for the given ProviderID. if its unable to interpret
 // it defaults to empty string.

pkg/cloud/alibaba/provider_test.go

@@ -112,6 +112,20 @@ func TestProcessDescribePriceAndCreateAlibabaPricing(t *testing.T) {
 			},
 			expectedError: nil,
 		},
+		{
+			name: "test General Purpose Type g8a instance family",
+			teststruct: &SlimK8sNode{
+				InstanceType:       "ecs.g8a.8xlarge",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "33554432KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "cn-hangzhou.i-test-01c",
+				InstanceTypeFamily: "g8a",
+			},
+			expectedError: nil,
+		},
 		{
 			name: "test Enhanced General Purpose Type g6e instance family",
 			teststruct: &SlimK8sNode{
@@ -856,3 +870,108 @@ func TestDeterminePVRegion(t *testing.T) {
 	}
 
 }
+
+func TestGetInstanceFamilyGenerationFromType(t *testing.T) {
+	cases := []struct {
+		name                             string
+		instanceType                     string
+		expectedInstanceFamilyGeneration int
+	}{
+		{
+			name:                             "test if ecs.[instance-family].[different-type] work",
+			instanceType:                     "ecs.sn2ne.2xlarge",
+			expectedInstanceFamilyGeneration: 2,
+		},
+		{
+			name:                             "test if ecs.[instance-family].[different-type] work",
+			instanceType:                     "ecs.g7.large",
+			expectedInstanceFamilyGeneration: 7,
+		},
+		{
+			name:                             "test if random word gives you ALIBABA_UNKNOWN_INSTANCE_FAMILY_TYPE value ",
+			instanceType:                     "random.value",
+			expectedInstanceFamilyGeneration: -1,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			returnValue := getInstanceFamilyGenerationFromType(c.instanceType)
+			if returnValue != c.expectedInstanceFamilyGeneration {
+				t.Fatalf("Case name %s: expected instance family generation of type %d but got %d", c.name, c.expectedInstanceFamilyGeneration, returnValue)
+			}
+		})
+	}
+}
+
+func TestCreateDescribeNodePriceACSRequest(t *testing.T) {
+
+	cases := []struct {
+		name                 string
+		testStruct           interface{}
+		expectedError        error
+		expectedDiskCategory string
+	}{
+		{
+			// Test case for instance type ecs.g6.large
+			name: "test request parma when instance type is ecs.g6.large",
+			testStruct: &SlimK8sNode{
+				InstanceType:       "ecs.g6.large",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "16KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "Ali-XXX-node-01",
+				InstanceTypeFamily: "g6",
+			},
+			expectedError:        nil,
+			expectedDiskCategory: "",
+		},
+		{
+			// Test case for instance type ecs.g7.large
+			name: "test request parma when instance type is ecs.g7.large",
+			testStruct: &SlimK8sNode{
+				InstanceType:       "ecs.g7.large",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "16KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "Ali-XXX-node-02",
+				InstanceTypeFamily: "g7",
+			},
+			expectedError:        nil,
+			expectedDiskCategory: ALIBABA_DISK_CLOUD_ESSD_CATEGORY,
+		},
+		{
+			// Test case for instance type ecs.g7.large, this instance type is in 'alibabaDefaultToCloudEssd'
+			name: "test request parma when instance type is ecs.g6e.large",
+			testStruct: &SlimK8sNode{
+				InstanceType:       "ecs.g6e.large",
+				RegionID:           "cn-hangzhou",
+				PriceUnit:          "Hour",
+				MemorySizeInKiB:    "16KiB",
+				IsIoOptimized:      true,
+				OSType:             "Linux",
+				ProviderID:         "Ali-XXX-node-03",
+				InstanceTypeFamily: "g6e",
+			},
+			expectedError:        nil,
+			expectedDiskCategory: ALIBABA_DISK_CLOUD_ESSD_CATEGORY,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			req, err := createDescribePriceACSRequest(c.testStruct)
+			t.Logf("Request Params SystemDisk.Category: %v", req.QueryParams["SystemDisk.Category"])
+			if err != nil && c.expectedError != nil {
+				t.Fatalf("Case name %s: Error converting to Alibaba cloud request", c.name)
+			}
+			if c.expectedDiskCategory != req.QueryParams["SystemDisk.Category"] {
+				t.Fatalf("Case name %s: Disk Category is not set correctly", c.name)
+			}
+		})
+	}
+}

pkg/cloud/aws/athenaintegration.go

@@ -14,6 +14,8 @@ import (
 )
 
 const LabelColumnPrefix = "resource_tags_user_"
+const AWSLabelColumnPrefix = "resource_tags_aws_"
+const AthenaResourceTagPrefix = "resource_tags_"
 
 // athenaDateLayout is the default AWS date format
 const AthenaDateLayout = "2006-01-02 15:04:05.000"
@@ -52,6 +54,7 @@ type AthenaQueryIndexes struct {
 	Query                  string
 	ColumnIndexes          map[string]int
 	TagColumns             []string
+	AWSTagColumns          []string
 	ListCostColumn         string
 	NetCostColumn          string
 	AmortizedNetCostColumn string
@@ -80,6 +83,8 @@ func (ai *AthenaIntegration) GetCloudCost(start, end time.Time) (*opencost.Cloud
 		"line_item_usage_account_id",
 		"line_item_product_code",
 		"line_item_usage_type",
+		"product_region_code",
+		"line_item_availability_zone",
 	}
 
 	// Create query indices
@@ -98,6 +103,10 @@ func (ai *AthenaIntegration) GetCloudCost(start, end time.Time) (*opencost.Cloud
 			groupByColumns = append(groupByColumns, quotedTag)
 			aqi.TagColumns = append(aqi.TagColumns, quotedTag)
 		}
+		if strings.HasPrefix(column, AWSLabelColumnPrefix) {
+			groupByColumns = append(groupByColumns, column)
+			aqi.AWSTagColumns = append(aqi.AWSTagColumns, column)
+		}
 	}
 	var selectColumns []string
 
@@ -333,7 +342,6 @@ func (ai *AthenaIntegration) RowToCloudCost(row types.Row, aqi AthenaQueryIndexe
 	// Iterate through the slice of tag columns, assigning
 	// values to the column names, minus the tag prefix.
 	labels := opencost.CloudCostLabels{}
-	labelValues := []string{}
 	for _, tagColumnName := range aqi.TagColumns {
 		// remove quotes
 		labelName := strings.TrimPrefix(tagColumnName, `"`)
@@ -343,7 +351,15 @@ func (ai *AthenaIntegration) RowToCloudCost(row types.Row, aqi AthenaQueryIndexe
 		value := GetAthenaRowValue(row, aqi.ColumnIndexes, tagColumnName)
 		if value != "" {
 			labels[labelName] = value
-			labelValues = append(labelValues, value)
+		}
+	}
+
+	for _, awsColumnName := range aqi.AWSTagColumns {
+		// partially remove prefix leaving "aws_"
+		labelName := strings.TrimPrefix(awsColumnName, AthenaResourceTagPrefix)
+		value := GetAthenaRowValue(row, aqi.ColumnIndexes, awsColumnName)
+		if value != "" {
+			labels[labelName] = value
 		}
 	}
 
@@ -353,6 +369,8 @@ func (ai *AthenaIntegration) RowToCloudCost(row types.Row, aqi AthenaQueryIndexe
 	providerID := GetAthenaRowValue(row, aqi.ColumnIndexes, "line_item_resource_id")
 	productCode := GetAthenaRowValue(row, aqi.ColumnIndexes, "line_item_product_code")
 	usageType := GetAthenaRowValue(row, aqi.ColumnIndexes, "line_item_usage_type")
+	regionCode := GetAthenaRowValue(row, aqi.ColumnIndexes, "product_region_code")
+	availabilityZone := GetAthenaRowValue(row, aqi.ColumnIndexes, "line_item_availability_zone")
 	isK8s, _ := strconv.ParseBool(GetAthenaRowValue(row, aqi.ColumnIndexes, aqi.IsK8sColumn))
 	k8sPct := 0.0
 	if isK8s {
@@ -396,13 +414,17 @@ func (ai *AthenaIntegration) RowToCloudCost(row types.Row, aqi AthenaQueryIndexe
 	}
 
 	properties := opencost.CloudCostProperties{
-		ProviderID:      providerID,
-		Provider:        opencost.AWSProvider,
-		AccountID:       accountID,
-		InvoiceEntityID: invoiceEntityID,
-		Service:         productCode,
-		Category:        category,
-		Labels:          labels,
+		ProviderID:        providerID,
+		Provider:          opencost.AWSProvider,
+		AccountID:         accountID,
+		AccountName:       accountID,
+		InvoiceEntityID:   invoiceEntityID,
+		InvoiceEntityName: invoiceEntityID,
+		RegionID:          regionCode,
+		AvailabilityZone:  availabilityZone,
+		Service:           productCode,
+		Category:          category,
+		Labels:            labels,
 	}
 
 	start, err := time.Parse(AthenaDateLayout, startStr)

pkg/cloud/aws/s3selectintegration.go

@@ -7,7 +7,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/opencost/opencost/core/pkg/log"
 	"github.com/opencost/opencost/core/pkg/opencost"
 )
@@ -15,13 +14,15 @@ import (
 const S3SelectDateLayout = "2006-01-02T15:04:05Z"
 
 // S3Object is aliased as "s" in queries
-const S3SelectAccountID = `s."bill/PayerAccountId"`
-
+const S3SelectBillPayerAccountID = `s."bill/PayerAccountId"`
+const S3SelectAccountID = `s."lineItem/UsageAccountId"`
 const S3SelectItemType = `s."lineItem/LineItemType"`
 const S3SelectStartDate = `s."lineItem/UsageStartDate"`
 const S3SelectProductCode = `s."lineItem/ProductCode"`
 const S3SelectResourceID = `s."lineItem/ResourceId"`
 const S3SelectUsageType = `s."lineItem/UsageType"`
+const S3SelectRegionCode = `s."product/regionCode"`
+const S3SelectAvailabilityZone = `s."lineItem/AvailabilityZone"`
 
 const S3SelectListCost = `s."lineItem/UnblendedCost"`
 const S3SelectNetCost = `s."lineItem/NetUnblendedCost"`
@@ -29,6 +30,12 @@ const S3SelectNetCost = `s."lineItem/NetUnblendedCost"`
 // These two may be used for Amortized<Net>Cost
 const S3SelectRICost = `s."reservation/EffectiveCost"`
 const S3SelectSPCost = `s."savingsPlan/SavingsPlanEffectiveCost"`
+const S3SelectNetRICost = `s."reservation/NetEffectiveCost"`
+const S3SelectNetSPCost = `s."savingsPlan/NetSavingsPlanEffectiveCost"`
+
+const S3SelectUserLabelPrefix = "resourceTags/user:"
+const S3SelectAWSLabelPrefix = "resourceTags/aws:"
+const S3SelectResourceTagsPrefix = "resourceTags/"
 
 type S3SelectIntegration struct {
 	S3SelectQuerier
@@ -44,15 +51,6 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 		opencost.NewWindow(&start, &end).String(),
 	)
 
-	// Set midnight yesterday as last point in time reconciliation data
-	// can be pulled from to ensure complete days of data
-	midnightYesterday := time.Now().In(
-		time.UTC,
-	).Truncate(time.Hour*24).AddDate(0, 0, -1)
-	if end.After(midnightYesterday) {
-		end = midnightYesterday
-	}
-
 	// ccsr to populate with cloudcosts.
 	ccsr, err := opencost.NewCloudCostSetRange(
 		start,
@@ -74,40 +72,71 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 		return nil, err
 	}
 	// Acquire headers
-	headers, err := s3si.GetHeaders(queryKeys, client)
+	headers, err := s3si.GetHeaders(queryKeys[0], client)
 	if err != nil {
 		return nil, err
 	}
-	// Exactly what it says on the tin. Though is there a set equivalent
-	// in Go? This seems like a good use case for that.
-	allColumns := map[string]bool{}
+
+	allColumns := map[string]struct{}{}
 	for _, header := range headers {
-		allColumns[header] = true
+		allColumns[header] = struct{}{}
 	}
 
 	formattedStart := start.Format("2006-01-02")
 	formattedEnd := end.Format("2006-01-02")
 	selectColumns := []string{
 		S3SelectStartDate,
+		S3SelectBillPayerAccountID,
 		S3SelectAccountID,
 		S3SelectResourceID,
 		S3SelectItemType,
 		S3SelectProductCode,
 		S3SelectUsageType,
+		S3SelectRegionCode,
+		S3SelectAvailabilityZone,
 		S3SelectListCost,
 	}
-	// OC equivalent to KCM env flags relevant at all?
+	_, checkNet := allColumns[S3SelectNetCost]
+	if checkNet {
+		selectColumns = append(selectColumns, S3SelectNetCost)
+	}
+
 	// Check for Reservation columns in CUR and query if available
-	checkReservations := allColumns[S3SelectRICost]
+	_, checkReservations := allColumns[S3SelectRICost]
 	if checkReservations {
 		selectColumns = append(selectColumns, S3SelectRICost)
 	}
+	_, checkNetReservations := allColumns[S3SelectNetRICost]
+	if checkNetReservations {
+		selectColumns = append(selectColumns, S3SelectNetRICost)
+	}
 
 	// Check for Savings Plan Columns in CUR and query if available
-	checkSavingsPlan := allColumns[S3SelectSPCost]
+	_, checkSavingsPlan := allColumns[S3SelectSPCost]
 	if checkSavingsPlan {
 		selectColumns = append(selectColumns, S3SelectSPCost)
 	}
+	_, checkNetSavingsPlan := allColumns[S3SelectNetSPCost]
+	if checkNetSavingsPlan {
+		selectColumns = append(selectColumns, S3SelectNetSPCost)
+	}
+
+	// Determine which columns are user-defined tags and add those to the list
+	// of columns to query.
+	labelColumns := []string{}
+	awsLabelColumns := []string{}
+	for column := range allColumns {
+		if strings.HasPrefix(column, S3SelectUserLabelPrefix) {
+			quotedTag := fmt.Sprintf(`s."%s"`, column)
+			selectColumns = append(selectColumns, quotedTag)
+			labelColumns = append(labelColumns, quotedTag)
+		}
+		if strings.HasPrefix(column, S3SelectAWSLabelPrefix) {
+			quotedTag := fmt.Sprintf(`s."%s"`, column)
+			selectColumns = append(selectColumns, quotedTag)
+			awsLabelColumns = append(awsLabelColumns, quotedTag)
+		}
+	}
 
 	// Build map of query columns to use for parsing query
 	columnIndexes := map[string]int{}
@@ -118,17 +147,8 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 	selectStr := strings.Join(selectColumns, ", ")
 	queryStr := `SELECT %s FROM s3object s
 	WHERE (CAST(s."lineItem/UsageStartDate" AS TIMESTAMP) BETWEEN CAST('%s' AS TIMESTAMP) AND CAST('%s' AS TIMESTAMP))
-	AND s."lineItem/ResourceId" <> ''
-	AND (
-		(
-			s."lineItem/ProductCode" = 'AmazonEC2' AND (
-				SUBSTRING(s."lineItem/ResourceId",1,2) = 'i-'
-				OR SUBSTRING(s."lineItem/ResourceId",1,4) = 'vol-'
-			)
-		)
-		OR s."lineItem/ProductCode" = 'AWSELB'
-       OR s."lineItem/ProductCode" = 'AmazonFSx'
-	)`
+	AND (s."lineItem/LineItemType" = 'Usage' OR s."lineItem/LineItemType" = 'DiscountedUsage' OR s."lineItem/LineItemType" = 'SavingsPlanCoveredUsage' OR s."lineItem/LineItemType" = 'EdpDiscount' OR s."lineItem/LineItemType" = 'PrivateRateDiscount')
+	`
 	query := fmt.Sprintf(queryStr, selectStr, formattedStart, formattedEnd)
 
 	processResults := func(reader *csv.Reader) error {
@@ -143,45 +163,107 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 			}
 
 			startStr := GetCSVRowValue(row, columnIndexes, S3SelectStartDate)
+			billPayerAccountID := GetCSVRowValue(row, columnIndexes, S3SelectBillPayerAccountID)
 			itemAccountID := GetCSVRowValue(row, columnIndexes, S3SelectAccountID)
 			itemProviderID := GetCSVRowValue(row, columnIndexes, S3SelectResourceID)
 			lineItemType := GetCSVRowValue(row, columnIndexes, S3SelectItemType)
 			itemProductCode := GetCSVRowValue(row, columnIndexes, S3SelectProductCode)
 			usageType := GetCSVRowValue(row, columnIndexes, S3SelectUsageType)
+			regionCode := GetCSVRowValue(row, columnIndexes, S3SelectRegionCode)
+			availabilityZone := GetCSVRowValue(row, columnIndexes, S3SelectAvailabilityZone)
+
+			// Iterate through the slice of tag columns, assigning
+			// values to the column names, minus the tag prefix.
+			labels := opencost.CloudCostLabels{}
+			for _, labelColumnName := range labelColumns {
+				// remove quotes
+				labelName := strings.TrimPrefix(labelColumnName, `s."`)
+				labelName = strings.TrimSuffix(labelName, `"`)
+				// remove prefix
+				labelName = strings.TrimPrefix(labelName, S3SelectUserLabelPrefix)
+				value := GetCSVRowValue(row, columnIndexes, labelColumnName)
+				if value != "" {
+					labels[labelName] = value
+				}
+			}
+			for _, awsLabelColumnName := range awsLabelColumns {
+				// remove quotes
+				labelName := strings.TrimPrefix(awsLabelColumnName, `s."`)
+				labelName = strings.TrimSuffix(labelName, `"`)
+				// partially remove prefix leaving "aws:"
+				labelName = strings.TrimPrefix(labelName, S3SelectResourceTagsPrefix)
+				value := GetCSVRowValue(row, columnIndexes, awsLabelColumnName)
+				if value != "" {
+					labels[labelName] = value
+				}
+			}
+
+			isKubernetes := 0.0
+			if itemProductCode == "AmazonEKS" || hasK8sLabel(labels) {
+				isKubernetes = 1.0
+			}
 
 			var (
-				amortizedCost float64
-				listCost      float64
-				netCost       float64
+				amortizedCost    float64
+				amortizedNetCost float64
+				listCost         float64
+				netCost          float64
 			)
 			// Get list and net costs
-			listCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectListCost)
-			if err != nil {
-				return err
+			if lineItemType != "EdpDiscount" && lineItemType != "PrivateRateDiscount" {
+				listCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectListCost)
+				if err != nil {
+					return err
+				}
 			}
-			netCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectNetCost)
-			if err != nil {
-				return err
+
+			// Get net cost if available
+			netCost = listCost
+			if checkNet {
+				netCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectNetCost)
+				if err != nil {
+					return err
+				}
 			}
 
 			// If there is a reservation_reservation_a_r_n on the line item use the awsRIPricingSUMColumn as cost
-			if checkReservations && lineItemType == "DiscountedUsage" {
-				amortizedCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectRICost)
-				if err != nil {
-					log.Errorf(err.Error())
-					continue
+			amortizedCost = listCost
+			amortizedNetCost = listCost
+			if lineItemType == "DiscountedUsage" {
+				if checkReservations {
+					amortizedCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectRICost)
+					if err != nil {
+						log.Errorf(err.Error())
+						continue
+					}
+					amortizedNetCost = amortizedCost
+				}
+				if checkNetReservations {
+					amortizedNetCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectNetRICost)
+					if err != nil {
+						log.Errorf(err.Error())
+						continue
+					}
 				}
 				// If there is a lineItemType of SavingsPlanCoveredUsage use the awsSPPricingSUMColumn
-			} else if checkSavingsPlan && lineItemType == "SavingsPlanCoveredUsage" {
-				amortizedCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectSPCost)
-				if err != nil {
-					log.Errorf(err.Error())
-					continue
+			} else if lineItemType == "SavingsPlanCoveredUsage" {
+				if checkSavingsPlan {
+					amortizedCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectSPCost)
+					if err != nil {
+						log.Errorf(err.Error())
+						continue
+					}
+					amortizedNetCost = amortizedCost
+				}
+				if checkNetSavingsPlan {
+					amortizedNetCost, err = GetCSVRowValueFloat(row, columnIndexes, S3SelectNetSPCost)
+					if err != nil {
+						log.Errorf(err.Error())
+						continue
+					}
 				}
-			} else {
-				// Default to listCost
-				amortizedCost = listCost
 			}
+
 			category := SelectAWSCategory(itemProviderID, usageType, itemProductCode)
 			// Retrieve final stanza of product code for ProviderID
 			if itemProductCode == "AWSELB" || itemProductCode == "AmazonFSx" {
@@ -190,10 +272,16 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 
 			properties := opencost.CloudCostProperties{}
 			properties.Provider = opencost.AWSProvider
+			properties.InvoiceEntityID = billPayerAccountID
+			properties.InvoiceEntityName = billPayerAccountID
 			properties.AccountID = itemAccountID
+			properties.AccountName = itemAccountID
 			properties.Category = category
 			properties.Service = itemProductCode
 			properties.ProviderID = itemProviderID
+			properties.RegionID = regionCode
+			properties.AvailabilityZone = availabilityZone
+			properties.Labels = labels
 
 			itemStart, err := time.Parse(S3SelectDateLayout, startStr)
 			if err != nil {
@@ -211,19 +299,24 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 				Properties: &properties,
 				Window:     opencost.NewWindow(&itemStart, &itemEnd),
 				ListCost: opencost.CostMetric{
-					Cost: listCost,
+					Cost:              listCost,
+					KubernetesPercent: isKubernetes,
 				},
 				NetCost: opencost.CostMetric{
-					Cost: netCost,
+					Cost:              netCost,
+					KubernetesPercent: isKubernetes,
 				},
 				AmortizedNetCost: opencost.CostMetric{
-					Cost: amortizedCost,
+					Cost:              amortizedCost,
+					KubernetesPercent: isKubernetes,
 				},
 				AmortizedCost: opencost.CostMetric{
-					Cost: amortizedCost,
+					Cost:              amortizedNetCost,
+					KubernetesPercent: isKubernetes,
 				},
 				InvoicedCost: opencost.CostMetric{
-					Cost: netCost,
+					Cost:              netCost,
+					KubernetesPercent: isKubernetes,
 				},
 			}
 			ccsr.LoadCloudCost(cc)
@@ -237,25 +330,34 @@ func (s3si *S3SelectIntegration) GetCloudCost(
 	return ccsr, nil
 }
 
-func (s3si *S3SelectIntegration) GetHeaders(queryKeys []string, client *s3.Client) ([]string, error) {
-	// Query to grab only header line from file
-	query := "SELECT * FROM S3OBJECT LIMIT 1"
-	var record []string
+const (
+	TagAWSEKSClusterName     = "aws:eks:cluster-name"
+	TagEKSClusterName        = "eks:cluster-name"
+	TagEKSCtlClusterName     = "alpha.eksctl.io/cluster-name"
+	TagKubernetesServiceName = "kubernetes.io/service-name"
+	TagKubernetesPVCName     = "kubernetes.io/created-for/pvc/name"
+	TagKubernetesPVName      = "kubernetes.io/created-for/pv/name"
+)
 
-	proccessheaders := func(reader *csv.Reader) error {
-		var err error
-		record, err = reader.Read()
-		if err != nil {
-			return err
-		}
-		return nil
+// hsK8sLabel checks if the labels contain a k8s label
+func hasK8sLabel(labels opencost.CloudCostLabels) bool {
+	if _, ok := labels[TagAWSEKSClusterName]; ok {
+		return true
 	}
-
-	// Use only the first query key with assumption that files share schema
-	err := s3si.Query(query, []string{queryKeys[0]}, client, proccessheaders)
-	if err != nil {
-		return nil, err
+	if _, ok := labels[TagEKSClusterName]; ok {
+		return true
 	}
-
-	return record, nil
+	if _, ok := labels[TagEKSCtlClusterName]; ok {
+		return true
+	}
+	if _, ok := labels[TagKubernetesServiceName]; ok {
+		return true
+	}
+	if _, ok := labels[TagKubernetesPVCName]; ok {
+		return true
+	}
+	if _, ok := labels[TagKubernetesPVName]; ok {
+		return true
+	}
+	return false
 }

pkg/cloud/aws/s3selectquerier.go

@@ -45,6 +45,18 @@ func (s3sq *S3SelectQuerier) Query(query string, queryKeys []string, cli *s3.Cli
 	return nil
 }
 
+func (s3sq *S3SelectQuerier) GetHeaders(queryKey string, cli *s3.Client) ([]string, error) {
+	reader, err := s3sq.fetchCSVReader("SELECT * FROM S3Object LIMIT 1", queryKey, cli, s3Types.FileHeaderInfoNone)
+	if err != nil {
+		return nil, err
+	}
+	record, err := reader.Read()
+	if err != nil {
+		return nil, err
+	}
+	return record, nil
+}
+
 // GetQueryKeys returns a list of s3 object names, where the there are 1 object for each month within the range between
 // start and end
 func (s3sq *S3SelectQuerier) GetQueryKeys(start, end time.Time, client *s3.Client) ([]string, error) {

pkg/cloud/azure/azurestorageintegration.go

@@ -34,13 +34,16 @@ func (asi *AzureStorageIntegration) GetCloudCost(start, end time.Time) (*opencos
 		// until we can revisit and spend the time to do the calculations correctly
 		cc := &opencost.CloudCost{
 			Properties: &opencost.CloudCostProperties{
-				ProviderID:      providerID,
-				Provider:        opencost.AzureProvider,
-				AccountID:       abv.SubscriptionID,
-				InvoiceEntityID: abv.InvoiceEntityID,
-				Service:         abv.Service,
-				Category:        SelectAzureCategory(abv.MeterCategory),
-				Labels:          abv.Tags,
+				ProviderID:        providerID,
+				Provider:          opencost.AzureProvider,
+				AccountID:         abv.SubscriptionID,
+				AccountName:       abv.SubscriptionName,
+				InvoiceEntityID:   abv.InvoiceEntityID,
+				InvoiceEntityName: abv.InvoiceEntityName,
+				RegionID:          abv.Region,
+				Service:           abv.Service,
+				Category:          SelectAzureCategory(abv.MeterCategory),
+				Labels:            abv.Tags,
 			},
 			Window: window,
 			AmortizedNetCost: opencost.CostMetric{

pkg/cloud/azure/billingexportparser.go

@@ -19,16 +19,19 @@ var groupRegex = regexp.MustCompile("(/[^/]+)")
 
 // BillingRowValues holder for Azure Billing Values
 type BillingRowValues struct {
-	Date            time.Time
-	MeterCategory   string
-	SubscriptionID  string
-	InvoiceEntityID string
-	InstanceID      string
-	Service         string
-	Tags            map[string]string
-	AdditionalInfo  map[string]any
-	Cost            float64
-	NetCost         float64
+	Date              time.Time
+	MeterCategory     string
+	SubscriptionID    string
+	SubscriptionName  string
+	InvoiceEntityID   string
+	InvoiceEntityName string
+	Region            string
+	InstanceID        string
+	Service           string
+	Tags              map[string]string
+	AdditionalInfo    map[string]any
+	Cost              float64
+	NetCost           float64
 }
 
 func (brv *BillingRowValues) IsCompute(category string) bool {
@@ -52,17 +55,20 @@ func (brv *BillingRowValues) IsCompute(category string) bool {
 
 // BillingExportParser holds indexes of relevent fields in Azure Billing CSV in addition to the correct data format
 type BillingExportParser struct {
-	Date            int
-	MeterCategory   int
-	InvoiceEntityID int
-	SubscriptionID  int
-	InstanceID      int
-	Service         int
-	Tags            int
-	AdditionalInfo  int
-	Cost            int
-	NetCost         int
-	DateFormat      string
+	Date              int
+	MeterCategory     int
+	InvoiceEntityID   int
+	InvoiceEntityName int
+	SubscriptionID    int
+	SubscriptionName  int
+	Region            int
+	InstanceID        int
+	Service           int
+	Tags              int
+	AdditionalInfo    int
+	Cost              int
+	NetCost           int
+	DateFormat        string
 }
 
 // match "SubscriptionGuid" in "Abonnement-GUID (SubscriptionGuid)"
@@ -106,6 +112,14 @@ func NewBillingParseSchema(headers []string) (*BillingExportParser, error) {
 		return nil, fmt.Errorf("NewBillingParseSchema: failed to find Subscription ID field")
 	}
 
+	// set Subscription Name
+	if i, ok := headerIndexes["subscriptionname"]; ok {
+		abp.SubscriptionName = i
+	} else {
+		// if no subscription name column use subscriptionID column
+		abp.SubscriptionName = abp.SubscriptionID
+	}
+
 	// Set Billing ID
 	if i, ok := headerIndexes["billingaccountid"]; ok {
 		abp.InvoiceEntityID = i
@@ -116,6 +130,25 @@ func NewBillingParseSchema(headers []string) (*BillingExportParser, error) {
 		abp.InvoiceEntityID = abp.SubscriptionID
 	}
 
+	// Set Billing Account Name
+	if i, ok := headerIndexes["billingaccountname"]; ok {
+		abp.InvoiceEntityName = i
+	} else {
+		// if no billing name column is present use billing ID index
+		abp.InvoiceEntityName = abp.InvoiceEntityID
+	}
+
+	// Set Region
+	if i, ok := headerIndexes["resourcelocation"]; ok {
+		abp.Region = i
+	} else if j, ok2 := headerIndexes["meterregion"]; ok2 {
+		abp.Region = j
+	} else if k, ok3 := headerIndexes["location"]; ok3 {
+		abp.Region = k
+	} else {
+		return nil, fmt.Errorf("NewBillingParseSchema: failed to find Region field")
+	}
+
 	// Set Instance ID
 	if i, ok := headerIndexes["instanceid"]; ok {
 		abp.InstanceID = i
@@ -237,16 +270,19 @@ func (bep *BillingExportParser) ParseRow(start, end time.Time, record []string)
 	}
 
 	return &BillingRowValues{
-		Date:            usageDate,
-		MeterCategory:   record[bep.MeterCategory],
-		SubscriptionID:  record[bep.SubscriptionID],
-		InvoiceEntityID: record[bep.InvoiceEntityID],
-		InstanceID:      record[bep.InstanceID],
-		Service:         record[bep.Service],
-		Tags:            tags,
-		AdditionalInfo:  additionalInfo,
-		Cost:            cost,
-		NetCost:         netCost,
+		Date:              usageDate,
+		MeterCategory:     record[bep.MeterCategory],
+		SubscriptionID:    record[bep.SubscriptionID],
+		SubscriptionName:  record[bep.SubscriptionName],
+		InvoiceEntityID:   record[bep.InvoiceEntityID],
+		InvoiceEntityName: record[bep.InvoiceEntityName],
+		Region:            record[bep.Region],
+		InstanceID:        record[bep.InstanceID],
+		Service:           record[bep.Service],
+		Tags:              tags,
+		AdditionalInfo:    additionalInfo,
+		Cost:              cost,
+		NetCost:           netCost,
 	}
 }
 

pkg/cloud/azure/resources/billingexports/values/MissingBrackets.csv

@@ -1,2 +1,2 @@
-subscriptionid,billingaccountid,UsageDateTime,MeterCategory,costinbillingcurrency,paygcostinbillingcurrency,ConsumedService,InstanceId,Tags,AdditionalInfo
-11111111-12ab-34dc-56ef-123456abcdef,11111111-12ab-34dc-56ef-123456abcdef,2021-02-01,Virtual Machines,4,5,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss""","""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": ""aks-nodepool1-12345678-vmss_0"",  ""VCPUs"": 2"
+subscriptionid,billingaccountid,UsageDateTime,MeterCategory,costinbillingcurrency,paygcostinbillingcurrency,ConsumedService,InstanceId,Tags,AdditionalInfo,resourcelocation
+11111111-12ab-34dc-56ef-123456abcdef,11111111-12ab-34dc-56ef-123456abcdef,2021-02-01,Virtual Machines,4,5,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss""","""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": ""aks-nodepool1-12345678-vmss_0"",  ""VCPUs"": 2",""

pkg/cloud/azure/resources/billingexports/values/Template.csv

@@ -1,4 +1,4 @@
-subscriptionid,billingaccountid,UsageDateTime,MeterCategory,costinbillingcurrency,paygcostinbillingcurrency,ConsumedService,InstanceId,Tags,AdditionalInfo
+subscriptionid,billingaccountid,UsageDateTime,MeterCategory,costinbillingcurrency,paygcostinbillingcurrency,ConsumedService,InstanceId,Tags,AdditionalInfo,resourcelocation
 11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Load Balancer,0.075,0.075,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/loadBalancers/kubernetes,,
 11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Virtual Machines,3.504,3.504,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"{""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss"",""orchestrator"":""Kubernetes:1.15.7"",""poolName"":""nodepool1""}","{  ""UsageType"": ""ComputeHR"",  ""ImageType"": ""Canonical"",  ""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": ""aks-nodepool1-12345678-vmss_0"",  ""VMProperties"": ""Microsoft.AKS.Compute.AKS.Linux.Billing"",  ""VCPUs"": 2,  ""CPUs"": 0}"
 11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.0000045,0.0000045,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd03,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-pushgateway"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd03"",""created-by"":""kubernetes-azure-dd""}",

pkg/cloud/azure/resources/billingexports/values/VirtualMachine.csv

@@ -1,2 +1,2 @@
-subscriptionid,billingaccountid,UsageDateTime,MeterCategory,costinbillingcurrency,paygcostinbillingcurrency,ConsumedService,InstanceId,Tags,AdditionalInfo
-11111111-12ab-34dc-56ef-123456abcdef,11111111-12ab-34dc-56ef-123456billing,2021-02-01,Virtual Machines,4,5,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"{""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss""}","{ ""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": ""aks-nodepool1-12345678-vmss_0"",  ""VCPUs"": 2  }"
+subscriptionid,billingaccountid,UsageDateTime,MeterCategory,costinbillingcurrency,paygcostinbillingcurrency,ConsumedService,InstanceId,Tags,AdditionalInfo,resourcelocation
+11111111-12ab-34dc-56ef-123456abcdef,11111111-12ab-34dc-56ef-123456billing,2021-02-01,Virtual Machines,4,5,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"{""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss""}","{ ""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": ""aks-nodepool1-12345678-vmss_0"",  ""VCPUs"": 2  }",""

pkg/cloud/azure/storagebillingparser.go

@@ -69,7 +69,8 @@ func (asbp *AzureStorageBillingParser) ParseBillingData(start, end time.Time, re
 		for _, blob := range blobInfos {
 			blobName := *blob.Name
 
-			localFilePath := filepath.Join(localPath, filepath.Base(blobName))
+			// Use entire blob name to prevent collision with other files from previous months or other integrations (ex "part_0_0001.csv")
+			localFilePath := filepath.Join(localPath, strings.ReplaceAll(blobName, "/", "_"))
 
 			err := asbp.DownloadBlobToFile(localFilePath, blob, client, ctx)
 			if err != nil {
@@ -232,7 +233,7 @@ func (asbp *AzureStorageBillingParser) getMostRecentBlobs(start, end time.Time,
 		if err != nil {
 			return nil, fmt.Errorf("failed to retrieve manifest %w", err)
 		}
-		
+
 		var manifest manifestJson
 		err = json.Unmarshal(manifestBytes, &manifest)
 		if err != nil {
@@ -294,38 +295,3 @@ func (asbp *AzureStorageBillingParser) timeToMonthString(input time.Time) string
 	endOfMonth := input.AddDate(0, 1, -input.Day())
 	return startOfMonth.Format(format) + "-" + endOfMonth.Format(format)
 }
-
-// deleteFilesOlderThan7d recursively walks the directory specified and deletes
-// files which have not been modified in the last 7 days. Returns a list of
-// files deleted.
-func (asbp *AzureStorageBillingParser) deleteFilesOlderThan7d(localPath string) ([]string, error) {
-	duration := 7 * 24 * time.Hour
-	cleaned := []string{}
-	errs := []string{}
-
-	if _, err := os.Stat(localPath); err != nil {
-		return cleaned, nil // localPath does not exist
-	}
-
-	filepath.Walk(localPath, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			errs = append(errs, err.Error())
-			return err
-		}
-
-		if time.Since(info.ModTime()) > duration {
-			err := os.Remove(path)
-			if err != nil {
-				errs = append(errs, err.Error())
-			}
-			cleaned = append(cleaned, path)
-		}
-		return nil
-	})
-
-	if len(errs) == 0 {
-		return cleaned, nil
-	} else {
-		return cleaned, fmt.Errorf("deleteFilesOlderThan7d: %v", errs)
-	}
-}

pkg/cloud/azure/storageconnection.go

@@ -7,6 +7,8 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
+	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
@@ -17,6 +19,7 @@ import (
 // StorageConnection provides access to Azure Storage
 type StorageConnection struct {
 	StorageConfiguration
+	lock             sync.Mutex
 	ConnectionStatus cloud.ConnectionStatus
 }
 
@@ -82,6 +85,9 @@ func (sc *StorageConnection) StreamBlob(blobName string, client *azblob.Client)
 
 // DownloadBlobToFile downloads the Azure Billing CSV to a local file
 func (sc *StorageConnection) DownloadBlobToFile(localFilePath string, blob container.BlobItem, client *azblob.Client, ctx context.Context) error {
+	// Lock to prevent accessing a file which may not be fully downloaded
+	sc.lock.Lock()
+	defer sc.lock.Unlock()
 	blobName := *blob.Name
 	// Check if file already exists
 	if fileInfo, err := os.Stat(localFilePath); err == nil {
@@ -107,12 +113,59 @@ func (sc *StorageConnection) DownloadBlobToFile(localFilePath string, blob conta
 	defer fp.Close()
 
 	// Download newest Azure Billing CSV to disk
+
+	// Time out to prevent deadlock on download
+	timeoutCtx, cancel := context.WithTimeout(ctx, 30*time.Minute)
+	defer cancel()
+
 	log.Infof("CloudCost: Azure: DownloadBlobToFile: retrieving blob: %v", blobName)
-	filesize, err := client.DownloadFile(ctx, sc.Container, blobName, fp, nil)
+	filesize, err := client.DownloadFile(timeoutCtx, sc.Container, blobName, fp, nil)
 	if err != nil {
+		// Clean up file from failed download
+		err2 := os.Remove(localFilePath)
+		if err2 != nil {
+			log.Errorf("CloudCost: Azure: DownloadBlobToFile: failed to remove file %s after failed download %s", localFilePath, err2.Error())
+		}
 		return fmt.Errorf("CloudCost: Azure: DownloadBlobToFile: failed to download %w", err)
 	}
 	log.Infof("CloudCost: Azure: DownloadBlobToFile: retrieved %v of size %dMB", blobName, filesize/1024/1024)
 
 	return nil
 }
+
+// deleteFilesOlderThan7d recursively walks the directory specified and deletes
+// files which have not been modified in the last 7 days. Returns a list of
+// files deleted.
+func (sc *StorageConnection) deleteFilesOlderThan7d(localPath string) ([]string, error) {
+	sc.lock.Lock()
+	defer sc.lock.Unlock()
+	duration := 7 * 24 * time.Hour
+	cleaned := []string{}
+	errs := []string{}
+
+	if _, err := os.Stat(localPath); err != nil {
+		return cleaned, nil // localPath does not exist
+	}
+
+	filepath.Walk(localPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			errs = append(errs, err.Error())
+			return err
+		}
+
+		if time.Since(info.ModTime()) > duration {
+			err := os.Remove(path)
+			if err != nil {
+				errs = append(errs, err.Error())
+			}
+			cleaned = append(cleaned, path)
+		}
+		return nil
+	})
+
+	if len(errs) == 0 {
+		return cleaned, nil
+	} else {
+		return cleaned, fmt.Errorf("deleteFilesOlderThan7d: %v", errs)
+	}
+}

pkg/cloud/config/configurations.go

@@ -10,6 +10,7 @@ import (
 	"github.com/opencost/opencost/pkg/cloud/aws"
 	"github.com/opencost/opencost/pkg/cloud/azure"
 	"github.com/opencost/opencost/pkg/cloud/gcp"
+	"github.com/opencost/opencost/pkg/cloud/oracle"
 )
 
 // MultiCloudConfig struct is used to unmarshal cloud configs for each provider out of cloud-integration file
@@ -66,22 +67,31 @@ type Configurations struct {
 	GCP     *GCPConfigs     `json:"gcp,omitempty"`
 	Azure   *AzureConfigs   `json:"azure,omitempty"`
 	Alibaba *AlibabaConfigs `json:"alibaba,omitempty"`
+	OCI     *OCIConfigs     `json:"oci,omitempty"`
 }
 
 // UnmarshalJSON custom json unmarshalling to maintain support for MultiCloudConfig format
 func (c *Configurations) UnmarshalJSON(bytes []byte) error {
-	// Attempt to unmarshal into old config object
-	multiConfig := &MultiCloudConfig{}
-	err := json.Unmarshal(bytes, multiConfig)
-	// If unmarshal is successful, move values into config and return
-	if err == nil {
-		multiConfig.loadConfigurations(c)
-		return nil
-	}
+	// This has been tested for backwards compatability, and it works in both config formats.
+	// It also coincidentally works if you mix-and-match both the old format and the new
+	// format.
 	// Create inline type to gain access to default Unmarshalling
 	type ConfUnmarshaller *Configurations
 	var conf ConfUnmarshaller = c
-	return json.Unmarshal(bytes, conf)
+	err := json.Unmarshal(bytes, conf)
+	// If unmarshal is successful, return
+	if err == nil {
+		return nil
+	}
+
+	// Attempt to unmarshal into old config object
+	multiConfig := &MultiCloudConfig{}
+	err = json.Unmarshal(bytes, multiConfig)
+	if err != nil {
+		return err
+	}
+	multiConfig.loadConfigurations(c)
+	return nil
 }
 
 func (c *Configurations) Equals(that *Configurations) bool {
@@ -108,6 +118,10 @@ func (c *Configurations) Equals(that *Configurations) bool {
 		return false
 	}
 
+	if !c.OCI.Equals(that.OCI) {
+		return false
+	}
+
 	return true
 }
 
@@ -138,6 +152,11 @@ func (c *Configurations) Insert(keyedConfig cloud.Config) error {
 			c.Alibaba = &AlibabaConfigs{}
 		}
 		c.Alibaba.BOA = append(c.Alibaba.BOA, keyedConfig.(*alibaba.BOAConfiguration))
+	case *oracle.UsageApiConfiguration:
+		if c.OCI == nil {
+			c.OCI = &OCIConfigs{}
+		}
+		c.OCI.UsageAPI = append(c.OCI.UsageAPI, keyedConfig.(*oracle.UsageApiConfiguration))
 	default:
 		return fmt.Errorf("Configurations: Insert: failed to insert config of type: %T", keyedConfig)
 	}
@@ -174,6 +193,12 @@ func (c *Configurations) ToSlice() []cloud.KeyedConfig {
 		}
 	}
 
+	if c.OCI != nil {
+		for _, usageConfig := range c.OCI.UsageAPI {
+			keyedConfigs = append(keyedConfigs, usageConfig)
+		}
+	}
+
 	return keyedConfigs
 
 }
@@ -289,3 +314,28 @@ func (ac *AlibabaConfigs) Equals(that *AlibabaConfigs) bool {
 
 	return true
 }
+
+type OCIConfigs struct {
+	UsageAPI []*oracle.UsageApiConfiguration `json:"usageApi,omitempty"`
+}
+
+func (oc *OCIConfigs) Equals(that *OCIConfigs) bool {
+	if oc == nil && that == nil {
+		return true
+	}
+	if oc == nil || that == nil {
+		return false
+	}
+	// Check Usage API
+	if len(oc.UsageAPI) != len(that.UsageAPI) {
+		return false
+	}
+	for i, thisUsageAPI := range oc.UsageAPI {
+		thatUsageAPI := that.UsageAPI[i]
+		if !thisUsageAPI.Equals(thatUsageAPI) {
+			return false
+		}
+	}
+
+	return true
+}

pkg/cloud/config/statuses.go

@@ -9,6 +9,7 @@ import (
 	"github.com/opencost/opencost/pkg/cloud/aws"
 	"github.com/opencost/opencost/pkg/cloud/azure"
 	"github.com/opencost/opencost/pkg/cloud/gcp"
+	"github.com/opencost/opencost/pkg/cloud/oracle"
 )
 
 const (
@@ -16,6 +17,7 @@ const (
 	AthenaConfigType       = "athena"
 	BigQueryConfigType     = "bigquery"
 	AzureStorageConfigType = "azurestorage"
+	UsageApiConfigType     = "usageapi"
 )
 
 func ConfigTypeFromConfig(config cloud.KeyedConfig) (string, error) {
@@ -28,6 +30,8 @@ func ConfigTypeFromConfig(config cloud.KeyedConfig) (string, error) {
 		return BigQueryConfigType, nil
 	case *azure.StorageConfiguration:
 		return AzureStorageConfigType, nil
+	case *oracle.UsageApiConfiguration:
+		return UsageApiConfigType, nil
 	}
 	return "", fmt.Errorf("failed to config type for config with key: %s, type %T", config.Key(), config)
 }
@@ -114,6 +118,8 @@ func (s *Status) UnmarshalJSON(b []byte) error {
 		config = &gcp.BigQueryConfiguration{}
 	case AzureStorageConfigType:
 		config = &azure.StorageConfiguration{}
+	case UsageApiConfigType:
+		config = &oracle.UsageApiConfiguration{}
 	default:
 		return fmt.Errorf("Status: UnmarshalJSON: config type '%s' is not recognized", configType)
 	}

pkg/cloud/gcp/bigqueryintegration.go

@@ -20,6 +20,9 @@ const (
 	UsageDateColumnName          = "usage_date"
 	BillingAccountIDColumnName   = "billing_id"
 	ProjectIDColumnName          = "project_id"
+	ProjectNameColumnName        = "project_name"
+	RegionColumnName             = "region"
+	ZoneColumnName               = "zone"
 	ServiceDescriptionColumnName = "service"
 	SKUDescriptionColumnName     = "description"
 	LabelsColumnName             = "labels"
@@ -44,6 +47,9 @@ func (bqi *BigQueryIntegration) GetCloudCost(start time.Time, end time.Time) (*o
 		fmt.Sprintf("TIMESTAMP_TRUNC(usage_start_time, day) as %s", UsageDateColumnName),
 		fmt.Sprintf("billing_account_id as %s", BillingAccountIDColumnName),
 		fmt.Sprintf("project.id as %s", ProjectIDColumnName),
+		fmt.Sprintf("project.name as %s", ProjectNameColumnName),
+		fmt.Sprintf("location.region as %s", RegionColumnName),
+		fmt.Sprintf("location.zone as %s", ZoneColumnName),
 		fmt.Sprintf("service.description as %s", ServiceDescriptionColumnName),
 		fmt.Sprintf("sku.description as %s", SKUDescriptionColumnName),
 		fmt.Sprintf("resource.name as %s", ResourceNameColumnName),
@@ -58,6 +64,9 @@ func (bqi *BigQueryIntegration) GetCloudCost(start time.Time, end time.Time) (*o
 		UsageDateColumnName,
 		BillingAccountIDColumnName,
 		ProjectIDColumnName,
+		ProjectNameColumnName,
+		RegionColumnName,
+		ZoneColumnName,
 		ServiceDescriptionColumnName,
 		SKUDescriptionColumnName,
 		LabelsColumnName,
@@ -181,7 +190,7 @@ func (bqi *BigQueryIntegration) queryFlexibleCUDTotalCosts(start time.Time, end
 		  IFNULL(SUM((Select SUM(amount) FROM bd.credits)),0),
 		FROM %s
 		WHERE %s
-		GROUP BY usage_date, sku.description
+		GROUP BY usage_date
 	`
 
 	table := fmt.Sprintf(" `%s` bd ", bqi.GetBillingDataDataset())
@@ -214,7 +223,7 @@ func (bqi *BigQueryIntegration) queryFlexibleCUDTotalCredits(start time.Time, en
 	FROM %s
 	CROSS JOIN UNNEST(bd.credits) AS credits
 	WHERE %s
-	GROUP BY usage_date, credits.id
+	GROUP BY usage_date
 	`
 
 	table := fmt.Sprintf(" `%s` bd ", bqi.GetBillingDataDataset())

pkg/cloud/gcp/bigqueryintegration_test.go

@@ -13,7 +13,7 @@ import (
 func TestBigQueryIntegration_GetCloudCost(t *testing.T) {
 	bigQueryConfigPath := os.Getenv("BIGQUERY_CONFIGURATION")
 	if bigQueryConfigPath == "" {
-		t.Skip("skipping integration test, set environment variable ATHENA_CONFIGURATION")
+		t.Skip("skipping integration test, set environment variable BIGQUERY_CONFIGURATION\"")
 	}
 	bigQueryConfigBin, err := os.ReadFile(bigQueryConfigPath)
 	if err != nil {

pkg/cloud/gcp/bigqueryintegration_types.go

@@ -61,6 +61,8 @@ func (ccl *CloudCostLoader) Load(values []bigquery.Value, schema bigquery.Schema
 				invoiceEntityID = ""
 			}
 			properties.InvoiceEntityID = invoiceEntityID
+			// Use InvoiceEntityID as InvoiceEntityName
+			properties.InvoiceEntityName = invoiceEntityID
 		case ProjectIDColumnName:
 			accountID, ok := values[i].(string)
 			if !ok {
@@ -68,6 +70,27 @@ func (ccl *CloudCostLoader) Load(values []bigquery.Value, schema bigquery.Schema
 				accountID = ""
 			}
 			properties.AccountID = accountID
+		case ProjectNameColumnName:
+			accountName, ok := values[i].(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", ProjectNameColumnName, values[i])
+				accountName = ""
+			}
+			properties.AccountName = accountName
+		case RegionColumnName:
+			regionID, ok := values[i].(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", RegionColumnName, values[i])
+				regionID = ""
+			}
+			properties.RegionID = regionID
+		case ZoneColumnName:
+			zone, ok := values[i].(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", ZoneColumnName, values[i])
+				zone = ""
+			}
+			properties.AvailabilityZone = zone
 		case ServiceDescriptionColumnName:
 			service, ok := values[i].(string)
 			if !ok {

pkg/cloud/gcp/provider.go

@@ -495,7 +495,8 @@ func (gcp *GCP) GetOrphanedResources() ([]models.OrphanedResource, error) {
 				desc := map[string]string{}
 				if disk.Description != "" {
 					if err := json.Unmarshal([]byte(disk.Description), &desc); err != nil {
-						return nil, fmt.Errorf("error converting string to map: %s", err)
+						log.Errorf("ignoring orphaned disk %s, failed to convert disk description to map: %s", disk.Name, err)
+						continue
 					}
 				}
 

pkg/cloud/oracle/authorizer.go

@@ -0,0 +1,130 @@
+package oracle
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/oracle/oci-go-sdk/v65/common"
+)
+
+const RawConfigProviderAuthorizerType = "OCIRawConfigProvider"
+
+// Authorizer provides which is used in when creating clients in the OCI SDK
+type Authorizer interface {
+	cloud.Authorizer
+	CreateOCIConfig() (common.ConfigurationProvider, error)
+}
+
+// SelectAuthorizerByType is an implementation of AuthorizerSelectorFn and acts as a register for Authorizer types
+func SelectAuthorizerByType(typeStr string) (Authorizer, error) {
+	switch typeStr {
+	case RawConfigProviderAuthorizerType:
+		return &RawConfigProvider{}, nil
+	default:
+		return nil, fmt.Errorf("OCI: provider authorizer type '%s' is not valid", typeStr)
+	}
+}
+
+// RawConfigProvider holds OCI credentials and fulfils the common.ConfigurationProvider interface
+type RawConfigProvider struct {
+	TenancyID            string  `json:"tenancyID"`
+	UserID               string  `json:"userID"`
+	Region               string  `json:"region"`
+	Fingerprint          string  `json:"fingerprint"`
+	PrivateKey           string  `json:"privateKey"`
+	PrivateKeyPassphrase *string `json:"privateKeyPassphrase"`
+}
+
+// MarshalJSON custom json marshalling functions, sets properties as tagged in struct and sets the authorizer type property
+func (ak *RawConfigProvider) MarshalJSON() ([]byte, error) {
+	fmap := make(map[string]any, 6)
+	fmap[cloud.AuthorizerTypeProperty] = RawConfigProviderAuthorizerType
+	fmap["tenancyId"] = ak.TenancyID
+	fmap["userId"] = ak.UserID
+	fmap["region"] = ak.Region
+	fmap["fingerprint"] = ak.Fingerprint
+	fmap["privateKey"] = ak.PrivateKey
+	fmap["privateKeyPassphrase"] = ak.PrivateKeyPassphrase
+	return json.Marshal(fmap)
+}
+
+func (ak *RawConfigProvider) Validate() error {
+	if ak.TenancyID == "" {
+		return fmt.Errorf("RawConfigProvider: missing tenancy ID")
+	}
+	if ak.UserID == "" {
+		return fmt.Errorf("RawConfigProvider: missing user ID")
+	}
+	if ak.Fingerprint == "" {
+		return fmt.Errorf("RawConfigProvider: missing key fingerprint")
+	}
+	if ak.Region == "" {
+		return fmt.Errorf("RawConfigProvider: missing region")
+	}
+	if ak.PrivateKey == "" {
+		return fmt.Errorf("RawConfigProvider: missing private key")
+	}
+	if ak.PrivateKeyPassphrase != nil {
+		if *ak.PrivateKeyPassphrase == "" {
+			return fmt.Errorf("RawConfigProvider: missing private key passphrase")
+		}
+	}
+
+	return nil
+}
+
+func (ak *RawConfigProvider) Equals(config cloud.Config) bool {
+	if config == nil {
+		return false
+	}
+	thatConfig, ok := config.(*RawConfigProvider)
+	if !ok {
+		return false
+	}
+
+	if ak.TenancyID != thatConfig.TenancyID {
+		return false
+	}
+	if ak.UserID != thatConfig.UserID {
+		return false
+	}
+	if ak.Fingerprint != thatConfig.Fingerprint {
+		return false
+	}
+	if ak.Region != thatConfig.Region {
+		return false
+	}
+	if ak.PrivateKey != thatConfig.PrivateKey {
+		return false
+	}
+	if ak.PrivateKeyPassphrase == nil && thatConfig.PrivateKeyPassphrase != nil {
+		return false
+	}
+	if ak.PrivateKeyPassphrase != nil && thatConfig.PrivateKeyPassphrase == nil {
+		return false
+	}
+	if ak.PrivateKeyPassphrase != nil && thatConfig.PrivateKeyPassphrase != nil {
+		if *ak.PrivateKeyPassphrase != *thatConfig.PrivateKeyPassphrase {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (ak *RawConfigProvider) Sanitize() cloud.Config {
+	redacted := cloud.Redacted
+	return &RawConfigProvider{
+		TenancyID:            ak.TenancyID,
+		UserID:               ak.UserID,
+		Fingerprint:          ak.Fingerprint,
+		Region:               ak.Region,
+		PrivateKey:           cloud.Redacted,
+		PrivateKeyPassphrase: &redacted,
+	}
+}
+
+func (ak *RawConfigProvider) CreateOCIConfig() (common.ConfigurationProvider, error) {
+	return common.NewRawConfigurationProvider(ak.TenancyID, ak.UserID, ak.Region, ak.Fingerprint, ak.PrivateKey, ak.PrivateKeyPassphrase), nil
+}

pkg/cloud/oracle/partnumbers/shape_part_numbers.json

@@ -41,6 +41,12 @@
     "GPU": "B98415",
     "Disk": ""
   },
+  "BM.GPU.L40S.4": {
+    "OCPU": "",
+    "Memory": "",
+    "GPU": "B109479",
+    "Disk": ""
+  },
   "BM.GPU2.2": {
     "OCPU": "",
     "Memory": "",
@@ -185,6 +191,84 @@
     "GPU": "",
     "Disk": ""
   },
+  "PostgreSQL.VM.Standard.Flex.E4": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.16.256GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.2.32GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.32.512GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.4.64GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.64.1024GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard.Flex.E4.8.128GB": {
+    "OCPU": "B93113",
+    "Memory": "B93114",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex.16.256GB": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex.2.32GB": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex.32.512GB": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex.4.64GB": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
+  "PostgreSQL.VM.Standard3.Flex.8.128GB": {
+    "OCPU": "B92306",
+    "Memory": "B92307",
+    "GPU": "",
+    "Disk": ""
+  },
   "VM.DenseIO.E4.Flex": {
     "OCPU": "B93121",
     "Memory": "B93122",
@@ -227,6 +311,12 @@
     "GPU": "",
     "Disk": ""
   },
+  "VM.DenseIO2.8": {
+    "OCPU": "B88516",
+    "Memory": "",
+    "GPU": "",
+    "Disk": ""
+  },
   "VM.GPU.A10.1": {
     "OCPU": "",
     "Memory": "",
@@ -275,6 +365,12 @@
     "GPU": "",
     "Disk": ""
   },
+  "VM.Standard.A2.Flex": {
+    "OCPU": "B109529",
+    "Memory": "B109530",
+    "GPU": "",
+    "Disk": ""
+  },
   "VM.Standard.E3.Flex": {
     "OCPU": "B92306",
     "Memory": "B92307",
@@ -335,4 +431,4 @@
     "GPU": "",
     "Disk": ""
   }
-}
+}

pkg/cloud/oracle/provider.go

@@ -256,7 +256,7 @@ func (o *Oracle) Regions() []string {
 		log.Debugf("Overriding Oracle regions with configured region list: %+v", regionOverrides)
 		return regionOverrides
 	}
-	return oracleRegions
+	return oracleRegions()
 }
 
 func (o *Oracle) PricingSourceSummary() interface{} {

pkg/cloud/oracle/provider_test.go

@@ -73,6 +73,11 @@ func TestGetPVKey(t *testing.T) {
 	assert.Equal(t, providerID, pvkey.ID())
 }
 
+func TestRegions(t *testing.T) {
+	regions := (&Oracle{}).Regions()
+	assert.Len(t, regions, 39)
+}
+
 func testNode(gpus int) *v1.Node {
 	capacity := map[v1.ResourceName]resource.Quantity{}
 	if gpus > 0 {

pkg/cloud/oracle/region.go

@@ -2,40 +2,46 @@ package oracle
 
 // Regions retrieved from https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm.
 // May also be listed using the OCI CLI, "oci iam region list"
-var oracleRegions = []string{
-	"eu-amsterdam-1",
-	"eu-stockholm-1",
-	"me-abudhabi-1",
-	"ap-mumbai-1",
-	"eu-paris-1",
-	"uk-cardiff-1",
-	"me-dubai-1",
-	"eu-frankfurt-1",
-	"sa-saopaulo-1",
-	"ap-hyderabad-1",
-	"us-ashburn-1",
-	"ap-seoul-1",
-	"me-jeddah-1",
-	"af-johannesburg-1",
-	"ap-osaka-1",
-	"uk-london-1",
-	"eu-milan-1",
-	"eu-madrid-1",
-	"ap-melbourne-1",
-	"eu-marseille-1",
-	"mx-monterrey-1",
-	"il-jerusalem-1",
-	"ap-tokyo-1",
-	"us-chicago-1",
-	"us-phoenix-1",
-	"mx-queretaro-1",
-	"sa-santiago-1",
-	"ap-singapore-1",
-	"us-sanjose-1",
-	"ap-sydney-1",
-	"sa-vinhedo-1",
-	"ap-chuncheon-1",
-	"ca-montreal-1",
-	"ca-toronto-1",
-	"eu-zurich-1",
+func oracleRegions() []string {
+	return []string{
+		"af-johannesburg-1",
+		"ap-chuncheon-1",
+		"ap-hyderabad-1",
+		"ap-melbourne-1",
+		"ap-mumbai-1",
+		"ap-osaka-1",
+		"ap-seoul-1",
+		"ap-singapore-1",
+		"ap-singapore-2",
+		"ap-sydney-1",
+		"ap-tokyo-1",
+		"ca-montreal-1",
+		"ca-toronto-1",
+		"eu-amsterdam-1",
+		"eu-frankfurt-1",
+		"eu-madrid-1",
+		"eu-marseille-1",
+		"eu-milan-1",
+		"eu-paris-1",
+		"eu-stockholm-1",
+		"eu-zurich-1",
+		"il-jerusalem-1",
+		"me-abudhabi-1",
+		"me-dubai-1",
+		"me-jeddah-1",
+		"me-riyadh-1",
+		"mx-monterrey-1",
+		"mx-queretaro-1",
+		"sa-bogota-1",
+		"sa-santiago-1",
+		"sa-saopaulo-1",
+		"sa-valparaiso-1",
+		"sa-vinhedo-1",
+		"uk-cardiff-1",
+		"uk-london-1",
+		"us-ashburn-1",
+		"us-chicago-1",
+		"us-phoenix-1",
+		"us-sanjose-1",
+	}
 }

pkg/cloud/oracle/region_test.go

@@ -0,0 +1,17 @@
+package oracle
+
+import (
+	"testing"
+
+	"github.com/oracle/oci-go-sdk/v65/common"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRegionValidation(t *testing.T) {
+	for _, r := range oracleRegions() {
+		// Use the OCI SDK to validate static regions.
+		region := common.StringToRegion(r)
+		_, err := region.RealmID()
+		assert.NoError(t, err)
+	}
+}

pkg/cloud/oracle/usageapiconfiguration.go

@@ -0,0 +1,131 @@
+package oracle
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/opencost/opencost/core/pkg/opencost"
+	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/oracle/oci-go-sdk/v65/usageapi"
+)
+
+type UsageApiConfiguration struct {
+	TenancyID  string     `json:"tenancyID"`
+	Region     string     `json:"region"`
+	Authorizer Authorizer `json:"authorizer"`
+}
+
+func (uac *UsageApiConfiguration) Validate() error {
+	// Validate Authorizer
+	if uac.Authorizer == nil {
+		return fmt.Errorf("UsageApiConfiguration: missing Authorizer")
+	}
+
+	err := uac.Authorizer.Validate()
+	if err != nil {
+		return fmt.Errorf("UsageApiConfiguration: %s", err)
+	}
+
+	// Validate base properties
+	if uac.TenancyID == "" {
+		return fmt.Errorf("UsageApiConfiguration: missing tenancyID")
+	}
+
+	if uac.Region == "" {
+		return fmt.Errorf("UsageApiConfiguration: missing region")
+	}
+
+	return nil
+}
+
+func (uac *UsageApiConfiguration) Equals(config cloud.Config) bool {
+	if config == nil {
+		return false
+	}
+	thatConfig, ok := config.(*UsageApiConfiguration)
+	if !ok {
+		return false
+	}
+
+	if uac.Authorizer != nil {
+		if !uac.Authorizer.Equals(thatConfig.Authorizer) {
+			return false
+		}
+	} else {
+		if thatConfig.Authorizer != nil {
+			return false
+		}
+	}
+
+	if uac.TenancyID != thatConfig.TenancyID {
+		return false
+	}
+
+	if uac.Region != thatConfig.Region {
+		return false
+	}
+
+	return true
+}
+
+func (uac *UsageApiConfiguration) Sanitize() cloud.Config {
+	return &UsageApiConfiguration{
+		TenancyID:  uac.TenancyID,
+		Region:     uac.Region,
+		Authorizer: uac.Authorizer.Sanitize().(Authorizer),
+	}
+}
+
+func (uac *UsageApiConfiguration) Key() string {
+	return uac.TenancyID
+}
+
+func (uac *UsageApiConfiguration) Provider() string {
+	return opencost.OracleProvider
+}
+
+func (uac *UsageApiConfiguration) GetUsageApiClient() (*usageapi.UsageapiClient, error) {
+	configProvider, err := uac.Authorizer.CreateOCIConfig()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create oci config: %s", err.Error())
+	}
+	client, err := usageapi.NewUsageapiClientWithConfigurationProvider(configProvider)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create usage api client: %s", err.Error())
+	}
+	return &client, nil
+}
+
+func (uac *UsageApiConfiguration) UnmarshalJSON(b []byte) error {
+	var f interface{}
+	err := json.Unmarshal(b, &f)
+	if err != nil {
+		return err
+	}
+
+	fmap := f.(map[string]interface{})
+
+	tenancyId, err := cloud.GetInterfaceValue[string](fmap, "tenancyID")
+	if err != nil {
+		return fmt.Errorf("UsageApiConfiguration: UnmarshalJSON: %w", err)
+	}
+	uac.TenancyID = tenancyId
+
+	region, err := cloud.GetInterfaceValue[string](fmap, "region")
+	if err != nil {
+		return fmt.Errorf("UsageApiConfiguration: UnmarshalJSON: %w", err)
+	}
+	uac.Region = region
+
+	authAny, ok := fmap["authorizer"]
+	if !ok {
+		return fmt.Errorf("UsageApiConfiguration: UnmarshalJSON: missing authorizer")
+	}
+	authorizer, err := cloud.AuthorizerFromInterface(authAny, SelectAuthorizerByType)
+	if err != nil {
+		return fmt.Errorf("UsageApiConfiguration: UnmarshalJSON: %w", err)
+	}
+	uac.Authorizer = authorizer
+
+	return nil
+}

pkg/cloud/oracle/usageapiconfiguration_test.go

@@ -0,0 +1,318 @@
+package oracle
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/opencost/opencost/core/pkg/log"
+	"github.com/opencost/opencost/core/pkg/util/json"
+	"github.com/opencost/opencost/pkg/cloud"
+)
+
+func TestUsageApiConfiguration_Validate(t *testing.T) {
+	testCases := map[string]struct {
+		config   UsageApiConfiguration
+		expected error
+	}{
+		"valid config OCI Key": {
+			config: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			expected: nil,
+		},
+		"invalid authorizer": {
+			config: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "",
+					PrivateKey:  "",
+				},
+			},
+			expected: fmt.Errorf("UsageApiConfiguration: RawConfigProvider: missing key fingerprint"),
+		},
+		"missing authorizer": {
+			config: UsageApiConfiguration{
+				TenancyID:  "tenancyID",
+				Region:     "region",
+				Authorizer: nil,
+			},
+			expected: fmt.Errorf("UsageApiConfiguration: missing Authorizer"),
+		},
+		"missing tenancyID": {
+			config: UsageApiConfiguration{
+				TenancyID: "",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			expected: fmt.Errorf("UsageApiConfiguration: missing tenancyID"),
+		},
+		"missing region": {
+			config: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			expected: fmt.Errorf("UsageApiConfiguration: missing region"),
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual := testCase.config.Validate()
+			actualString := "nil"
+			if actual != nil {
+				actualString = actual.Error()
+			}
+			expectedString := "nil"
+			if testCase.expected != nil {
+				expectedString = testCase.expected.Error()
+			}
+			if actualString != expectedString {
+				t.Errorf("errors do not match: Actual: '%s', Expected: '%s", actualString, expectedString)
+			}
+		})
+	}
+}
+
+func TestUsageApiConfiguration_Equals(t *testing.T) {
+	testCases := map[string]struct {
+		left     UsageApiConfiguration
+		right    cloud.Config
+		expected bool
+	}{
+		"matching config": {
+			left: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			right: &UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			expected: true,
+		},
+		"different configurer": {
+			left: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint2",
+					PrivateKey:  "key",
+				},
+			},
+			right: &UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			expected: false,
+		},
+		"missing both configurer": {
+			left: UsageApiConfiguration{
+				TenancyID:  "tenancyID",
+				Region:     "region",
+				Authorizer: nil,
+			},
+			right: &UsageApiConfiguration{
+				TenancyID:  "tenancyID",
+				Region:     "region",
+				Authorizer: nil,
+			},
+			expected: true,
+		},
+		"missing left configurer": {
+			left: UsageApiConfiguration{
+				TenancyID:  "tenancyID",
+				Region:     "region",
+				Authorizer: nil,
+			},
+			right: &UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			expected: false,
+		},
+		"missing right configurer": {
+			left: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			right: &UsageApiConfiguration{
+				TenancyID:  "tenancyID",
+				Region:     "region",
+				Authorizer: nil,
+			},
+			expected: false,
+		},
+		"different tenancyID": {
+			left: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			right: &UsageApiConfiguration{
+				TenancyID: "tenancyID2",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID2",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			expected: false,
+		},
+		"different region": {
+			left: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			right: &UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region2",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region2",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+			expected: false,
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual := testCase.left.Equals(testCase.right)
+			if actual != testCase.expected {
+				t.Errorf("incorrect result: Actual: '%t', Expected: '%t", actual, testCase.expected)
+			}
+		})
+	}
+}
+
+func TestUsageApiConfiguration_JSON(t *testing.T) {
+	testCases := map[string]struct {
+		config UsageApiConfiguration
+	}{
+		"Empty Config": {
+			config: UsageApiConfiguration{},
+		},
+		"Nil Authorizer": {
+			config: UsageApiConfiguration{
+				TenancyID:  "tenancyID",
+				Region:     "region",
+				Authorizer: nil,
+			},
+		},
+		"RawConfigProviderAuthorizer": {
+			config: UsageApiConfiguration{
+				TenancyID: "tenancyID",
+				Region:    "region",
+				Authorizer: &RawConfigProvider{
+					TenancyID:   "tenancyID",
+					UserID:      "userID",
+					Region:      "region2",
+					Fingerprint: "fingerprint",
+					PrivateKey:  "key",
+				},
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+
+			// test JSON Marshalling
+			configJSON, err := json.Marshal(testCase.config)
+			if err != nil {
+				t.Errorf("failed to marshal configuration: %s", err.Error())
+			}
+			log.Info(string(configJSON))
+			unmarshalledConfig := &UsageApiConfiguration{}
+			err = json.Unmarshal(configJSON, unmarshalledConfig)
+			if err != nil {
+				t.Errorf("failed to unmarshal configuration: %s", err.Error())
+			}
+			if !testCase.config.Equals(unmarshalledConfig) {
+				t.Error("config does not equal unmarshalled config")
+			}
+		})
+	}
+}

pkg/cloud/oracle/usageapiintegration.go

@@ -0,0 +1,171 @@
+package oracle
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/opencost/opencost/core/pkg/opencost"
+	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/oracle/oci-go-sdk/v65/common"
+	"github.com/oracle/oci-go-sdk/v65/usageapi"
+)
+
+type UsageApiIntegration struct {
+	UsageApiConfiguration
+	ConnectionStatus cloud.ConnectionStatus
+}
+
+func (uai *UsageApiIntegration) GetCloudCost(start time.Time, end time.Time) (*opencost.CloudCostSetRange, error) {
+	client, err := uai.GetUsageApiClient()
+	if err != nil {
+		uai.ConnectionStatus = cloud.FailedConnection
+		return nil, fmt.Errorf("getting oracle usage api client: %s", err.Error())
+	}
+
+	req := usageapi.RequestSummarizedUsagesRequest{
+		RequestSummarizedUsagesDetails: usageapi.RequestSummarizedUsagesDetails{
+			Granularity:       usageapi.RequestSummarizedUsagesDetailsGranularityDaily,
+			GroupBy:           []string{"resourceId", "service", "subscriptionId", "tenantName"},
+			IsAggregateByTime: common.Bool(false),
+			TimeUsageStarted:  &common.SDKTime{Time: start},
+			TimeUsageEnded:    &common.SDKTime{Time: end},
+			QueryType:         usageapi.RequestSummarizedUsagesDetailsQueryTypeCost,
+			TenantId:          common.String(uai.TenancyID),
+		},
+		Limit: common.Int(500),
+	}
+
+	resp, err := client.RequestSummarizedUsages(context.Background(), req)
+	if err != nil {
+		uai.ConnectionStatus = cloud.FailedConnection
+		return nil, fmt.Errorf("failed to query usage: %w", err)
+	}
+
+	ccsr, err := opencost.NewCloudCostSetRange(start, end, opencost.AccumulateOptionDay, uai.Key())
+	if err != nil {
+		return nil, err
+	}
+
+	// Set status to missing data if query comes back empty and the status isn't already successful
+	if len(resp.Items) == 0 && uai.ConnectionStatus != cloud.SuccessfulConnection {
+		uai.ConnectionStatus = cloud.MissingData
+		return ccsr, nil
+	}
+
+	for _, item := range resp.Items {
+		resourceId := ""
+		if item.ResourceId != nil {
+			resourceId = *item.ResourceId
+		}
+
+		tenantName := ""
+		if item.TenantName != nil {
+			tenantName = *item.TenantName
+		}
+
+		subscriptionId := ""
+		if item.SubscriptionId != nil {
+			subscriptionId = *item.SubscriptionId
+		}
+
+		service := ""
+		if item.Service != nil {
+			service = *item.Service
+		}
+
+		category := SelectOCICategory(service)
+
+		// Iterate through the slice of tags, assigning
+		// keys and values to the map of labels
+		labels := opencost.CloudCostLabels{}
+		for _, tag := range item.Tags {
+			if tag.Key == nil || tag.Value == nil {
+				continue
+			}
+			labels[*tag.Key] = *tag.Value
+		}
+
+		properties := &opencost.CloudCostProperties{
+			ProviderID:      resourceId,
+			Provider:        opencost.OracleProvider,
+			AccountID:       uai.TenancyID,
+			AccountName:     tenantName,
+			InvoiceEntityID: subscriptionId,
+			RegionID:        uai.Region,
+			Service:         service,
+			Category:        category,
+			Labels:          labels,
+		}
+
+		winStart := item.TimeUsageStarted.Time
+		winEnd := start.AddDate(0, 0, 1)
+
+		listRate := 0.0
+		if item.ListRate != nil {
+			listRate = float64(*item.ListRate)
+		}
+
+		attrCostToParse := ""
+		if item.AttributedCost != nil {
+			attrCostToParse = *item.AttributedCost
+		}
+
+		attrCost, err := strconv.ParseFloat(attrCostToParse, 64)
+		if err != nil {
+			return nil, fmt.Errorf("unable to parse float '%s': %s", attrCostToParse, err.Error())
+		}
+
+		computedAmt := 0.0
+		if item.ComputedAmount != nil {
+			computedAmt = float64(*item.ComputedAmount)
+		}
+
+		cc := &opencost.CloudCost{
+			Properties: properties,
+			Window:     opencost.NewWindow(&winStart, &winEnd),
+			//todo: which returned costs go where?
+			ListCost: opencost.CostMetric{
+				Cost: listRate,
+			},
+			NetCost: opencost.CostMetric{
+				Cost: computedAmt,
+			},
+			AmortizedNetCost: opencost.CostMetric{
+				Cost: attrCost,
+			},
+			AmortizedCost: opencost.CostMetric{
+				Cost: attrCost,
+			},
+			InvoicedCost: opencost.CostMetric{
+				Cost: computedAmt,
+			},
+		}
+
+		ccsr.LoadCloudCost(cc)
+	}
+
+	uai.ConnectionStatus = cloud.SuccessfulConnection
+	return ccsr, nil
+}
+
+func (uai *UsageApiIntegration) GetStatus() cloud.ConnectionStatus {
+	// initialize status if it has not done so; this can happen if the integration is inactive
+	if uai.ConnectionStatus.String() == "" {
+		uai.ConnectionStatus = cloud.InitialStatus
+	}
+	return uai.ConnectionStatus
+}
+
+func SelectOCICategory(service string) string {
+	if service == "Compute" {
+		return opencost.ComputeCategory
+	} else if service == "Block Storage" || service == "Object Storage" {
+		return opencost.StorageCategory
+	} else if service == "Load Balancer" || service == "Virtual Cloud Network" {
+		return opencost.NetworkCategory
+	} else {
+		return opencost.OtherCategory
+	}
+}

pkg/cloud/oracle/usageapiintegration_test.go

@@ -0,0 +1,61 @@
+package oracle
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/opencost/opencost/core/pkg/util/timeutil"
+)
+
+func TestUsageAPIIntegration_GetCloudCost(t *testing.T) {
+	usageApiConfigPath := os.Getenv("USAGEAPI_CONFIGURATION")
+	if usageApiConfigPath == "" {
+		t.Skip("skipping integration test, set environment variable USAGEAPI_CONFIGURATION")
+	}
+	usageApiConfigBin, err := os.ReadFile(usageApiConfigPath)
+	if err != nil {
+		t.Fatalf("failed to read config file: %s", err.Error())
+	}
+	var usageApiConfig UsageApiConfiguration
+	err = json.Unmarshal(usageApiConfigBin, &usageApiConfig)
+	if err != nil {
+		t.Fatalf("failed to unmarshal config from JSON: %s", err.Error())
+	}
+	testCases := map[string]struct {
+		integration *UsageApiIntegration
+		start       time.Time
+		end         time.Time
+		expected    bool
+	}{
+		// No CUR data is expected within 2 days of now
+		"too_recent_window": {
+			integration: &UsageApiIntegration{
+				UsageApiConfiguration: usageApiConfig,
+			},
+			end:      time.Now(),
+			start:    time.Now().Add(-timeutil.Day),
+			expected: true,
+		},
+		// CUR data should be available
+		"last week window": {
+			integration: &UsageApiIntegration{
+				UsageApiConfiguration: usageApiConfig,
+			},
+			end:      time.Now().Add(-7 * timeutil.Day),
+			start:    time.Now().Add(-8 * timeutil.Day),
+			expected: false,
+		},
+	}
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual, err := testCase.integration.GetCloudCost(testCase.start, testCase.end)
+			if err != nil {
+				t.Errorf("Other error during testing %s", err)
+			} else if actual.IsEmpty() != testCase.expected {
+				t.Errorf("Incorrect result, actual emptiness: %t, expected: %t", actual.IsEmpty(), testCase.expected)
+			}
+		})
+	}
+}

pkg/cloudcost/integration.go

@@ -9,6 +9,7 @@ import (
 	"github.com/opencost/opencost/pkg/cloud/aws"
 	"github.com/opencost/opencost/pkg/cloud/azure"
 	"github.com/opencost/opencost/pkg/cloud/gcp"
+	"github.com/opencost/opencost/pkg/cloud/oracle"
 )
 
 // CloudCostIntegration is an interface for retrieving daily granularity CloudCost data for a given range
@@ -57,15 +58,24 @@ func GetIntegrationFromConfig(kc cloud.KeyedConfig) CloudCostIntegration {
 	case *azure.StorageConnection:
 		return &azure.AzureStorageIntegration{
 			AzureStorageBillingParser: azure.AzureStorageBillingParser{
-				StorageConnection: *keyedConfig,
+				StorageConnection: azure.StorageConnection{
+					StorageConfiguration: keyedConfig.StorageConfiguration},
 			},
 		}
 	case *azure.AzureStorageBillingParser:
 		return &azure.AzureStorageIntegration{
-			AzureStorageBillingParser: *keyedConfig,
+			AzureStorageBillingParser: azure.AzureStorageBillingParser{
+				StorageConnection: azure.StorageConnection{
+					StorageConfiguration: keyedConfig.StorageConfiguration},
+			},
 		}
 	case *azure.AzureStorageIntegration:
-		return keyedConfig
+		return &azure.AzureStorageIntegration{
+			AzureStorageBillingParser: azure.AzureStorageBillingParser{
+				StorageConnection: azure.StorageConnection{
+					StorageConfiguration: keyedConfig.StorageConfiguration},
+			},
+		}
 	// S3SelectIntegration
 	case *aws.S3Configuration:
 		return &aws.S3SelectIntegration{
@@ -90,6 +100,10 @@ func GetIntegrationFromConfig(kc cloud.KeyedConfig) CloudCostIntegration {
 	// Alibaba BOA Integration
 	case *alibaba.BOAConfiguration:
 		return nil
+	case *oracle.UsageApiConfiguration:
+		return &oracle.UsageApiIntegration{
+			UsageApiConfiguration: *keyedConfig,
+		}
 	default:
 		return nil
 	}

pkg/costmodel/allocation.go

@@ -28,7 +28,7 @@ const (
 	queryFmtCPURequests                 = `avg(avg_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
 	queryFmtCPUUsageAvg                 = `avg(rate(container_cpu_usage_seconds_total{container!="", container_name!="POD", container!="POD", %s}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
 	queryFmtGPUsRequested               = `avg(avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
-	queryFmtGPUsUsageAvg                = `avg(avg_over_time(DCGM_FI_DEV_GPU_UTIL{container!=""}[%s])) by (container, pod, namespace, %s)`
+	queryFmtGPUsUsageAvg                = `avg(avg_over_time(DCGM_FI_PROF_GR_ENGINE_ACTIVE{container!=""}[%s])) by (container, pod, namespace, %s)`
 	queryFmtGPUsAllocated               = `avg(avg_over_time(container_gpu_allocation{container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
 	queryFmtNodeCostPerCPUHr            = `avg(avg_over_time(node_cpu_hourly_cost{%s}[%s])) by (node, %s, instance_type, provider_id)`
 	queryFmtNodeCostPerRAMGiBHr         = `avg(avg_over_time(node_ram_hourly_cost{%s}[%s])) by (node, %s, instance_type, provider_id)`

pkg/costmodel/allocation_helpers.go

@@ -615,7 +615,7 @@ func applyRAMBytesUsedMax(podMap map[podKey]*pod, resRAMBytesUsedMax []*prom.Que
 }
 
 func applyGPUUsageAvg(podMap map[podKey]*pod, resGPUUsageAvg []*prom.QueryResult, podUIDKeyMap map[podKey][]podKey) {
-	// Example PromQueryResult: {container="dcgmproftester12", namespace="gpu", pod="dcgmproftester3-deployment-fc89c8dd6-ph7z5"} 99
+	// Example PromQueryResult: {container="dcgmproftester12", namespace="gpu", pod="dcgmproftester3-deployment-fc89c8dd6-ph7z5"} 0.997307
 	for _, res := range resGPUUsageAvg {
 		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
 		if err != nil {
@@ -649,9 +649,8 @@ func applyGPUUsageAvg(podMap map[podKey]*pod, resGPUUsageAvg []*prom.QueryResult
 				thisPod.appendContainer(container)
 			}
 
-			// DCGM_FI_DEV_GPU_UTIL metric is a number 0-100. Scale down to a
-			// percentage so it is consistent with other fields.
-			thisPod.Allocations[container].GPUUsageAverage = res.Values[0].Value * 0.01
+			// DCGM_FI_PROF_GR_ENGINE_ACTIVE metric is a float between 0-1.
+			thisPod.Allocations[container].GPUUsageAverage = res.Values[0].Value
 		}
 	}
 }

pkg/costmodel/cluster.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/opencost/opencost/pkg/cloud/provider"
@@ -44,6 +45,15 @@ const (
 
 const MAX_LOCAL_STORAGE_SIZE = 1024 * 1024 * 1024 * 1024
 
+// When ASSET_INCLUDE_LOCAL_DISK_COST is set to false, local storage
+// provisioned by sig-storage-local-static-provisioner is excluded
+// by checking if the volume is prefixed by "local-pv-".
+//
+// This is based on the sig-storage-local-static-provisioner implementation,
+// which creates all PVs with the "local-pv-" prefix. For reference, see:
+// https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner/blob/b6f465027bd059e92c0032c81dd1e1d90e35c909/pkg/discovery/discovery.go#L410-L417
+const SIG_STORAGE_LOCAL_PROVISIONER_PREFIX = "local-pv-"
+
 // Costs represents cumulative and monthly cluster costs over a given duration. Costs
 // are broken down by cores, memory, and storage.
 type ClusterCosts struct {
@@ -532,6 +542,10 @@ func ClusterDisks(client prometheus.Client, cp models.Provider, start, end time.
 		}
 	}
 
+	if !env.GetAssetIncludeLocalDiskCost() {
+		return filterOutLocalPVs(diskMap), nil
+	}
+
 	return diskMap, nil
 }
 
@@ -1548,11 +1562,13 @@ func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActi
 				log.Debugf("ClusterDisks: pv claim data missing volumename")
 				continue
 			}
+
 			thatClaimName, err := thatRes.GetString("persistentvolumeclaim")
 			if err != nil {
 				log.Debugf("ClusterDisks: pv claim data missing persistentvolumeclaim")
 				continue
 			}
+
 			thatClaimNamespace, err := thatRes.GetString("namespace")
 			if err != nil {
 				log.Debugf("ClusterDisks: pv claim data missing namespace")
@@ -1589,6 +1605,7 @@ func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActi
 			log.Debugf("ClusterDisks: pv usage data missing persistentvolumeclaim")
 			continue
 		}
+
 		claimNamespace, err := result.GetString("namespace")
 		if err != nil {
 			log.Debugf("ClusterDisks: pv usage data missing namespace")
@@ -1609,11 +1626,13 @@ func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActi
 				log.Debugf("ClusterDisks: pv claim data missing volumename")
 				continue
 			}
+
 			thatClaimName, err := thatRes.GetString("persistentvolumeclaim")
 			if err != nil {
 				log.Debugf("ClusterDisks: pv claim data missing persistentvolumeclaim")
 				continue
 			}
+
 			thatClaimNamespace, err := thatRes.GetString("namespace")
 			if err != nil {
 				log.Debugf("ClusterDisks: pv claim data missing namespace")
@@ -1639,3 +1658,22 @@ func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActi
 		diskMap[key].BytesUsedMaxPtr = &usage
 	}
 }
+
+// filterOutLocalPVs removes local Persistent Volumes (PVs) from the given disk map.
+// Local PVs are identified by the prefix "local-pv-" in their names, which is the
+// convention used by sig-storage-local-static-provisioner.
+//
+// Parameters:
+//   - diskMap: A map of DiskIdentifier to Disk pointers, representing all PVs.
+//
+// Returns:
+//   - A new map of DiskIdentifier to Disk pointers, containing only non-local PVs.
+func filterOutLocalPVs(diskMap map[DiskIdentifier]*Disk) map[DiskIdentifier]*Disk {
+	nonLocalPVDiskMap := map[DiskIdentifier]*Disk{}
+	for key, val := range diskMap {
+		if !strings.HasPrefix(key.Name, SIG_STORAGE_LOCAL_PROVISIONER_PREFIX) {
+			nonLocalPVDiskMap[key] = val
+		}
+	}
+	return nonLocalPVDiskMap
+}

pkg/costmodel/cluster_test.go

@@ -0,0 +1,54 @@
+package costmodel
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_filterOutLocalPVs(t *testing.T) {
+	testCases := []struct {
+		name     string
+		input    map[DiskIdentifier]*Disk
+		expected map[DiskIdentifier]*Disk
+	}{
+		{
+			name: "Filter out local PVs",
+			input: map[DiskIdentifier]*Disk{
+				{Cluster: "cluster1", Name: "pv1"}:              &Disk{Name: "pv1"},
+				{Cluster: "cluster1", Name: "local-pv-123"}:     &Disk{Name: "local-pv-123"},
+				{Cluster: "cluster2", Name: "pv2"}:              &Disk{Name: "pv2"},
+				{Cluster: "cluster2", Name: "local-pv-456"}:     &Disk{Name: "local-pv-456"},
+				{Cluster: "cluster3", Name: "not-local-pv-789"}: &Disk{Name: "not-local-pv-789"},
+			},
+			expected: map[DiskIdentifier]*Disk{
+				{Cluster: "cluster1", Name: "pv1"}:              &Disk{Name: "pv1"},
+				{Cluster: "cluster2", Name: "pv2"}:              &Disk{Name: "pv2"},
+				{Cluster: "cluster3", Name: "not-local-pv-789"}: &Disk{Name: "not-local-pv-789"},
+			},
+		},
+		{
+			name: "No local PVs to filter",
+			input: map[DiskIdentifier]*Disk{
+				{Cluster: "cluster1", Name: "pv1"}: &Disk{Name: "pv1"},
+				{Cluster: "cluster2", Name: "pv2"}: &Disk{Name: "pv2"},
+			},
+			expected: map[DiskIdentifier]*Disk{
+				{Cluster: "cluster1", Name: "pv1"}: &Disk{Name: "pv1"},
+				{Cluster: "cluster2", Name: "pv2"}: &Disk{Name: "pv2"},
+			},
+		},
+		{
+			name:     "Empty input",
+			input:    map[DiskIdentifier]*Disk{},
+			expected: map[DiskIdentifier]*Disk{},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := filterOutLocalPVs(tc.input)
+			assert.Equal(t, tc.expected, result)
+		})
+	}
+}

pkg/costmodel/costmodel.go

@@ -988,15 +988,6 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 	nodeList := cm.Cache.GetAllNodes()
 	nodes := make(map[string]*costAnalyzerCloud.Node)
 
-	vgpuCount, err := getAllocatableVGPUs(cm.Cache)
-	if err != nil {
-		return nil, err
-	}
-	vgpuCoeff := 10.0
-	if vgpuCount > 0.0 {
-		vgpuCoeff = vgpuCount
-	}
-
 	pmd := &costAnalyzerCloud.PricingMatchMetadata{
 		TotalNodes:        0,
 		PricingTypeCounts: make(map[costAnalyzerCloud.PricingType]int),
@@ -1028,6 +1019,8 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 			pmd.PricingTypeCounts[cnode.PricingType] = 1
 		}
 
+		// newCnode builds upon cnode but populates/overrides certain fields.
+		// cnode was populated leveraging cloud provider public pricing APIs.
 		newCnode := *cnode
 		if newCnode.InstanceType == "" {
 			it, _ := util.GetInstanceType(n.Labels)
@@ -1070,48 +1063,24 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 
 		newCnode.RAMBytes = fmt.Sprintf("%f", ram)
 
-		// Azure does not seem to provide a GPU count in its pricing API. GKE supports attaching multiple GPUs
-		// So the k8s api will often report more accurate results for GPU count under status > capacity > nvidia.com/gpu than the cloud providers billing data
-		// not all providers are guaranteed to use this, so don't overwrite a Provider assignment if we can't find something under that capacity exists
-		gpuc := 0.0
-		q, ok := n.Status.Capacity["nvidia.com/gpu"]
-		_, hasReplicas := n.Labels["nvidia.com/gpu.replicas"]
-
-		if ok && !hasReplicas {
-			gpuCount := q.Value()
-			if gpuCount != 0 {
-				newCnode.GPU = fmt.Sprintf("%d", gpuCount)
-				newCnode.VGPU = newCnode.GPU
-				gpuc = float64(gpuCount)
-			}
-		} else if hasReplicas { // See https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/gpu-sharing.html
-			if q.Value() == 0 {
-				q = n.Status.Capacity["nvidia.com/gpu.shared"]
-			}
-			g, ok := n.Labels["nvidia.com/gpu.count"]
-			if ok {
-				newCnode.GPU = g
-			} else {
-				newCnode.GPU = fmt.Sprintf("%d", 0)
-			}
-			newCnode.VGPU = fmt.Sprintf("%d", q.Value())
+		gpuc, err := strconv.ParseFloat(newCnode.GPU, 64)
+		if err != nil {
+			gpuc = 0.0
+		}
 
-		} else if g, ok := n.Status.Capacity["k8s.amazonaws.com/vgpu"]; ok {
-			gpuCount := g.Value()
-			if gpuCount != 0 {
-				newCnode.GPU = fmt.Sprintf("%d", int(float64(gpuCount)/vgpuCoeff))
-				newCnode.VGPU = fmt.Sprintf("%d", gpuCount)
-				gpuc = float64(gpuCount) / vgpuCoeff
-			}
-		} else {
-			gpuc, err = strconv.ParseFloat(newCnode.GPU, 64)
-			if err != nil {
-				gpuc = 0.0
-			}
+		// The k8s API will often report more accurate results for GPU count
+		// than cloud provider public pricing APIs. If found, override the
+		// original value.
+		gpuOverride, vgpuOverride, err := getGPUCount(cm.Cache, n)
+		if err != nil {
+			log.Warnf("Unable to get GPUCount for node %s: %s", n.Name, err.Error())
 		}
-		if math.IsNaN(gpuc) {
-			log.Warnf("gpu count parsed as NaN. Setting to 0.")
-			gpuc = 0.0
+		if gpuOverride > 0 {
+			newCnode.GPU = fmt.Sprintf("%f", gpuOverride)
+			gpuc = gpuOverride
+		}
+		if vgpuOverride > 0 {
+			newCnode.VGPU = fmt.Sprintf("%f", vgpuOverride)
 		}
 
 		// Special case for SUSE rancher, since it won't behave with normal
@@ -2358,6 +2327,68 @@ func getStatefulSetsOfPod(pod v1.Pod) []string {
 	return []string{}
 }
 
+// getGPUCount reads the node's Status and Labels (via the k8s API) to identify
+// the number of GPUs and vGPUs are equipped on the node. If unable to identify
+// a GPU count, it will return -1.
+func getGPUCount(cache clustercache.ClusterCache, n *v1.Node) (float64, float64, error) {
+	g, hasGpu := n.Status.Capacity["nvidia.com/gpu"]
+	_, hasReplicas := n.Labels["nvidia.com/gpu.replicas"]
+
+	// Case 1: Standard NVIDIA GPU
+	if hasGpu && g.Value() != 0 && !hasReplicas {
+		return float64(g.Value()), float64(g.Value()), nil
+	}
+
+	// Case 2: NVIDIA GPU with GPU Feature Discovery (GFD) Pod enabled.
+	// Ref: https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/gpu-sharing.html#verifying-the-gpu-time-slicing-configuration
+	// Ref: https://github.com/NVIDIA/k8s-device-plugin/blob/d899752a424818428f744a946d32b132ea2c0cf1/internal/lm/resource_test.go#L44-L45
+	// Ref: https://github.com/NVIDIA/k8s-device-plugin/blob/d899752a424818428f744a946d32b132ea2c0cf1/internal/lm/resource_test.go#L103-L118
+	if hasReplicas {
+		resultGPU := 0.0
+		resultVGPU := 0.0
+
+		if c, ok := n.Labels["nvidia.com/gpu.count"]; ok {
+			var err error
+			resultGPU, err = strconv.ParseFloat(c, 64)
+			if err != nil {
+				return -1, -1, fmt.Errorf("could not parse label \"nvidia.com/gpu.count\": %v", err)
+			}
+		}
+
+		if s, ok := n.Status.Capacity["nvidia.com/gpu.shared"]; ok { // GFD configured `renameByDefault=true`
+			resultVGPU = float64(s.Value())
+		} else if g, ok := n.Status.Capacity["nvidia.com/gpu"]; ok { // GFD configured `renameByDefault=false`
+			resultVGPU = float64(g.Value())
+		} else {
+			resultVGPU = resultGPU
+		}
+
+		return resultGPU, resultVGPU, nil
+	}
+
+	// Case 3: AWS vGPU
+	if vgpu, ok := n.Status.Capacity["k8s.amazonaws.com/vgpu"]; ok {
+		vgpuCount, err := getAllocatableVGPUs(cache)
+		if err != nil {
+			return -1, -1, err
+		}
+
+		vgpuCoeff := 10.0
+		if vgpuCount > 0.0 {
+			vgpuCoeff = vgpuCount
+		}
+
+		if vgpu.Value() != 0 {
+			resultGPU := float64(vgpu.Value()) / vgpuCoeff
+			resultVGPU := float64(vgpu.Value())
+			return resultGPU, resultVGPU, nil
+		}
+	}
+
+	// No GPU found
+	return -1, -1, nil
+}
+
 func getAllocatableVGPUs(cache clustercache.ClusterCache) (float64, error) {
 	daemonsets := cache.GetAllDaemonSets()
 	vgpuCount := 0.0

pkg/costmodel/costmodel_test.go

@@ -4,8 +4,95 @@ import (
 	"testing"
 
 	"github.com/opencost/opencost/core/pkg/util"
+	"github.com/stretchr/testify/assert"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+func TestGetGPUCount(t *testing.T) {
+	tests := []struct {
+		name          string
+		node          *v1.Node
+		expectedGPU   float64
+		expectedVGPU  float64
+		expectedError bool
+	}{
+		{
+			name: "Standard NVIDIA GPU",
+			node: &v1.Node{
+				Status: v1.NodeStatus{
+					Capacity: v1.ResourceList{
+						"nvidia.com/gpu": resource.MustParse("2"),
+					},
+				},
+			},
+			expectedGPU:  2.0,
+			expectedVGPU: 2.0,
+		},
+		{
+			name: "NVIDIA GPU with GFD - renameByDefault=true",
+			node: &v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"nvidia.com/gpu.replicas": "4",
+						"nvidia.com/gpu.count":    "1",
+					},
+				},
+				Status: v1.NodeStatus{
+					Capacity: v1.ResourceList{
+						"nvidia.com/gpu.shared": resource.MustParse("4"),
+					},
+				},
+			},
+			expectedGPU:  1.0,
+			expectedVGPU: 4.0,
+		},
+		{
+			name: "NVIDIA GPU with GFD - renameByDefault=false",
+			node: &v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"nvidia.com/gpu.replicas": "4",
+						"nvidia.com/gpu.count":    "1",
+					},
+				},
+				Status: v1.NodeStatus{
+					Capacity: v1.ResourceList{
+						"nvidia.com/gpu": resource.MustParse("4"),
+					},
+				},
+			},
+			expectedGPU:  1.0,
+			expectedVGPU: 4.0,
+		},
+		{
+			name: "No GPU",
+			node: &v1.Node{
+				Status: v1.NodeStatus{
+					Capacity: v1.ResourceList{},
+				},
+			},
+			expectedGPU:  -1.0,
+			expectedVGPU: -1.0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gpu, vgpu, err := getGPUCount(nil, tt.node)
+
+			if tt.expectedError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.expectedGPU, gpu)
+				assert.Equal(t, tt.expectedVGPU, vgpu)
+			}
+		})
+	}
+}
+
 func Test_CostData_GetController_CronJob(t *testing.T) {
 	cases := []struct {
 		name string