Merge branch 'develop' into daniwk-add-securitycontext

.github/PULL_REQUEST_TEMPLATE.md

@@ -16,5 +16,5 @@
 ## Does this PR require changes to documentation?
 * 
 
-## Have you labeled this PR and its corresponding Issue as "next release" if it should be part of the next Opencost release? If not, why not?
+## Have you labeled this PR and its corresponding Issue as "next release" if it should be part of the next OpenCost release? If not, why not?
 * 

.github/workflows/pr.yaml

@@ -1,4 +1,4 @@
-name: Develop PR - build, test
+name: Develop PR - build test
 
 on:
   pull_request:
@@ -6,27 +6,74 @@ on:
       - develop
 
 jobs:
-  build:
-    strategy:
-      matrix:
-        include:
-          - component: Frontend
-            location: ui
-          - component: Backend
-            location: .
+  backend:
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: ./
+
+      -
+        name: Install just
+        uses: extractions/setup-just@v1
+
+      -
+        name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 'stable'
+
+      # Saves us from having to redownload all modules
+      - name: Go Mod cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+
+      -
+        name: Test
+        run: |
+          just test
 
+      -
+        name: Build
+        run: |
+          just build-local
+
+  frontend:
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
         with:
           path: ./
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+      -
+        name: Install just
+        uses: extractions/setup-just@v1
 
-      - name: Build ${{ matrix.component }}
-        uses: docker/build-push-action@v2
+      -
+        name: Install node
+        uses: actions/setup-node@v3
         with:
-          context: ${{ matrix.location }}/
-          file: ${{ matrix.location }}/Dockerfile
-          push: false
+          node-version: '18.3.0'
+
+      - name: Get npm cache directory
+        id: npm-cache-dir
+        shell: bash
+        run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT}
+
+      - uses: actions/cache@v3
+        id: npm-cache # use this to check for `cache-hit` ==> if: steps.npm-cache.outputs.cache-hit != 'true'
+        with:
+          path: ${{ steps.npm-cache-dir.outputs.dir }}
+          key: ${{ runner.os }}-node-${{ hashFiles('./ui/**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      -
+        name: Build
+        working-directory: ./ui
+        run: |
+          just build-local

.gitignore

@@ -7,4 +7,6 @@ ui/.cache
 ui/dist
 ui/node_modules/
 cmd/costmodel/costmodel
+cmd/costmodel/costmodel-amd64
+cmd/costmodel/costmodel-arm64
 pkg/cloud/azureorphan_test.go

CONTRIBUTING.md

@@ -24,18 +24,31 @@ This repository's contribution workflow follows a typical open-source model:
 
 ## Building OpenCost
 
-Follow these steps to build the OpenCost cost-model and UI from source and deploy:
+Follow these steps to build the OpenCost cost-model and UI from source and
+deploy. The provided build tooling is natively multi-architecture (built images
+will run on both AMD64 and ARM64 clusters).
 
-1. `docker build --rm -f "Dockerfile" -t <repo>/opencost:<tag> .`
+Dependencies:
+1. Docker (with `buildx`)
+2. [just](https://github.com/casey/just) (if you don't want to install Just, read the `justfile` and run the commands manually)
+3. Multi-arch `buildx` builders set up via https://github.com/tonistiigi/binfmt
+4. `npm` (if you want to build the UI)
+
+### Build the backend
+
+1. `just build "<repo>/opencost:<tag>"`
 2. Edit the [pulled image](https://github.com/opencost/opencost/blob/develop/kubernetes/opencost.yaml#L145) in the `kubernetes/opencost.yaml` to `<repo>/opencost:<tag>`
 3. Set [this environment variable](https://github.com/opencost/opencost/blob/develop/kubernetes/opencost.yaml#L155) to the address of your Prometheus server
-4. `cd ui`
-5. `docker build --rm -f "Dockerfile" -t <repo>/opencost-ui:<tag> .`
-6. `cd ..`
-7. Edit the [pulled image](https://github.com/opencost/opencost/blob/develop/kubernetes/opencost.yaml#L162) in the `kubernetes/opencost.yaml` to `<repo>/opencost-ui:<tag>`
-8. `kubectl create namespace opencost`
-9. `kubectl apply -f kubernetes/opencost --namespace opencost`
-10. `kubectl -n opencost port-forward service/opencost 9090 9003`
+
+### Build the frontend
+1. `cd ui && just build-ui "<repo>/opencost-ui:<tag>"`
+2. Edit the [pulled image](https://github.com/opencost/opencost/blob/develop/kubernetes/opencost.yaml#L162) in the `kubernetes/opencost.yaml` to `<repo>/opencost-ui:<tag>`
+
+### Deploy to a cluster
+
+1. `kubectl create namespace opencost`
+2. `kubectl apply -f kubernetes/opencost --namespace opencost`
+3. `kubectl -n opencost port-forward service/opencost 9090 9003`
 
 To test, build the OpenCost containers and then push them to a Kubernetes cluster with a running Prometheus.
 

Dockerfile.cross

@@ -0,0 +1,18 @@
+FROM alpine:latest
+
+# The prebuilt binary path. This Dockerfile assumes the binary will be built
+# outside of Docker.
+ARG binarypath
+
+RUN apk add --update --no-cache ca-certificates
+
+ADD --chmod=644 ./configs/default.json /models/default.json
+ADD --chmod=644 ./configs/azure.json /models/azure.json
+ADD --chmod=644 ./configs/aws.json /models/aws.json
+ADD --chmod=644 ./configs/gcp.json /models/gcp.json
+ADD --chmod=644 ./configs/alibaba.json /models/alibaba.json
+
+COPY ${binarypath} /go/bin/app
+
+USER 1001
+ENTRYPOINT ["/go/bin/app"]

GOVERNANCE.md

@@ -0,0 +1,141 @@
+# OpenCost Governance
+
+> **Note**
+> OpenCost community governance is a work in progress as we expand beyond the initial Kubecost stewardship. Expanding the committers and maintainers to additional organizations will allow the project to become more self-managing.
+
+This document attempts to clarify how the [OpenCost](https://github.com/opencost/) projects are maintained. Anyone interested in improving the project may join the community, contribute to the project, and participate in shaping future releases. This document attempts to outline the general participation structure and set expectations within the project community.
+
+## Code of Conduct
+
+OpenCost community members are expected to adhere to our published [Code of Conduct](CODE_OF_CONDUCT.md).
+
+## Roles and Responsibilities
+
+There are 4 levels of membership in the OpenCost community.
+
+### Contributor
+
+Contributors are community members who contribute in concrete ways to the project. Anyone can contribute to the project and become a contributor, regardless of their skillset. There is no expectation of commitment to the project, no specific skill requirements, and no selection process. There are many ways to contribute to the project, which may be one or more of the following (but not limited to):
+
+- Participate in community discussions in the `#opencost` channel in [CNCF Slack](https://slack.cncf.io/) or at [community meetings](https://bit.ly/opencost-meeting)
+- Report, comment on, and sometimes resolve Issues
+- Occasionally submit PRs
+- Try out new releases
+- Contribute to the documentation
+- Improve the OpenCost website
+- Promote the project in public
+- Help other users
+
+For first-time contributors, it’s recommended to start by going through [Contributing to OpenCost](CONTRIBUTING.md) and joining our community Slack channel.
+
+### Member
+
+[Members](https://github.com/orgs/opencost/people) are continuously active contributors in the community. There are multiple ways to stay "active" and engaged with us - contributing to codes, raising issues, writing tutorials and case studies, and even answering questions.
+
+To become an OpenCost member, you are expected to:
+
+- Make multiple contributions, which may be one or more of the following (but not limited to):
+    - Authored PRs on GitHub.
+    - Filed, or commented on Issues on GitHub.
+    - Join community discussions (e.g. community meetings, Slack).
+- Sponsored by at least 1 OpenCost [maintainer or committer](MAINTAINERS.md)
+
+Contributors that meet the above requirements will then be invited to the GitHub organization "OpenCost" by a sponsor, and there would be an announcement published in the slack channel [(#opencost)](https://slack.cncf.io/).
+
+Members are expected to respond to issues and PRs assigned to them, and be the owners of the code or docs they have contributed. Members that have not contributed to the project or community for over 6 months may lose their membership.
+
+### Committer
+
+Committers are active community members who have shown that they are committed to the success of the project through ongoing engagement with the community. Committership allows contributors to more easily carry on with their project-related activities by giving them direct access to the project’s resources.
+
+Committers are granted `Triage` permissions for the OpenCost repository.
+
+Typically, a potential committer needs to show that they have a sufficient understanding of the project, its objectives, and its strategy. To become a committer, you are expected to:
+
+- Be an OpenCost member.
+- Express interest to the existing maintainers that you are interested in becoming a committer.
+- Have contributed 5 or more substantial PRs.
+- Have an above-average understanding of the project codebase, its goals, and directions.
+
+Members that meet the above requirements will be nominated by an existing maintainer to become a committer. It is recommended to describe the reasons for the nomination and the contribution of the nominee in the PR. The existing maintainers will confer and decide whether to grant committer status or not.
+
+Committers are expected to review issues and PRs. While committership indicates a valued member of the community who has demonstrated a healthy respect for the project’s aims and objectives, their work continues to be reviewed by the community before acceptance in an official release.
+
+### Maintainer
+
+Maintainers are first and foremost committers that have shown they are committed to the long term success of a project. They are the planners and designers of the OpenCost project. Maintainership is about building trust with the current maintainers of the project and being a person that they can depend on to make decisions in the best interest of the project in a consistent manner.
+
+Maintainers are granted `Write` permissions for the OpenCost repository.
+
+Committers wanting to become maintainers are expected to:
+
+- Enable adoptions or ecosystems.
+- Collaborate well.
+- Demonstrate a deep and comprehensive understanding of OpenCost's architecture, technical goals, and directions.
+- Actively engage with major OpenCost feature proposals and implementations.
+
+A new maintainer must be nominated by an existing maintainer. The nominating maintainer will create a PR to update the [Maintainers List](MAINTAINERS.md). It is recommended to describe the reasons for the nomination and the contribution of the nominee in the PR. Upon consensus of incumbent maintainers, the PR will be approved and the new maintainer becomes active.
+
+## Policies and Procedures
+
+### Community Meetings
+
+The [OpenCost Community Meeting](https://bit.ly/opencost-meeting) is every 2 weeks at 1pm Pacific. Community Members are encouraged to attend and discuss development, events, and any other OpenCost community items of interest.
+
+### Issue/PR Timelines
+
+Best efforts will be given to respond to new Issues and PRs within 24 hours on weekdays.
+
+### Approving PRs
+
+PRs may be merged only after receiving at least one approvals from committers or maintainers. However, maintainers can sidestep this rule under justifiable circumstances. For example:
+
+- If a CI tool is broken, may override the tool to still submit the change.
+- Minor typos or fixes for broken tests.
+- The change was approved through other means than the standard process.
+
+### Decision Making Process
+
+Ideally, all project decisions are resolved by consensus via a PR or GitHub issue. Any of the day-to-day project maintenance can be done by a [lazy consensus model](https://communitymgt.fandom.com/wiki/Lazy_consensus).
+
+Community or project level decisions such as RFC submission, creating a new project, maintainer promotion, and major updates on GOVERNANCE must be brought to broader awareness of the community via community meetings, GitHub discussions, and the Slack channel. A supermajority (2/3) approval from Maintainers is required for such approvals.
+
+In general, we prefer that technical issues and maintainer membership are amicably worked out between the persons involved. If a dispute cannot be decided independently, the maintainers can be called in to resolve the issue by voting. For voting, a specific statement of what is being voted on should be added to the relevant GitHub issue or PR, and a link to that issue or PR added to the maintainers meeting agenda document. Maintainers should indicate their yes/no vote on that issue or PR, and after a suitable period of time, the votes will be tallied and the outcome noted.
+
+### Conflict resolution and voting
+
+In general, we prefer that technical issues and membership are amicably worked out between the persons involved. If a dispute cannot be decided independently, the sponsors and core maintainers can be called in to decide an issue. If the sponsors and maintainers themselves cannot decide an issue, the issue will be resolved by voting.
+
+In all cases in this document where voting is mentioned, the voting process is a simple majority in which each sponsor receives two votes and each core maintainer receives one vote. If such a majority is reached, the vote is said to have _passed_.
+
+### Proposal process
+
+> **Note**
+> We intend to use a Request for Comments (RFC) process for any substantial changes to OpenCost, but this has yet to be developed.
+
+### Inactivity
+
+It is important for contributors to be and stay active to set an example and show commitment to the project. Inactivity is harmful to the project as it may lead to unexpected delays, contributor attrition, and a loss of trust in the project.
+
+Inactivity is measured by periods of no contributions without explanation, for longer than:
+
+- Maintainer: 3 months
+- Committer: 6 months
+- Member: 12 months
+
+Consequences of being inactive include:
+
+- Involuntary removal or demotion
+- Being asked to move to Emeritus status
+
+### Involuntary Removal or Demotion
+
+Involuntary removal/demotion of a contributor happens when responsibilities and requirements aren't being met. This may include repeated patterns of inactivity, extended period of inactivity, a period of failing to meet the requirements of your role, and/or a violation of the Code of Conduct. This process is important because it protects the community and its deliverables while also opens up opportunities for new contributors to step in.
+
+Involuntary removal or demotion is handled through a vote by a majority of the current Maintainers.
+
+### Stepping Down/Emeritus Process
+
+If and when Contributors' commitment levels change, Contributors can consider stepping down (moving down the Contributor ladder) vs moving to emeritus status (completely stepping away from the project).
+
+Contact the Maintainers about changing to Emeritus status, or reducing your contributor level. An Emeritus list will be added to the [MAINTAINERS.md](MAINTAINERS.md)

MAINTAINERS.md

@@ -1,8 +1,11 @@
-# OpenCost Maintainers
+# OpenCost Committers and Maintainers
 
-Official list of OpenCost Maintainers.
+Official list of OpenCost Committers and Maintainers. This is managed as documented in the [GOVERNANCE.md](GOVERNANCE.md).
 
-Please keep the below list sorted in ascending order.
+## Committers
+
+| Committer | GitHub ID | Affiliation | Email |
+| --------------- | --------- | ----------- | ----------- |
 
 ## Maintainers
 
@@ -10,8 +13,8 @@ Please keep the below list sorted in ascending order.
 | --------------- | --------- | ----------- | ----------- |
 | Ajay Tripathy | @AjayTripathy | Kubecost | <Ajay@kubecost.com> |
 | Matt Bolt | @​mbolt35 | Kubecost | <matt@kubecost.com> |
+| Matt Ray | @mattray | Kubecost | <mattray@kubecost.com> |
 | Michael Dresser | @michaelmdresser | Kubecost | <michael@kubecost.com> |
 | Niko Kovacevic | @nikovacevic | Kubecost | <niko@kubecost.com> |
 | Sean Holcomb | @Sean-Holcomb | Kubecost | <Sean@kubecost.com> |
 | Thomas Evans | @teevans | Kubecost | <thomas@kubecost.com> |
-| Matt Ray | @mattray | Kubecost | <mattray@kubecost.com> |

config/invalid.json

@@ -0,0 +1 @@
+{"provider":"base","description":"Default prices based on GCP us-central1","CPU":"0.031611","spotCPU":"0.006655","RAM":"0.004237","spotRAM":"0.000892","GPU":"0.95","spotGPU":"0.308","storage":"0.00005479452","zoneNetworkEgress":"0.01","regionNetworkEgress":"0.01","internetNetworkEgress":"0.12","firstFiveForwardingRulesCost":"","additionalForwardingRuleCost":"","LBIngressDataCost":"","athenaBucketName":"","athenaRegion":"","athenaDatabase":"","athenaTable":"","athenaWorkgroup":"","masterPayerARN":"","customPricesEnabled":"false","defaultIdle":"","azureSubscriptionID":"","azureClientID":"","azureClientSecret":"","azureTenantID":"","azureBillingRegion":"","azureOfferDurableID":"","azureStorageSubscriptionID":"","azureStorageAccount":"","azureStorageAccessKey":"","azureStorageContainer":"","azureContainerPath":"","azureCloud":"","currencyCode":"","discount":"","negotiatedDiscount":"","sharedOverhead":"","clusterName":"","sharedNamespaces":"","sharedLabelNames":"","sharedLabelValues":"","shareTenancyCosts":"true","readOnly":"","editorAccess":"","kubecostToken":"","googleAnalyticsTag":"","excludeProviderID":""}

configs/aws.json

@@ -12,11 +12,11 @@
     "internetNetworkEgress": "0.143",
     "spotLabel": "kops.k8s.io/instancegroup",
     "spotLabelValue": "spotinstance-nodes",
-    "awsServiceKeyName": "AKIXXX",
+    "awsServiceKeyName": "",
     "awsServiceKeySecret": "",
     "awsSpotDataRegion":"us-east-2",
     "awsSpotDataBucket": "x",
-    "awsSpotDataPrefix": "spotdata",
+    "awsSpotDataPrefix": "",
     "athenaBucketName": "s3://x",
     "athenaRegion": "us-east-1",
     "athenaDatabase": "",

configs/pricing_schema_pv_storageclass.csv

@@ -0,0 +1,2 @@
+EndTimestamp,InstanceID,Region,AssetClass,InstanceIDField,InstanceType,MarketPriceHourly,Version
+2019-04-17 23:34:22 UTC,storageClass0,,pv,spec.storageClassName,,0.1338,

configs/pricing_schema_special_char.csv

@@ -0,0 +1,2 @@
+EndTimestamp,InstanceID,Region,AssetClass,InstanceIDField,InstanceType,MarketPriceHourly,Version
+2019-04-17 23:34:22 UTC,gke-standard-cluster-1-pool-1-91dc432d-cg69,,node,metadata.labels.<http://metadata.label.servers.com/label|metadata.label.servers.com/label>,,0.1337,

go.mod

@@ -97,7 +97,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect

go.sum

@@ -56,14 +56,10 @@ github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVt
 github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k=
 github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw=
 github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.1-0.20230323231529-14c481f239ec h1:S83Dzhd3VLyvN2bgFI7/Lgk1etamk3Pk8QQhn3iXt4s=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.1-0.20230323231529-14c481f239ec/go.mod h1:IoxiGSzhL1QHFXa/mlAXCD+sUaP0rxg//yn2w/JH7wg=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.5.0 h1:xGLAFFd9D3iLGxYiUGPdITSzsFmU1K8VtfuUHWAoN7M=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.5.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 h1:uqM+VoHjVH6zdlkLF2b6O0ZANcHoj3rO0PoQ3jglUJA=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2/go.mod h1:twTKAa1E6hLmSDjLhaCkbTMQKc7p/rNLU40rLxGEOCI=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 h1:leh5DwKv6Ihwi+h60uHtn6UWAxBbZ0q8DwQVMzf61zw=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY=
@@ -224,8 +220,8 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
-github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw=
-github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE=
+github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=

justfile

@@ -0,0 +1,63 @@
+commonenv := "CGO_ENABLED=0"
+
+version := "dev"
+commit := `git rev-parse --short HEAD`
+
+default:
+    just --list
+
+# Run unit tests
+test:
+    {{commonenv}} go test ./...
+
+# Compile a local binary
+build-local:
+    cd ./cmd/costmodel && \
+        {{commonenv}} go build \
+        -ldflags \
+          "-X github.com/opencost/opencost/pkg/version.Version={{version}} \
+           -X github.com/opencost/opencost/pkg/version.GitCommit={{commit}}" \
+        -o ./costmodel
+
+# Build multiarch binaries
+build-binary VERSION=version:
+    cd ./cmd/costmodel && \
+        {{commonenv}} GOOS=linux GOARCH=amd64 go build \
+        -ldflags \
+          "-X github.com/opencost/opencost/pkg/version.Version={{VERSION}} \
+           -X github.com/opencost/opencost/pkg/version.GitCommit={{commit}}" \
+        -o ./costmodel-amd64
+
+    cd ./cmd/costmodel && \
+        {{commonenv}} GOOS=linux GOARCH=arm64 go build \
+        -ldflags \
+          "-X github.com/opencost/opencost/pkg/version.Version={{VERSION}} \
+           -X github.com/opencost/opencost/pkg/version.GitCommit={{commit}}" \
+        -o ./costmodel-arm64
+
+# Build and push a multi-arch Docker image
+build IMAGETAG VERSION=version: test (build-binary VERSION)
+    docker buildx build \
+        --rm \
+        --platform "linux/amd64" \
+        -f 'Dockerfile.cross' \
+        --build-arg binarypath=./cmd/costmodel/costmodel-amd64 \
+        --provenance=false \
+        -t {{IMAGETAG}}-amd64 \
+        --push \
+        .
+
+    docker buildx build \
+        --rm \
+        --platform "linux/arm64" \
+        -f 'Dockerfile.cross' \
+        --build-arg binarypath=./cmd/costmodel/costmodel-arm64 \
+        --provenance=false \
+        -t {{IMAGETAG}}-arm64 \
+        --push \
+        .
+
+    manifest-tool push from-args \
+        --platforms "linux/amd64,linux/arm64" \
+        --template {{IMAGETAG}}-ARCH \
+        --target {{IMAGETAG}}

pkg/cloud/alibaba/authorizer.go

@@ -0,0 +1,87 @@
+package alibaba
+
+import (
+	"fmt"
+
+	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth"
+	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials"
+	"github.com/opencost/opencost/pkg/cloud/config"
+	"github.com/opencost/opencost/pkg/util/json"
+)
+
+const AccessKeyAuthorizerType = "AlibabaAccessKey"
+
+// Authorizer provide *bssopenapi.Client for Alibaba cloud BOS for Billing related SDK calls
+type Authorizer interface {
+	config.Authorizer
+	GetCredentials() (auth.Credential, error)
+}
+
+// SelectAuthorizerByType is an implementation of AuthorizerSelectorFn and acts as a register for Authorizer types
+func SelectAuthorizerByType(typeStr string) (Authorizer, error) {
+	switch typeStr {
+	case AccessKeyAuthorizerType:
+		return &AccessKey{}, nil
+	default:
+		return nil, fmt.Errorf("alibaba: provider authorizer type '%s' is not valid", typeStr)
+	}
+}
+
+// AccessKey holds Alibaba credentials parsing from the service-key.json file.
+type AccessKey struct {
+	AccessKeyID     string `json:"accessKeyID"`
+	AccessKeySecret string `json:"accessKeySecret"`
+}
+
+// MarshalJSON custom json marshalling functions, sets properties as tagged in struct and sets the authorizer type property
+func (ak *AccessKey) MarshalJSON() ([]byte, error) {
+	fmap := make(map[string]any, 3)
+	fmap[config.AuthorizerTypeProperty] = AccessKeyAuthorizerType
+	fmap["accessKeyID"] = ak.AccessKeyID
+	fmap["accessKeySecret"] = ak.AccessKeySecret
+	return json.Marshal(fmap)
+}
+
+func (ak *AccessKey) Validate() error {
+	if ak.AccessKeyID == "" {
+		return fmt.Errorf("AccessKey: missing Access key ID")
+	}
+	if ak.AccessKeySecret == "" {
+		return fmt.Errorf("AccessKey: missing Access Key secret")
+	}
+	return nil
+}
+
+func (ak *AccessKey) Equals(config config.Config) bool {
+	if config == nil {
+		return false
+	}
+	thatConfig, ok := config.(*AccessKey)
+	if !ok {
+		return false
+	}
+
+	if ak.AccessKeyID != thatConfig.AccessKeyID {
+		return false
+	}
+	if ak.AccessKeySecret != thatConfig.AccessKeySecret {
+		return false
+	}
+	return true
+}
+
+func (ak *AccessKey) Sanitize() config.Config {
+	return &AccessKey{
+		AccessKeyID:     ak.AccessKeyID,
+		AccessKeySecret: config.Redacted,
+	}
+}
+
+// GetCredentials creates a credentials object to authorize the use of service sdk calls
+func (ak *AccessKey) GetCredentials() (auth.Credential, error) {
+	err := ak.Validate()
+	if err != nil {
+		return nil, err
+	}
+	return &credentials.AccessKeyCredential{AccessKeyId: ak.AccessKeyID, AccessKeySecret: ak.AccessKeySecret}, nil
+}

pkg/cloud/alibaba/boaconfiguration.go

@@ -0,0 +1,130 @@
+package alibaba
+
+import (
+	"fmt"
+
+	"github.com/opencost/opencost/pkg/cloud/config"
+	"github.com/opencost/opencost/pkg/util/json"
+)
+
+// BOAConfiguration is the BSS open API configuration for Alibaba's Billing information
+type BOAConfiguration struct {
+	Account    string     `json:"account"`
+	Region     string     `json:"region"`
+	Authorizer Authorizer `json:"authorizer"`
+}
+
+func (bc *BOAConfiguration) Validate() error {
+	// Validate Authorizer
+	if bc.Authorizer == nil {
+		return fmt.Errorf("BOAConfiguration: missing authorizer")
+	}
+
+	err := bc.Authorizer.Validate()
+	if err != nil {
+		return err
+	}
+
+	// Validate base properties
+	if bc.Region == "" {
+		return fmt.Errorf("BOAConfiguration: missing region")
+	}
+
+	if bc.Account == "" {
+		return fmt.Errorf("BOAConfiguration: missing account")
+	}
+	return nil
+}
+
+func (bc *BOAConfiguration) Equals(config config.Config) bool {
+	if config == nil {
+		return false
+	}
+	thatConfig, ok := config.(*BOAConfiguration)
+	if !ok {
+		return false
+	}
+
+	if bc.Authorizer != nil {
+		if !bc.Authorizer.Equals(thatConfig.Authorizer) {
+			return false
+		}
+	} else {
+		if thatConfig.Authorizer != nil {
+			return false
+		}
+	}
+
+	if bc.Account != thatConfig.Account {
+		return false
+	}
+
+	if bc.Region != thatConfig.Region {
+		return false
+	}
+	return true
+}
+
+func (bc *BOAConfiguration) Sanitize() config.Config {
+	return &BOAConfiguration{
+		Account:    bc.Account,
+		Region:     bc.Region,
+		Authorizer: bc.Authorizer.Sanitize().(Authorizer),
+	}
+}
+
+func (bc *BOAConfiguration) Key() string {
+	return fmt.Sprintf("%s/%s", bc.Account, bc.Region)
+}
+
+func (bc *BOAConfiguration) UnmarshalJSON(b []byte) error {
+	var f interface{}
+	err := json.Unmarshal(b, &f)
+	if err != nil {
+		return err
+	}
+
+	fmap := f.(map[string]interface{})
+
+	account, err := config.GetInterfaceValue[string](fmap, "account")
+	if err != nil {
+		return fmt.Errorf("BOAConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	bc.Account = account
+
+	region, err := config.GetInterfaceValue[string](fmap, "region")
+	if err != nil {
+		return fmt.Errorf("BOAConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	bc.Region = region
+
+	authAny, ok := fmap["authorizer"]
+	if !ok {
+		return fmt.Errorf("BOAConfiguration: UnmarshalJSON: missing authorizer")
+	}
+	authorizer, err := config.AuthorizerFromInterface(authAny, SelectAuthorizerByType)
+	if err != nil {
+		return fmt.Errorf("BOAConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	bc.Authorizer = authorizer
+
+	return nil
+}
+
+func ConvertAlibabaInfoToConfig(acc AlibabaInfo) config.KeyedConfig {
+	if acc.IsEmpty() {
+		return nil
+	}
+	var configurer Authorizer
+
+	configurer = &AccessKey{
+		AccessKeyID:     acc.AlibabaServiceKeyName,
+		AccessKeySecret: acc.AlibabaServiceKeySecret,
+	}
+
+	return &BOAConfiguration{
+		Account:    acc.AlibabaAccountID,
+		Region:     acc.AlibabaClusterRegion,
+		Authorizer: configurer,
+	}
+}

pkg/cloud/alibaba/boaconfiguration_test.go

@@ -0,0 +1,289 @@
+package alibaba
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/opencost/opencost/pkg/cloud/config"
+	"github.com/opencost/opencost/pkg/log"
+	"github.com/opencost/opencost/pkg/util/json"
+)
+
+func TestBoaConfiguration_Validate(t *testing.T) {
+	testCases := map[string]struct {
+		config   BOAConfiguration
+		expected error
+	}{
+		"valid config Azure AccessKey": {
+			config: BOAConfiguration{
+				Account: "Account Number",
+				Region:  "Region",
+				Authorizer: &AccessKey{
+					AccessKeyID:     "accessKeyID",
+					AccessKeySecret: "accessKeySecret",
+				},
+			},
+			expected: nil,
+		},
+		"access key invalid": {
+			config: BOAConfiguration{
+				Account: "Account Number",
+				Region:  "Region",
+				Authorizer: &AccessKey{
+					AccessKeySecret: "accessKeySecret",
+				},
+			},
+			expected: fmt.Errorf("AccessKey: missing Access key ID"),
+		},
+		"access secret invalid": {
+			config: BOAConfiguration{
+				Account: "Account Number",
+				Region:  "Region",
+				Authorizer: &AccessKey{
+					AccessKeyID: "accessKeyId",
+				},
+			},
+			expected: fmt.Errorf("AccessKey: missing Access Key secret"),
+		},
+		"missing authorizer": {
+			config: BOAConfiguration{
+				Account:    "Account Number",
+				Region:     "Region",
+				Authorizer: nil,
+			},
+			expected: fmt.Errorf("BOAConfiguration: missing authorizer"),
+		},
+		"missing Account": {
+			config: BOAConfiguration{
+				Account: "",
+				Region:  "Region",
+				Authorizer: &AccessKey{
+					AccessKeyID:     "accessKeyID",
+					AccessKeySecret: "accessKeySecret",
+				},
+			},
+			expected: fmt.Errorf("BOAConfiguration: missing account"),
+		},
+		"missing Region": {
+			config: BOAConfiguration{
+				Account: "Account",
+				Authorizer: &AccessKey{
+					AccessKeyID:     "accessKeyID",
+					AccessKeySecret: "accessKeySecret",
+				},
+			},
+			expected: fmt.Errorf("BOAConfiguration: missing region"),
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual := testCase.config.Validate()
+			actualString := "nil"
+			if actual != nil {
+				actualString = actual.Error()
+			}
+			expectedString := "nil"
+			if testCase.expected != nil {
+				expectedString = testCase.expected.Error()
+			}
+			if actualString != expectedString {
+				t.Errorf("errors do not match: Actual: '%s', Expected: '%s", actualString, expectedString)
+			}
+		})
+	}
+}
+
+func TestBOAConfiguration_Equals(t *testing.T) {
+	testCases := map[string]struct {
+		left     BOAConfiguration
+		right    config.Config
+		expected bool
+	}{
+		"matching config": {
+			left: BOAConfiguration{
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					AccessKeyID:     "id",
+					AccessKeySecret: "secret",
+				},
+			},
+			right: &BOAConfiguration{
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					AccessKeyID:     "id",
+					AccessKeySecret: "secret",
+				},
+			},
+			expected: true,
+		},
+		"different Authorizer": {
+			left: BOAConfiguration{
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					AccessKeyID:     "id",
+					AccessKeySecret: "secret",
+				},
+			},
+			right: &BOAConfiguration{
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					AccessKeyID:     "id2",
+					AccessKeySecret: "secret2",
+				},
+			},
+			expected: false,
+		},
+		"missing both Authorizer": {
+			left: BOAConfiguration{
+				Region:     "region",
+				Account:    "account",
+				Authorizer: nil,
+			},
+			right: &BOAConfiguration{
+				Region:     "region",
+				Account:    "account",
+				Authorizer: nil,
+			},
+			expected: true,
+		},
+		"missing left Authorizer": {
+			left: BOAConfiguration{
+				Region:     "region",
+				Account:    "account",
+				Authorizer: nil,
+			},
+			right: &BOAConfiguration{
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					AccessKeyID:     "id",
+					AccessKeySecret: "secret",
+				},
+			},
+			expected: false,
+		},
+		"missing right Authorizer": {
+			left: BOAConfiguration{
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					AccessKeyID:     "id",
+					AccessKeySecret: "secret",
+				},
+			},
+			right: &BOAConfiguration{
+				Region:     "region",
+				Account:    "account",
+				Authorizer: nil,
+			},
+			expected: false,
+		},
+		"different region": {
+			left: BOAConfiguration{
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					AccessKeyID:     "id",
+					AccessKeySecret: "secret",
+				},
+			},
+			right: &BOAConfiguration{
+				Region:  "region2",
+				Account: "account",
+				Authorizer: &AccessKey{
+					AccessKeyID:     "id",
+					AccessKeySecret: "secret",
+				},
+			},
+			expected: false,
+		},
+		"different account": {
+			left: BOAConfiguration{
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					AccessKeyID:     "id",
+					AccessKeySecret: "secret",
+				},
+			},
+			right: &BOAConfiguration{
+				Region:  "region",
+				Account: "account2",
+				Authorizer: &AccessKey{
+					AccessKeyID:     "id",
+					AccessKeySecret: "secret",
+				},
+			},
+			expected: false,
+		},
+		"different config": {
+			left: BOAConfiguration{
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					AccessKeyID:     "id",
+					AccessKeySecret: "secret",
+				},
+			},
+			right: &AccessKey{
+				AccessKeyID:     "id",
+				AccessKeySecret: "secret",
+			},
+			expected: false,
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual := testCase.left.Equals(testCase.right)
+			if actual != testCase.expected {
+				t.Errorf("incorrect result: Actual: '%t', Expected: '%t", actual, testCase.expected)
+			}
+		})
+	}
+}
+
+func TestBOAConfiguration_JSON(t *testing.T) {
+	testCases := map[string]struct {
+		config BOAConfiguration
+	}{
+		"Empty Config": {
+			config: BOAConfiguration{},
+		},
+		"AccessKey": {
+			config: BOAConfiguration{
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					AccessKeyID:     "id",
+					AccessKeySecret: "secret",
+				},
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			// test JSON Marshalling
+			configJSON, err := json.Marshal(testCase.config)
+			if err != nil {
+				t.Errorf("failed to marshal configuration: %s", err.Error())
+			}
+			log.Info(string(configJSON))
+			unmarshalledConfig := &BOAConfiguration{}
+			err = json.Unmarshal(configJSON, unmarshalledConfig)
+			if err != nil {
+				t.Errorf("failed to unmarshal configuration: %s", err.Error())
+			}
+
+			if !testCase.config.Equals(unmarshalledConfig) {
+				t.Error("config does not equal unmarshalled config")
+			}
+		})
+	}
+}

pkg/cloud/alibaba/boaquerier.go

@@ -0,0 +1,127 @@
+package alibaba
+
+import (
+	"fmt"
+	"strings"
+
+	cloudconfig "github.com/opencost/opencost/pkg/cloud/config"
+
+	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
+	"github.com/aliyun/alibaba-cloud-sdk-go/services/bssopenapi"
+	"github.com/opencost/opencost/pkg/kubecost"
+	"github.com/opencost/opencost/pkg/log"
+)
+
+const (
+	boaIsNode    = "i-"    // isNode if prefix of instance_id is i-
+	boaIsDisk    = "d-"    // isDisk if prefix is disk is d-
+	boaIsNetwork = "piece" //usage unit of network resource in Alibaba is Piece
+)
+
+type BoaQuerier struct {
+	BOAConfiguration
+}
+
+func (bq *BoaQuerier) Equals(config cloudconfig.Config) bool {
+	thatConfig, ok := config.(*BoaQuerier)
+	if !ok {
+		return false
+	}
+
+	return bq.BOAConfiguration.Equals(&thatConfig.BOAConfiguration)
+}
+
+// QueryInstanceBill performs the request to the BSS client and get the response for the current page number
+func (bq *BoaQuerier) QueryInstanceBill(client *bssopenapi.Client, isBillingItem bool, invocationScheme, granularity, billingCycle, billingDate string, pageNum int) (*bssopenapi.QueryInstanceBillResponse, error) {
+	log.Debugf("QueryInstanceBill: query for BSS Open API for billing date: %s with pageNum: %d ", billingDate, pageNum)
+	request := bssopenapi.CreateQueryInstanceBillRequest()
+	request.Scheme = invocationScheme
+	request.BillingCycle = billingCycle
+	request.IsBillingItem = requests.NewBoolean(true)
+	request.Granularity = granularity
+	request.BillingDate = billingDate
+	request.PageNum = requests.NewInteger(pageNum)
+	response, err := client.QueryInstanceBill(request)
+	if err != nil {
+		return nil, fmt.Errorf("QueryInstanceBill: Failed to hit the BSS Open API with error for page num %d: %v", pageNum, err)
+	}
+	log.Debugf("QueryInstanceBill: Total Number of total items for billing Date: %s pageNum: %d is %d", billingDate, pageNum, response.Data.TotalCount)
+	return response, nil
+}
+
+// QueryBoaPaginated Calls the API in a paginated fashion. There's no paramter in API that can distinguish if it hasMorePages
+// hence the logic of processedItem <= TotalItem.
+func (bq *BoaQuerier) QueryBoaPaginated(client *bssopenapi.Client, isBillingItem bool, invocationScheme, granularity, billingCycle, billingDate string, fn func(*bssopenapi.QueryInstanceBillResponse) bool) error {
+	pageNum := 1
+	processedItem := 0 // setting default here to hit the API for the first time
+	totalItem := 1
+	for processedItem < totalItem {
+		log.Debugf("QueryBoaPaginated: query for BSS Open API for billing date: %s with pageNum: %d", billingDate, pageNum)
+		response, err := bq.QueryInstanceBill(client, isBillingItem, invocationScheme, granularity, billingCycle, billingDate, pageNum)
+		if err != nil {
+			return fmt.Errorf("QueryBoaPaginated for billing cycle : %s, billing date: %s, page num %d: %v", billingCycle, billingDate, pageNum, err)
+		}
+		fn(response)
+		totalItem = response.Data.TotalCount
+		processedItem += response.Data.PageSize
+		pageNum += 1
+	}
+	return nil
+}
+
+// GetBoaQueryInstanceBillFunc gives the item to the handler function in boaIntegration.go to process
+// computeItem, topNItem and aggregatedItem
+func GetBoaQueryInstanceBillFunc(fn func(bssopenapi.Item) error, billingDate string) func(output *bssopenapi.QueryInstanceBillResponse) bool {
+	processBOAItems := func(output *bssopenapi.QueryInstanceBillResponse) bool {
+		// This could be connection error were unable to fetch response output from Client
+		if output == nil {
+			log.Errorf("BoaQuerier: No Response from the ALibaba BSS Open API client for billing Date: %s", billingDate)
+			return false
+		}
+
+		// These infer that the rest call was successful but the Cloud Usage resource for those days were 0
+		if output.Data.TotalCount == 0 {
+			log.Warnf("BoaQuerier: Total Item Count is 0 for billing Date: %s ", billingDate)
+			return false
+		}
+
+		for _, item := range output.Data.Items.Item {
+			fn(item)
+		}
+		return true
+	}
+	return processBOAItems
+}
+
+// SelectAlibabaCategory processes the Alibaba service to associated Kubecost category
+func SelectAlibabaCategory(item bssopenapi.Item) string {
+	if (item != bssopenapi.Item{}) {
+		// Provider ID has prefix "i-" for node in Alibaba
+		if strings.HasPrefix(item.InstanceID, boaIsNode) {
+			return kubecost.ComputeCategory
+		}
+		// Provider ID for disk start with "d-" for storage type in Alibaba
+		if strings.HasPrefix(item.InstanceID, boaIsDisk) {
+			return kubecost.StorageCategory
+		}
+		// Network has the highest priority and is based on the usage type of "piece" in Alibaba
+		if item.UsageUnit == boaIsNetwork {
+			return kubecost.NetworkCategory
+		}
+	}
+
+	// Alibaba CUR integration report has service lower case mostly unlike AWS
+	// TO-DO: Can investigate further product codes but bare minimal differentiation for start
+	switch strings.ToLower(item.ProductCode) {
+	case "slb", "eip", "nis", "gtm":
+		return kubecost.NetworkCategory
+	case "ecs", "eds", "sas":
+		return kubecost.ComputeCategory
+	case "ack":
+		return kubecost.ManagementCategory
+	case "ebs", "oss", "scu":
+		return kubecost.StorageCategory
+	default:
+		return kubecost.OtherCategory
+	}
+}

pkg/cloud/aliyunprovider.go → pkg/cloud/alibaba/provider.go

@@ -1,4 +1,4 @@
-package cloud
+package alibaba
 
 import (
 	"errors"
@@ -122,8 +122,9 @@ var alibabaInstanceFamilies = []string{
 }
 
 // AlibabaInfo contains configuration for Alibaba's CUR integration
+// Deprecated: v1.104 Use BOAConfiguration instead
 type AlibabaInfo struct {
-	AlibabaClusterRegion    string `json:"clusterRegion"`
+	AlibabaClusterRegion    string `json:"ClusterRegion"`
 	AlibabaServiceKeyName   string `json:"serviceKeyName"`
 	AlibabaServiceKeySecret string `json:"serviceKeySecret"`
 	AlibabaAccountID        string `json:"accountID"`
@@ -138,6 +139,7 @@ func (ai *AlibabaInfo) IsEmpty() bool {
 }
 
 // AlibabaAccessKey holds Alibaba credentials parsing from the service-key.json file.
+// Deprecated: v1.104 Use AccessKey instead
 type AlibabaAccessKey struct {
 	AccessKeyID     string `json:"alibaba_access_key_id"`
 	SecretAccessKey string `json:"alibaba_secret_access_key"`
@@ -323,15 +325,14 @@ type Alibaba struct {
 	// Lock Needed to provide thread safe
 	DownloadPricingDataLock sync.RWMutex
 	Clientset               clustercache.ClusterCache
-	Config                  *ProviderConfig
-	*CustomProvider
+	Config                  models.ProviderConfig
+	ServiceAccountChecks    *models.ServiceAccountChecks
+	ClusterAccountId        string
+	ClusterRegion           string
 
 	// The following fields are unexported because of avoiding any leak of secrets of these keys.
 	// Alibaba Access key used specifically in signer interface used to sign API calls
-	serviceAccountChecks *models.ServiceAccountChecks
-	clusterAccountId     string
-	clusterRegion        string
-	accessKey            *credentials.AccessKeyCredential
+	accessKey *credentials.AccessKeyCredential
 	// Map of regionID to sdk.client to call API for that region
 	clients map[string]*sdk.Client
 }
@@ -461,10 +462,10 @@ func (alibaba *Alibaba) DownloadPricingData() error {
 	}
 
 	// set the first occurrence of region from the node
-	if alibaba.clusterRegion == "" {
+	if alibaba.ClusterRegion == "" {
 		for _, node := range nodeList {
 			if regionID, ok := node.Labels["topology.kubernetes.io/region"]; ok {
-				alibaba.clusterRegion = regionID
+				alibaba.ClusterRegion = regionID
 				break
 			}
 		}
@@ -478,7 +479,7 @@ func (alibaba *Alibaba) DownloadPricingData() error {
 	for _, pv := range pvList {
 		pvRegion := determinePVRegion(pv)
 		if pvRegion == "" {
-			pvRegion = alibaba.clusterRegion
+			pvRegion = alibaba.ClusterRegion
 		}
 		pricingObj := &AlibabaPricing{}
 		slimK8sDisk := generateSlimK8sDiskFromV1PV(pv, pvRegion)
@@ -685,8 +686,8 @@ func (alibaba *Alibaba) ClusterInfo() (map[string]string, error) {
 	m := make(map[string]string)
 	m["name"] = clusterName
 	m["provider"] = kubecost.AlibabaProvider
-	m["project"] = alibaba.clusterAccountId
-	m["region"] = alibaba.clusterRegion
+	m["project"] = alibaba.ClusterAccountId
+	m["region"] = alibaba.ClusterRegion
 	m["id"] = env.GetClusterID()
 	return m, nil
 }
@@ -912,7 +913,7 @@ func (alibaba *Alibaba) GetPVKey(pv *v1.PersistentVolume, parameters map[string]
 	regionID := defaultRegion
 	// If default Region is not passed default it to cluster region ID.
 	if defaultRegion == "" {
-		regionID = alibaba.clusterRegion
+		regionID = alibaba.ClusterRegion
 	}
 	slimK8sDisk := generateSlimK8sDiskFromV1PV(pv, defaultRegion)
 	return &AlibabaPVKey{
@@ -1356,7 +1357,7 @@ func determinePVRegion(pv *v1.PersistentVolume) string {
 
 	if pvZone == "" {
 		// zone and regionID labels are optional in Alibaba PV creation, while PV through UI creation put's a zone PV is associated with and the region
-		// can be determined from this information. If pv is provision via yaml and the block is missing that's the only time it gets defaulted to clusterRegion.
+		// can be determined from this information. If pv is provision via yaml and the block is missing that's the only time it gets defaulted to ClusterRegion.
 		if pv.Spec.NodeAffinity != nil {
 			nodeAffinity := pv.Spec.NodeAffinity
 			if nodeAffinity.Required != nil && nodeAffinity.Required.NodeSelectorTerms != nil {

pkg/cloud/aliyunprovider_test.go → pkg/cloud/alibaba/provider_test.go

@@ -1,4 +1,4 @@
-package cloud
+package alibaba
 
 import (
 	"fmt"
@@ -9,7 +9,7 @@ import (
 	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/signers"
 	"github.com/opencost/opencost/pkg/cloud/models"
 	v1 "k8s.io/api/core/v1"
-	resource "k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/api/resource"
 )
 
 func TestCreateDescribePriceACSRequest(t *testing.T) {

pkg/cloud/aws/athenaconfiguration.go

@@ -0,0 +1,223 @@
+package aws
+
+import (
+	"fmt"
+
+	"github.com/opencost/opencost/pkg/cloud/config"
+	"github.com/opencost/opencost/pkg/util/json"
+)
+
+// AthenaConfiguration
+type AthenaConfiguration struct {
+	Bucket     string     `json:"bucket"`
+	Region     string     `json:"region"`
+	Database   string     `json:"database"`
+	Table      string     `json:"table"`
+	Workgroup  string     `json:"workgroup"`
+	Account    string     `json:"account"`
+	Authorizer Authorizer `json:"authorizer"`
+}
+
+func (ac *AthenaConfiguration) Validate() error {
+
+	// Validate Authorizer
+	if ac.Authorizer == nil {
+		return fmt.Errorf("AthenaConfiguration: missing Authorizer")
+	}
+
+	err := ac.Authorizer.Validate()
+	if err != nil {
+		return fmt.Errorf("AthenaConfiguration: %s", err)
+	}
+
+	// Validate base properties
+	if ac.Bucket == "" {
+		return fmt.Errorf("AthenaConfiguration: missing bucket")
+	}
+
+	if ac.Region == "" {
+		return fmt.Errorf("AthenaConfiguration: missing region")
+	}
+
+	if ac.Database == "" {
+		return fmt.Errorf("AthenaConfiguration: missing database")
+	}
+
+	if ac.Table == "" {
+		return fmt.Errorf("AthenaConfiguration: missing table")
+	}
+
+	if ac.Account == "" {
+		return fmt.Errorf("AthenaConfiguration: missing account")
+	}
+
+	return nil
+}
+
+func (ac *AthenaConfiguration) Equals(config config.Config) bool {
+	if config == nil {
+		return false
+	}
+	thatConfig, ok := config.(*AthenaConfiguration)
+	if !ok {
+		return false
+	}
+
+	if ac.Authorizer != nil {
+		if !ac.Authorizer.Equals(thatConfig.Authorizer) {
+			return false
+		}
+	} else {
+		if thatConfig.Authorizer != nil {
+			return false
+		}
+	}
+
+	if ac.Bucket != thatConfig.Bucket {
+		return false
+	}
+
+	if ac.Region != thatConfig.Region {
+		return false
+	}
+
+	if ac.Database != thatConfig.Database {
+		return false
+	}
+
+	if ac.Table != thatConfig.Table {
+		return false
+	}
+
+	if ac.Workgroup != thatConfig.Workgroup {
+		return false
+	}
+
+	if ac.Account != thatConfig.Account {
+		return false
+	}
+
+	return true
+}
+
+func (ac *AthenaConfiguration) Sanitize() config.Config {
+	return &AthenaConfiguration{
+		Bucket:     ac.Bucket,
+		Region:     ac.Region,
+		Database:   ac.Database,
+		Table:      ac.Table,
+		Workgroup:  ac.Workgroup,
+		Account:    ac.Account,
+		Authorizer: ac.Authorizer.Sanitize().(Authorizer),
+	}
+}
+
+func (ac *AthenaConfiguration) Key() string {
+	return fmt.Sprintf("%s/%s", ac.Account, ac.Bucket)
+}
+
+func (ac *AthenaConfiguration) UnmarshalJSON(b []byte) error {
+	var f interface{}
+	err := json.Unmarshal(b, &f)
+	if err != nil {
+		return err
+	}
+
+	fmap := f.(map[string]interface{})
+
+	bucket, err := config.GetInterfaceValue[string](fmap, "bucket")
+	if err != nil {
+		return fmt.Errorf("AthenaConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	ac.Bucket = bucket
+
+	region, err := config.GetInterfaceValue[string](fmap, "region")
+	if err != nil {
+		return fmt.Errorf("AthenaConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	ac.Region = region
+
+	database, err := config.GetInterfaceValue[string](fmap, "database")
+	if err != nil {
+		return fmt.Errorf("AthenaConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	ac.Database = database
+
+	table, err := config.GetInterfaceValue[string](fmap, "table")
+	if err != nil {
+		return fmt.Errorf("AthenaConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	ac.Table = table
+
+	workgroup, err := config.GetInterfaceValue[string](fmap, "workgroup")
+	if err != nil {
+		return fmt.Errorf("AthenaConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	ac.Workgroup = workgroup
+
+	account, err := config.GetInterfaceValue[string](fmap, "account")
+	if err != nil {
+		return fmt.Errorf("AthenaConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	ac.Account = account
+
+	authAny, ok := fmap["authorizer"]
+	if !ok {
+		return fmt.Errorf("AthenaConfiguration: UnmarshalJSON: missing authorizer")
+	}
+	authorizer, err := config.AuthorizerFromInterface(authAny, SelectAuthorizerByType)
+	if err != nil {
+		return fmt.Errorf("AthenaConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	ac.Authorizer = authorizer
+
+	return nil
+}
+
+// ConvertAwsAthenaInfoToConfig takes a legacy config and generates a Config based on the presence of properties to match
+// legacy behavior
+func ConvertAwsAthenaInfoToConfig(aai AwsAthenaInfo) config.KeyedConfig {
+	if aai.IsEmpty() {
+		return nil
+	}
+
+	var authorizer Authorizer
+	if aai.ServiceKeyName == "" && aai.ServiceKeySecret == "" {
+		authorizer = &ServiceAccount{}
+	} else {
+		authorizer = &AccessKey{
+			ID:     aai.ServiceKeyName,
+			Secret: aai.ServiceKeySecret,
+		}
+	}
+
+	// Wrap Authorizer with AssumeRole if MasterPayerArn is set
+	if aai.MasterPayerARN != "" {
+		authorizer = &AssumeRole{
+			Authorizer: authorizer,
+			RoleARN:    aai.MasterPayerARN,
+		}
+	}
+
+	var config config.KeyedConfig
+	if aai.AthenaTable != "" || aai.AthenaDatabase != "" {
+		config = &AthenaConfiguration{
+			Bucket:     aai.AthenaBucketName,
+			Region:     aai.AthenaRegion,
+			Database:   aai.AthenaDatabase,
+			Table:      aai.AthenaTable,
+			Workgroup:  aai.AthenaWorkgroup,
+			Account:    aai.AccountID,
+			Authorizer: authorizer,
+		}
+	} else {
+		config = &S3Configuration{
+			Bucket:     aai.AthenaBucketName,
+			Region:     aai.AthenaRegion,
+			Account:    aai.AccountID,
+			Authorizer: authorizer,
+		}
+	}
+
+	return config
+}

pkg/cloud/aws/athenaconfiguration_test.go

@@ -0,0 +1,594 @@
+package aws
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/opencost/opencost/pkg/cloud/config"
+	"github.com/opencost/opencost/pkg/log"
+	"github.com/opencost/opencost/pkg/util/json"
+)
+
+func TestAthenaConfiguration_Validate(t *testing.T) {
+	testCases := map[string]struct {
+		config   AthenaConfiguration
+		expected error
+	}{
+		"valid config access key": {
+			config: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			expected: nil,
+		},
+		"valid config service account": {
+			config: AthenaConfiguration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Database:   "database",
+				Table:      "table",
+				Workgroup:  "workgroup",
+				Account:    "account",
+				Authorizer: &ServiceAccount{},
+			},
+			expected: nil,
+		},
+		"access key invalid": {
+			config: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID: "id",
+				},
+			},
+			expected: fmt.Errorf("AthenaConfiguration: AccessKey: missing Secret"),
+		},
+		"missing Authorizer": {
+			config: AthenaConfiguration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Database:   "database",
+				Table:      "table",
+				Workgroup:  "workgroup",
+				Account:    "account",
+				Authorizer: nil,
+			},
+			expected: fmt.Errorf("AthenaConfiguration: missing Authorizer"),
+		},
+		"missing bucket": {
+			config: AthenaConfiguration{
+				Bucket:     "",
+				Region:     "region",
+				Database:   "database",
+				Table:      "table",
+				Workgroup:  "workgroup",
+				Account:    "account",
+				Authorizer: &ServiceAccount{},
+			},
+			expected: fmt.Errorf("AthenaConfiguration: missing bucket"),
+		},
+		"missing region": {
+			config: AthenaConfiguration{
+				Bucket:     "bucket",
+				Region:     "",
+				Database:   "database",
+				Table:      "table",
+				Workgroup:  "workgroup",
+				Account:    "account",
+				Authorizer: &ServiceAccount{},
+			},
+			expected: fmt.Errorf("AthenaConfiguration: missing region"),
+		},
+		"missing database": {
+			config: AthenaConfiguration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Database:   "",
+				Table:      "table",
+				Workgroup:  "workgroup",
+				Account:    "account",
+				Authorizer: &ServiceAccount{},
+			},
+			expected: fmt.Errorf("AthenaConfiguration: missing database"),
+		},
+		"missing table": {
+			config: AthenaConfiguration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Database:   "database",
+				Table:      "",
+				Workgroup:  "workgroup",
+				Account:    "account",
+				Authorizer: &ServiceAccount{},
+			},
+			expected: fmt.Errorf("AthenaConfiguration: missing table"),
+		},
+		"missing workgroup": {
+			config: AthenaConfiguration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Database:   "database",
+				Table:      "table",
+				Workgroup:  "",
+				Account:    "account",
+				Authorizer: &ServiceAccount{},
+			},
+			expected: nil,
+		},
+		"missing account": {
+			config: AthenaConfiguration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Database:   "database",
+				Table:      "table",
+				Workgroup:  "workgroup",
+				Account:    "",
+				Authorizer: &ServiceAccount{},
+			},
+			expected: fmt.Errorf("AthenaConfiguration: missing account"),
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual := testCase.config.Validate()
+			actualString := "nil"
+			if actual != nil {
+				actualString = actual.Error()
+			}
+			expectedString := "nil"
+			if testCase.expected != nil {
+				expectedString = testCase.expected.Error()
+			}
+			if actualString != expectedString {
+				t.Errorf("errors do not match: Actual: '%s', Expected: '%s", actualString, expectedString)
+			}
+		})
+	}
+}
+
+func TestAthenaConfiguration_Equals(t *testing.T) {
+	testCases := map[string]struct {
+		left     AthenaConfiguration
+		right    config.Config
+		expected bool
+	}{
+		"matching config": {
+			left: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			expected: true,
+		},
+		"different Authorizer": {
+			left: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &AthenaConfiguration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Database:   "database",
+				Table:      "table",
+				Workgroup:  "workgroup",
+				Account:    "account",
+				Authorizer: &ServiceAccount{},
+			},
+			expected: false,
+		},
+		"missing both Authorizer": {
+			left: AthenaConfiguration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Database:   "database",
+				Table:      "table",
+				Workgroup:  "workgroup",
+				Account:    "account",
+				Authorizer: nil,
+			},
+			right: &AthenaConfiguration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Database:   "database",
+				Table:      "table",
+				Workgroup:  "workgroup",
+				Account:    "account",
+				Authorizer: nil,
+			},
+			expected: true,
+		},
+		"missing left Authorizer": {
+			left: AthenaConfiguration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Database:   "database",
+				Table:      "table",
+				Workgroup:  "workgroup",
+				Account:    "account",
+				Authorizer: nil,
+			},
+			right: &AthenaConfiguration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Database:   "database",
+				Table:      "table",
+				Workgroup:  "workgroup",
+				Account:    "account",
+				Authorizer: &ServiceAccount{},
+			},
+			expected: false,
+		},
+		"missing right Authorizer": {
+			left: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &AthenaConfiguration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Database:   "database",
+				Table:      "table",
+				Workgroup:  "workgroup",
+				Account:    "account",
+				Authorizer: nil,
+			},
+			expected: false,
+		},
+		"different bucket": {
+			left: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &AthenaConfiguration{
+				Bucket:    "bucket2",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			expected: false,
+		},
+		"different region": {
+			left: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region2",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			expected: false,
+		},
+		"different database": {
+			left: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database2",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			expected: false,
+		},
+		"different table": {
+			left: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table2",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			expected: false,
+		},
+		"different workgroup": {
+			left: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup2",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			expected: false,
+		},
+		"different account": {
+			left: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account2",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			expected: false,
+		},
+		"different config": {
+			left: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &AccessKey{
+				ID:     "id",
+				Secret: "secret",
+			},
+			expected: false,
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual := testCase.left.Equals(testCase.right)
+			if actual != testCase.expected {
+				t.Errorf("incorrect result: Actual: '%t', Expected: '%t", actual, testCase.expected)
+			}
+		})
+	}
+}
+
+func TestAthenaConfiguration_JSON(t *testing.T) {
+	testCases := map[string]struct {
+		config AthenaConfiguration
+	}{
+		"Empty Config": {
+			config: AthenaConfiguration{},
+		},
+		"AccessKey": {
+			config: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+		},
+
+		"ServiceAccount": {
+			config: AthenaConfiguration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Database:   "database",
+				Table:      "table",
+				Workgroup:  "workgroup",
+				Account:    "account",
+				Authorizer: &ServiceAccount{},
+			},
+		},
+		"AssumeRole with AccessKey": {
+			config: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AssumeRole{
+					Authorizer: &AccessKey{
+						ID:     "id",
+						Secret: "secret",
+					},
+					RoleARN: "12345",
+				},
+			},
+		},
+		"AssumeRole with ServiceAccount": {
+			config: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AssumeRole{
+					Authorizer: &ServiceAccount{},
+					RoleARN:    "12345",
+				},
+			},
+		},
+		"RoleArnNil": {
+			config: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AssumeRole{
+					Authorizer: nil,
+					RoleARN:    "12345",
+				},
+			},
+		},
+		"AssumeRole with AssumeRole with ServiceAccount": {
+			config: AthenaConfiguration{
+				Bucket:    "bucket",
+				Region:    "region",
+				Database:  "database",
+				Table:     "table",
+				Workgroup: "workgroup",
+				Account:   "account",
+				Authorizer: &AssumeRole{
+					Authorizer: &AssumeRole{
+						RoleARN:    "12345",
+						Authorizer: &ServiceAccount{},
+					},
+					RoleARN: "12345",
+				},
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			// test JSON Marshalling
+			configJSON, err := json.Marshal(testCase.config)
+			if err != nil {
+				t.Errorf("failed to marshal configuration: %s", err.Error())
+			}
+			log.Info(string(configJSON))
+			unmarshalledConfig := &AthenaConfiguration{}
+			err = json.Unmarshal(configJSON, unmarshalledConfig)
+			if err != nil {
+				t.Errorf("failed to unmarshal configuration: %s", err.Error())
+			}
+
+			if !testCase.config.Equals(unmarshalledConfig) {
+				t.Error("config does not equal unmarshalled config")
+			}
+		})
+	}
+}

pkg/cloud/aws/athenaintegration.go

@@ -0,0 +1,520 @@
+package aws
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/athena/types"
+	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/opencost/opencost/pkg/kubecost"
+	"github.com/opencost/opencost/pkg/log"
+	"github.com/opencost/opencost/pkg/util/timeutil"
+)
+
+const LabelColumnPrefix = "resource_tags_user_"
+
+// athenaDateLayout is the default AWS date format
+const AthenaDateLayout = "2006-01-02 15:04:05.000"
+
+// Cost Columns
+const AthenaPricingColumn = "line_item_unblended_cost"
+
+// Amortized Cost Columns
+const AthenaRIPricingColumn = "reservation_effective_cost"
+const AthenaSPPricingColumn = "savings_plan_savings_plan_effective_cost"
+
+// Net Cost Columns
+const AthenaNetPricingColumn = "line_item_net_unblended_cost"
+
+// Amortized Net Cost Columns
+const AthenaNetRIPricingColumn = "reservation_net_effective_cost"
+const AthenaNetSPPricingColumn = "savings_plan_net_savings_plan_effective_cost"
+
+// Category Columns
+const AthenaIsNode = "SUBSTRING(line_item_resource_id,1,2) = 'i-'"
+const AthenaIsVol = "SUBSTRING(line_item_resource_id, 1, 4) = 'vol-'"
+const AthenaIsNetwork = "line_item_usage_type LIKE '%Bytes'"
+
+// athenaDateTruncColumn Aggregates line items from the hourly level to daily. "line_item_usage_start_date" is used because at
+// all time values 00:00-23:00 it will truncate to the correct date.
+const AthenaDateColumn = "line_item_usage_start_date"
+const AthenaDateTruncColumn = "DATE_TRUNC('day'," + AthenaDateColumn + ") as usage_date"
+
+const AthenaWhereDateFmt = `line_item_usage_start_date >= date '%s' AND line_item_usage_start_date < date '%s'`
+const AthenaWhereUsage = "(line_item_line_item_type = 'Usage' OR line_item_line_item_type = 'DiscountedUsage' OR line_item_line_item_type = 'SavingsPlanCoveredUsage' OR line_item_line_item_type = 'EdpDiscount' OR line_item_line_item_type = 'PrivateRateDiscount')"
+
+// AthenaQueryIndexes is a struct for holding the context of a query
+type AthenaQueryIndexes struct {
+	Query                     string
+	ColumnIndexes             map[string]int
+	TagColumns                []string
+	ListCostColumn            string
+	ListK8sCostColumn         string
+	NetCostColumn             string
+	NetK8sCostColumn          string
+	AmortizedNetCostColumn    string
+	AmortizedNetK8sCostColumn string
+	AmortizedCostColumn       string
+	AmortizedK8sCostColumn    string
+	InvoicedCostColumn        string
+	InvoicedK8sCostColumn     string
+}
+
+type AthenaIntegration struct {
+	AthenaQuerier
+}
+
+// Query Athena for CUR data and build a new CloudCostSetRange containing the info
+func (ai *AthenaIntegration) GetCloudCost(start, end time.Time) (*kubecost.CloudCostSetRange, error) {
+	log.Infof("AthenaIntegration[%s]: GetCloudCost: %s", ai.Key(), kubecost.NewWindow(&start, &end).String())
+	// Query for all column names
+	allColumns, err := ai.GetColumns()
+	if err != nil {
+		return nil, fmt.Errorf("GetCloudCost: error getting Athena columns: %w", err)
+	}
+
+	// List known, hard-coded columns to query
+	groupByColumns := []string{
+		AthenaDateTruncColumn,
+		"line_item_resource_id",
+		"bill_payer_account_id",
+		"line_item_usage_account_id",
+		"line_item_product_code",
+		"line_item_usage_type",
+		AthenaIsNode,
+		AthenaIsVol,
+		AthenaIsNetwork,
+	}
+
+	// Create query indices
+	aqi := AthenaQueryIndexes{}
+
+	// Determine which columns are user-defined tags and add those to the list
+	// of columns to query.
+	for column := range allColumns {
+		if strings.HasPrefix(column, LabelColumnPrefix) {
+			groupByColumns = append(groupByColumns, column)
+			aqi.TagColumns = append(aqi.TagColumns, column)
+		}
+	}
+	var selectColumns []string
+
+	// Duplicate GroupBy Columns into select columns
+	selectColumns = append(selectColumns, groupByColumns...)
+
+	// Clean Up group by columns
+	ai.RemoveColumnAliases(groupByColumns)
+
+	// Build list cost column and add it to the select columns
+	listCostColumn := fmt.Sprintf("SUM(%s) as list_cost", ai.GetListCostColumn())
+	selectColumns = append(selectColumns, listCostColumn)
+	aqi.ListCostColumn = listCostColumn
+	listK8sCostColumn := fmt.Sprintf(
+		"SUM(%s) as list_kubernetes_cost",
+		ai.GetKubernetesCostColumn(allColumns, ai.GetListCostColumn()),
+	)
+	selectColumns = append(selectColumns, listK8sCostColumn)
+	aqi.ListK8sCostColumn = listK8sCostColumn
+
+	// Build net cost column and add it to select columns
+	netCostColumn := fmt.Sprintf("SUM(%s) as net_cost", ai.GetNetCostColumn(allColumns))
+	selectColumns = append(selectColumns, netCostColumn)
+	aqi.NetCostColumn = netCostColumn
+	netK8sCostColumn := fmt.Sprintf(
+		"SUM(%s) as net_kubernetes_cost",
+		ai.GetKubernetesCostColumn(allColumns, ai.GetNetCostColumn(allColumns)),
+	)
+	selectColumns = append(selectColumns, netK8sCostColumn)
+	aqi.NetK8sCostColumn = netK8sCostColumn
+
+	// Build amortized net cost column and add it to select columns
+	amortizedNetCostColumn := fmt.Sprintf("SUM(%s) as amortized_net_cost", ai.GetAmortizedNetCostColumn(allColumns))
+	selectColumns = append(selectColumns, amortizedNetCostColumn)
+	aqi.AmortizedNetCostColumn = amortizedNetCostColumn
+	amortizedNetK8sCostColumn := fmt.Sprintf(
+		"SUM(%s) as amortized_net_kubernetes_cost",
+		ai.GetKubernetesCostColumn(allColumns, ai.GetNetCostColumn(allColumns)),
+	)
+	selectColumns = append(selectColumns, amortizedNetK8sCostColumn)
+	aqi.AmortizedNetK8sCostColumn = amortizedNetK8sCostColumn
+
+	// Build Amortized cost column and add it to select columns
+	amortizedCostColumn := fmt.Sprintf("SUM(%s) as amortized_cost", ai.GetAmortizedCostCase(allColumns))
+	selectColumns = append(selectColumns, amortizedCostColumn)
+	aqi.AmortizedCostColumn = amortizedCostColumn
+	amortizedK8sCostColumn := fmt.Sprintf(
+		"SUM(%s) as amortized_kubernetes_cost",
+		ai.GetKubernetesCostColumn(allColumns, ai.GetAmortizedCostCase(allColumns)),
+	)
+	selectColumns = append(selectColumns, amortizedK8sCostColumn)
+	aqi.AmortizedK8sCostColumn = amortizedK8sCostColumn
+
+	// We are using Net Cost for Invoiced Cost for now as it is the closest approximation
+	invoicedCostColumn := netCostColumn
+	selectColumns = append(selectColumns, invoicedCostColumn)
+	aqi.InvoicedCostColumn = invoicedCostColumn
+	invoicedK8sCostColumn := netK8sCostColumn
+	selectColumns = append(selectColumns, invoicedK8sCostColumn)
+	aqi.InvoicedK8sCostColumn = invoicedK8sCostColumn
+
+	// Build map of query columns to use for parsing query
+	aqi.ColumnIndexes = map[string]int{}
+	for i, column := range selectColumns {
+		aqi.ColumnIndexes[column] = i
+	}
+	athenaWhereDate := fmt.Sprintf(AthenaWhereDateFmt, start.Format("2006-01-02"), end.Format("2006-01-02"))
+
+	// Query for all line items with a resource_id or from AWS Marketplace, which did not end before
+	// the range or start after it. This captures all costs with any amount of
+	// overlap with the range, for which we will only extract the relevant costs
+	whereConjuncts := []string{
+		athenaWhereDate,
+		AthenaWhereUsage,
+	}
+	columnStr := strings.Join(selectColumns, ", ")
+	whereClause := strings.Join(whereConjuncts, " AND ")
+	groupByStr := strings.Join(groupByColumns, ", ")
+	queryStr := `
+		SELECT %s
+		FROM %s
+		WHERE %s
+		GROUP BY %s
+	`
+	aqi.Query = fmt.Sprintf(queryStr, columnStr, ai.Table, whereClause, groupByStr)
+
+	ccsr, err := kubecost.NewCloudCostSetRange(start, end, timeutil.Day, ai.Key())
+	if err != nil {
+		return nil, err
+	}
+
+	// Generate row handling function.
+	rowHandler := func(row types.Row) {
+		err2 := ai.RowToCloudCost(row, aqi, ccsr)
+		if err2 != nil {
+			log.Errorf("AthenaIntegration: GetCloudCost: error while parsing row: %s", err2.Error())
+		}
+	}
+	log.Debugf("AthenaIntegration[%s]: GetCloudCost: querying: %s", ai.Key(), aqi.Query)
+	// Query CUR data and fill out CCSR
+	err = ai.Query(context.TODO(), aqi.Query, GetAthenaQueryFunc(rowHandler))
+	if err != nil {
+		return nil, err
+	}
+
+	for _, ccs := range ccsr.CloudCostSets {
+		log.Debugf("AthenaIntegration[%s]: GetCloudCost: writing compute items for window %s: %d", ai.Key(), ccs.Window, len(ccs.CloudCosts))
+		ai.ConnectionStatus = ai.GetConnectionStatusFromResult(ccs, ai.ConnectionStatus)
+	}
+	return ccsr, nil
+
+}
+
+func (ai *AthenaIntegration) GetListCostColumn() string {
+	var listCostBuilder strings.Builder
+	listCostBuilder.WriteString("CASE line_item_line_item_type")
+	listCostBuilder.WriteString(" WHEN 'EdpDiscount' THEN 0")
+	listCostBuilder.WriteString(" WHEN 'PrivateRateDiscount' THEN 0")
+	listCostBuilder.WriteString(" ELSE ")
+	listCostBuilder.WriteString(AthenaPricingColumn)
+	listCostBuilder.WriteString(" END")
+	return listCostBuilder.String()
+}
+
+func (ai *AthenaIntegration) GetNetCostColumn(allColumns map[string]bool) string {
+	netCostColumn := ""
+	if allColumns[AthenaNetPricingColumn] { // if Net pricing exists
+		netCostColumn = AthenaNetPricingColumn
+	} else { // Non-net for if there's no net pricing.
+		netCostColumn = AthenaPricingColumn
+	}
+	return netCostColumn
+}
+
+func (ai *AthenaIntegration) GetAmortizedNetCostColumn(allColumns map[string]bool) string {
+	amortizedNetCostCase := ""
+	if allColumns[AthenaNetPricingColumn] { // if Net pricing exists
+		amortizedNetCostCase = ai.GetAmortizedNetCostCase(allColumns)
+	} else { // Non-net for if there's no net pricing.
+		amortizedNetCostCase = ai.GetAmortizedCostCase(allColumns)
+	}
+	return amortizedNetCostCase
+}
+
+// getIsKubernetesColumn generates a boolean column which determines whether a line item is from kubernetes
+func (ai *AthenaIntegration) GetIsKubernetesColumn(allColumns map[string]bool) string {
+	return ai.GetIsKubernetesCase(allColumns)
+}
+
+// getKubernetesCostColumn generates a double column which determines the cost of k8s items in an aggregate
+func (ai *AthenaIntegration) GetKubernetesCostColumn(allColumns map[string]bool, pricingCase string) string {
+	k8sCase := ai.GetIsKubernetesCase(allColumns)
+	return fmt.Sprintf("CAST((%s) as double) * (%s)", k8sCase, pricingCase)
+
+}
+
+func (ai *AthenaIntegration) RemoveColumnAliases(columns []string) {
+	for i, column := range columns {
+		if strings.Contains(column, " as ") {
+			columnValues := strings.Split(column, " as ")
+			columns[i] = columnValues[0]
+		}
+	}
+}
+
+func (ai *AthenaIntegration) ConvertLabelToAWSTag(label string) string {
+	// if the label already has the column prefix assume that it is in the correct format
+	if strings.HasPrefix(label, LabelColumnPrefix) {
+		return label
+	}
+	// replace characters with underscore
+	tag := label
+	tag = strings.ReplaceAll(tag, ".", "_")
+	tag = strings.ReplaceAll(tag, "/", "_")
+	tag = strings.ReplaceAll(tag, ":", "_")
+	tag = strings.ReplaceAll(tag, "-", "_")
+	// add prefix and return
+	return LabelColumnPrefix + tag
+}
+
+func (ai *AthenaIntegration) GetAmortizedCostCase(allColumns map[string]bool) string {
+	// Use unblended costs if Reserved Instances/Savings Plans aren't in use
+	if !allColumns[AthenaRIPricingColumn] && !allColumns[AthenaSPPricingColumn] {
+		return AthenaPricingColumn
+	}
+
+	var costBuilder strings.Builder
+	costBuilder.WriteString("CASE line_item_line_item_type")
+	if allColumns[AthenaRIPricingColumn] {
+		costBuilder.WriteString(" WHEN 'DiscountedUsage' THEN ")
+		costBuilder.WriteString(AthenaRIPricingColumn)
+	}
+
+	if allColumns[AthenaSPPricingColumn] {
+		costBuilder.WriteString(" WHEN 'SavingsPlanCoveredUsage' THEN ")
+		costBuilder.WriteString(AthenaSPPricingColumn)
+	}
+
+	costBuilder.WriteString(" ELSE ")
+	costBuilder.WriteString(AthenaPricingColumn)
+	costBuilder.WriteString(" END")
+	return costBuilder.String()
+}
+
+func (ai *AthenaIntegration) GetAmortizedNetCostCase(allColumns map[string]bool) string {
+	// Use net unblended costs if Reserved Instances/Savings Plans aren't in use
+	if !allColumns[AthenaNetRIPricingColumn] && !allColumns[AthenaNetSPPricingColumn] {
+		return AthenaNetPricingColumn
+	}
+
+	var costBuilder strings.Builder
+	costBuilder.WriteString("CASE line_item_line_item_type")
+	if allColumns[AthenaNetRIPricingColumn] {
+		costBuilder.WriteString(" WHEN 'DiscountedUsage' THEN ")
+		costBuilder.WriteString(AthenaNetRIPricingColumn)
+	}
+
+	if allColumns[AthenaNetSPPricingColumn] {
+		costBuilder.WriteString(" WHEN 'SavingsPlanCoveredUsage' THEN ")
+		costBuilder.WriteString(AthenaNetSPPricingColumn)
+	}
+
+	costBuilder.WriteString(" ELSE ")
+	costBuilder.WriteString(AthenaNetPricingColumn)
+	costBuilder.WriteString(" END")
+	return costBuilder.String()
+}
+
+// GetIsKubernetesCase builds a "CASE" clause which attempts to determine if a line item is kubernetes based on labels
+// that may be available in the CUR
+func (ai *AthenaIntegration) GetIsKubernetesCase(allColumns map[string]bool) string {
+	// k8sColumns is a list of columns where the presence of a value indicates that a resource is part of a kubernetes cluster
+	k8sColumns := []string{
+		"resource_tags_aws_eks_cluster_name",
+		"resource_tags_user_eks_cluster_name",
+		"resource_tags_user_alpha_eksctl_io_cluster_name",
+		"resource_tags_user_kubernetes_io_service_name",
+		"resource_tags_user_kubernetes_io_created_for_pvc_name",
+		"resource_tags_user_kubernetes_io_created_for_pv_name",
+	}
+	var k8sBuilder strings.Builder
+
+	k8sBuilder.WriteString("CASE ")
+	// EKS is always kubernetes
+	k8sBuilder.WriteString("WHEN line_item_product_code = 'AmazonEKS' THEN TRUE ")
+	for _, k8sColumn := range k8sColumns {
+		if _, ok := allColumns[k8sColumn]; ok {
+			k8sBuilder.WriteString("WHEN ")
+			k8sBuilder.WriteString(k8sColumn)
+			k8sBuilder.WriteString(" <> '' THEN TRUE ")
+		}
+	}
+
+	k8sBuilder.WriteString("ELSE FALSE END")
+	return k8sBuilder.String()
+}
+
+func (ai *AthenaIntegration) RowToCloudCost(row types.Row, aqi AthenaQueryIndexes, ccsr *kubecost.CloudCostSetRange) error {
+	if len(row.Data) < len(aqi.ColumnIndexes) {
+		return fmt.Errorf("rowToCloudCost: row with fewer than %d columns (has only %d)", len(aqi.ColumnIndexes), len(row.Data))
+	}
+
+	// Iterate through the slice of tag columns, assigning
+	// values to the column names, minus the tag prefix.
+	labels := kubecost.CloudCostLabels{}
+	labelValues := []string{}
+	for _, tagColumnName := range aqi.TagColumns {
+		labelName := strings.TrimPrefix(tagColumnName, LabelColumnPrefix)
+		value := GetAthenaRowValue(row, aqi.ColumnIndexes, tagColumnName)
+		if value != "" {
+			labels[labelName] = value
+			labelValues = append(labelValues, value)
+		}
+	}
+
+	invoiceEntityID := GetAthenaRowValue(row, aqi.ColumnIndexes, "bill_payer_account_id")
+	accountID := GetAthenaRowValue(row, aqi.ColumnIndexes, "line_item_usage_account_id")
+	startStr := GetAthenaRowValue(row, aqi.ColumnIndexes, AthenaDateTruncColumn)
+	providerID := GetAthenaRowValue(row, aqi.ColumnIndexes, "line_item_resource_id")
+	productCode := GetAthenaRowValue(row, aqi.ColumnIndexes, "line_item_product_code")
+	usageType := GetAthenaRowValue(row, aqi.ColumnIndexes, "line_item_usage_type")
+	isNode, _ := strconv.ParseBool(GetAthenaRowValue(row, aqi.ColumnIndexes, AthenaIsNode))
+	isVol, _ := strconv.ParseBool(GetAthenaRowValue(row, aqi.ColumnIndexes, AthenaIsVol))
+	isNetwork, _ := strconv.ParseBool(GetAthenaRowValue(row, aqi.ColumnIndexes, AthenaIsNetwork))
+
+	listCost, err := GetAthenaRowValueFloat(row, aqi.ColumnIndexes, aqi.ListCostColumn)
+	if err != nil {
+		return err
+	}
+
+	listK8sCost, err := GetAthenaRowValueFloat(row, aqi.ColumnIndexes, aqi.ListK8sCostColumn)
+	if err != nil {
+		return err
+	}
+
+	netCost, err := GetAthenaRowValueFloat(row, aqi.ColumnIndexes, aqi.NetCostColumn)
+	if err != nil {
+		return err
+	}
+
+	netK8sCost, err := GetAthenaRowValueFloat(row, aqi.ColumnIndexes, aqi.NetK8sCostColumn)
+	if err != nil {
+		return err
+	}
+
+	amortizedNetCost, err := GetAthenaRowValueFloat(row, aqi.ColumnIndexes, aqi.AmortizedNetCostColumn)
+	if err != nil {
+		return err
+	}
+
+	amortizedNetK8sCost, err := GetAthenaRowValueFloat(row, aqi.ColumnIndexes, aqi.AmortizedNetK8sCostColumn)
+	if err != nil {
+		return err
+	}
+	amortizedCost, err := GetAthenaRowValueFloat(row, aqi.ColumnIndexes, aqi.AmortizedCostColumn)
+	if err != nil {
+		return err
+	}
+
+	amortizedK8sCost, err := GetAthenaRowValueFloat(row, aqi.ColumnIndexes, aqi.AmortizedK8sCostColumn)
+	if err != nil {
+		return err
+	}
+
+	invoicedCost, err := GetAthenaRowValueFloat(row, aqi.ColumnIndexes, aqi.InvoicedCostColumn)
+	if err != nil {
+		return err
+	}
+
+	invoicedK8sCost, err := GetAthenaRowValueFloat(row, aqi.ColumnIndexes, aqi.InvoicedK8sCostColumn)
+	if err != nil {
+		return err
+	}
+
+	// Identify resource category in the CUR
+	category := SelectAWSCategory(isNode, isVol, isNetwork, providerID, productCode)
+
+	// Retrieve final stanza of product code for ProviderID
+	if productCode == "AWSELB" || productCode == "AmazonFSx" {
+		providerID = ParseARN(providerID)
+	}
+
+	if productCode == "AmazonEKS" && category == kubecost.ComputeCategory {
+		if strings.Contains(usageType, "CPU") {
+			providerID = fmt.Sprintf("%s/CPU", providerID)
+		} else if strings.Contains(usageType, "GB") {
+			providerID = fmt.Sprintf("%s/RAM", providerID)
+		}
+	}
+
+	properties := kubecost.CloudCostProperties{
+		ProviderID:      providerID,
+		Provider:        kubecost.AWSProvider,
+		AccountID:       accountID,
+		InvoiceEntityID: invoiceEntityID,
+		Service:         productCode,
+		Category:        category,
+		Labels:          labels,
+	}
+
+	start, err := time.Parse(AthenaDateLayout, startStr)
+	if err != nil {
+		return fmt.Errorf("unable to parse %s: '%s'", AthenaDateTruncColumn, err.Error())
+	}
+	end := start.AddDate(0, 0, 1)
+
+	cc := &kubecost.CloudCost{
+		Properties: &properties,
+		Window:     kubecost.NewWindow(&start, &end),
+		ListCost: kubecost.CostMetric{
+			Cost:              listCost,
+			KubernetesPercent: ai.CalculateK8sPercent(listCost, listK8sCost),
+		},
+		NetCost: kubecost.CostMetric{
+			Cost:              netCost,
+			KubernetesPercent: ai.CalculateK8sPercent(netCost, netK8sCost),
+		},
+		AmortizedNetCost: kubecost.CostMetric{
+			Cost:              amortizedNetCost,
+			KubernetesPercent: ai.CalculateK8sPercent(amortizedNetCost, amortizedNetK8sCost),
+		},
+		AmortizedCost: kubecost.CostMetric{
+			Cost:              amortizedCost,
+			KubernetesPercent: ai.CalculateK8sPercent(amortizedCost, amortizedK8sCost),
+		},
+		InvoicedCost: kubecost.CostMetric{
+			Cost:              invoicedCost,
+			KubernetesPercent: ai.CalculateK8sPercent(invoicedCost, invoicedK8sCost),
+		},
+	}
+
+	ccsr.LoadCloudCost(cc)
+	return nil
+}
+
+func (ai *AthenaIntegration) CalculateK8sPercent(cost, k8sCost float64) float64 {
+	// Calculate percent of cost that is k8s with the k8sCost
+	k8sPercent := 0.0
+	if k8sCost != 0.0 && cost != 0.0 {
+		k8sPercent = k8sCost / cost
+	}
+	return k8sPercent
+}
+
+func (ai *AthenaIntegration) GetConnectionStatusFromResult(result cloud.EmptyChecker, currentStatus cloud.ConnectionStatus) cloud.ConnectionStatus {
+	if result.IsEmpty() && currentStatus != cloud.SuccessfulConnection {
+		return cloud.MissingData
+	}
+	return cloud.SuccessfulConnection
+}
+
+func (ai *AthenaIntegration) GetConnectionStatus() string {
+	// initialize status if it has not done so; this can happen if the integration is inactive
+	if ai.ConnectionStatus.String() == "" {
+		ai.ConnectionStatus = cloud.InitialStatus
+	}
+
+	return ai.ConnectionStatus.String()
+}

pkg/cloud/aws/athenaintegration_test.go

@@ -0,0 +1,65 @@
+package aws
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/opencost/opencost/pkg/util/json"
+	"github.com/opencost/opencost/pkg/util/timeutil"
+)
+
+func TestAthenaIntegration_GetCloudCost(t *testing.T) {
+	athenaConfigPath := os.Getenv("ATHENA_CONFIGURATION")
+	if athenaConfigPath == "" {
+		t.Skip("skipping integration test, set environment variable ATHENA_CONFIGURATION")
+	}
+	athenaConfigBin, err := os.ReadFile(athenaConfigPath)
+	if err != nil {
+		t.Fatalf("failed to read config file: %s", err.Error())
+	}
+	var athenaConfig AthenaConfiguration
+	err = json.Unmarshal(athenaConfigBin, &athenaConfig)
+	if err != nil {
+		t.Fatalf("failed to unmarshal config from JSON: %s", err.Error())
+	}
+	testCases := map[string]struct {
+		integration *AthenaIntegration
+		start       time.Time
+		end         time.Time
+		expected    bool
+	}{
+		// No CUR data is expected within 2 days of now
+		"too_recent_window": {
+			integration: &AthenaIntegration{
+				AthenaQuerier: AthenaQuerier{
+					AthenaConfiguration: athenaConfig,
+				},
+			},
+			end:      time.Now(),
+			start:    time.Now().Add(-timeutil.Day),
+			expected: true,
+		},
+		// CUR data should be available
+		"last week window": {
+			integration: &AthenaIntegration{
+				AthenaQuerier: AthenaQuerier{
+					AthenaConfiguration: athenaConfig,
+				},
+			},
+			end:      time.Now().Add(-7 * timeutil.Day),
+			start:    time.Now().Add(-8 * timeutil.Day),
+			expected: false,
+		},
+	}
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual, err := testCase.integration.GetCloudCost(testCase.start, testCase.end)
+			if err != nil {
+				t.Errorf("Other error during testing %s", err)
+			} else if actual.IsEmpty() != testCase.expected {
+				t.Errorf("Incorrect result, actual emptiness: %t, expected: %t", actual.IsEmpty(), testCase.expected)
+			}
+		})
+	}
+}

pkg/cloud/aws/athenaquerier.go

@@ -0,0 +1,259 @@
+package aws
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/opencost/opencost/pkg/cloud"
+	cloudconfig "github.com/opencost/opencost/pkg/cloud/config"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/athena"
+	"github.com/aws/aws-sdk-go-v2/service/athena/types"
+	"github.com/opencost/opencost/pkg/kubecost"
+	"github.com/opencost/opencost/pkg/log"
+	"github.com/opencost/opencost/pkg/util/stringutil"
+)
+
+type AthenaQuerier struct {
+	AthenaConfiguration
+	ConnectionStatus cloud.ConnectionStatus
+}
+
+func (aq *AthenaQuerier) Equals(config cloudconfig.Config) bool {
+	thatConfig, ok := config.(*AthenaQuerier)
+	if !ok {
+		return false
+	}
+
+	return aq.AthenaConfiguration.Equals(&thatConfig.AthenaConfiguration)
+}
+
+// GetColumns returns a list of the names of all columns in the configured
+// Athena table
+func (aq *AthenaQuerier) GetColumns() (map[string]bool, error) {
+	columnSet := map[string]bool{}
+
+	// This Query is supported by Athena tables and views
+	q := `SELECT column_name FROM information_schema.columns WHERE table_schema = '%s' AND table_name = '%s'`
+	query := fmt.Sprintf(q, aq.Database, aq.Table)
+
+	athenaErr := aq.Query(context.TODO(), query, GetAthenaQueryFunc(func(row types.Row) {
+		columnSet[*row.Data[0].VarCharValue] = true
+	}))
+
+	if athenaErr != nil {
+		return columnSet, athenaErr
+	}
+
+	if len(columnSet) == 0 {
+		log.Infof("No columns retrieved from Athena")
+	}
+
+	return columnSet, nil
+}
+
+func (aq *AthenaQuerier) Query(ctx context.Context, query string, fn func(*athena.GetQueryResultsOutput) bool) error {
+	err := aq.Validate()
+	if err != nil {
+		aq.ConnectionStatus = cloud.InvalidConfiguration
+		return err
+	}
+
+	log.Debugf("AthenaQuerier[%s]: Performing Query: %s", aq.Key(), query)
+	err = aq.queryAthenaPaginated(ctx, query, fn)
+	if err != nil {
+		aq.ConnectionStatus = cloud.FailedConnection
+		return err
+	}
+	return nil
+}
+
+func (aq *AthenaQuerier) GetAthenaClient() (*athena.Client, error) {
+	cfg, err := aq.Authorizer.CreateAWSConfig(aq.Region)
+	if err != nil {
+		return nil, err
+	}
+	cli := athena.NewFromConfig(cfg)
+	return cli, nil
+}
+
+// QueryAthenaPaginated executes athena query and processes results. An error from this method indicates a
+// FAILED_CONNECTION CloudConnectionStatus and should immediately stop the caller to maintain the correct CloudConnectionStatus
+func (aq *AthenaQuerier) queryAthenaPaginated(ctx context.Context, query string, fn func(*athena.GetQueryResultsOutput) bool) error {
+
+	queryExecutionCtx := &types.QueryExecutionContext{
+		Database: aws.String(aq.Database),
+	}
+
+	resultConfiguration := &types.ResultConfiguration{
+		OutputLocation: aws.String(aq.Bucket),
+	}
+	startQueryExecutionInput := &athena.StartQueryExecutionInput{
+		QueryString:           aws.String(query),
+		QueryExecutionContext: queryExecutionCtx,
+		ResultConfiguration:   resultConfiguration,
+	}
+
+	// Only set if there is a value, the default input is nil
+	if aq.Workgroup != "" {
+		startQueryExecutionInput.WorkGroup = aws.String(aq.Workgroup)
+	}
+
+	// Create Athena Client
+	cli, err := aq.GetAthenaClient()
+
+	// Query Athena
+	startQueryExecutionOutput, err := cli.StartQueryExecution(ctx, startQueryExecutionInput)
+	if err != nil {
+		return fmt.Errorf("QueryAthenaPaginated: start query error: %s", err.Error())
+	}
+	err = waitForQueryToComplete(ctx, cli, startQueryExecutionOutput.QueryExecutionId)
+	if err != nil {
+		return fmt.Errorf("QueryAthenaPaginated: query execution error: %s", err.Error())
+	}
+	queryResultsInput := &athena.GetQueryResultsInput{
+		QueryExecutionId: startQueryExecutionOutput.QueryExecutionId,
+	}
+	getQueryResultsPaginator := athena.NewGetQueryResultsPaginator(cli, queryResultsInput)
+	for getQueryResultsPaginator.HasMorePages() {
+		pg, err := getQueryResultsPaginator.NextPage(ctx)
+		if err != nil {
+			log.Errorf("queryAthenaPaginated: NextPage error: %s", err.Error())
+			continue
+		}
+		fn(pg)
+	}
+	return nil
+}
+
+func waitForQueryToComplete(ctx context.Context, client *athena.Client, queryExecutionID *string) error {
+	inp := &athena.GetQueryExecutionInput{
+		QueryExecutionId: queryExecutionID,
+	}
+	isQueryStillRunning := true
+	for isQueryStillRunning {
+		qe, err := client.GetQueryExecution(ctx, inp)
+		if err != nil {
+			return err
+		}
+		if qe.QueryExecution.Status.State == "SUCCEEDED" {
+			isQueryStillRunning = false
+			continue
+		}
+		if qe.QueryExecution.Status.State != "RUNNING" && qe.QueryExecution.Status.State != "QUEUED" {
+			return fmt.Errorf("no query results available for query %s", *queryExecutionID)
+		}
+		time.Sleep(2 * time.Second)
+	}
+	return nil
+}
+
+// GetAthenaRowValue retrieve value from athena row based on column names and used stringutil.Bank() to prevent duplicate
+// allocation of strings
+func GetAthenaRowValue(row types.Row, queryColumnIndexes map[string]int, columnName string) string {
+	columnIndex, ok := queryColumnIndexes[columnName]
+	if !ok {
+		return ""
+	}
+	valuePointer := row.Data[columnIndex].VarCharValue
+	if valuePointer == nil {
+		return ""
+	}
+	return stringutil.Bank(*valuePointer)
+}
+
+// getAthenaRowValueFloat retrieve value from athena row based on column names and convert to float if possible
+func GetAthenaRowValueFloat(row types.Row, queryColumnIndexes map[string]int, columnName string) (float64, error) {
+
+	columnIndex, ok := queryColumnIndexes[columnName]
+	if !ok {
+		return 0.0, fmt.Errorf("getAthenaRowValueFloat: missing column index: %s", columnName)
+	}
+
+	valuePointer := row.Data[columnIndex].VarCharValue
+	if valuePointer == nil {
+		return 0.0, fmt.Errorf("getAthenaRowValueFloat: nil field")
+	}
+
+	cost, err := strconv.ParseFloat(*valuePointer, 64)
+	if err != nil {
+		return cost, fmt.Errorf("getAthenaRowValueFloat: failed to parse %s: '%s': %s", columnName, *valuePointer, err.Error())
+	}
+	return cost, nil
+}
+
+func SelectAWSCategory(isNode, isVol, isNetwork bool, providerID, service string) string {
+	// Network has the highest priority and is based on the usage type ending in "Bytes"
+	if isNetwork {
+		return kubecost.NetworkCategory
+	}
+	// The node and volume conditions are mutually exclusive.
+	// Provider ID has prefix "i-"
+	if isNode {
+		return kubecost.ComputeCategory
+	}
+	// Provider ID has prefix "vol-"
+	if isVol {
+		return kubecost.StorageCategory
+	}
+
+	// Default categories based on service
+	switch strings.ToUpper(service) {
+	case "AWSELB", "AWSGLUE", "AMAZONROUTE53":
+		return kubecost.NetworkCategory
+	case "AMAZONEC2", "AWSLAMBDA", "AMAZONELASTICACHE":
+		return kubecost.ComputeCategory
+	case "AMAZONEKS":
+		// Check if line item is a fargate pod
+		if strings.Contains(providerID, ":pod/") {
+			return kubecost.ComputeCategory
+		}
+		return kubecost.ManagementCategory
+	case "AMAZONS3", "AMAZONATHENA", "AMAZONRDS", "AMAZONDYNAMODB", "AWSSECRETSMANAGER", "AMAZONFSX":
+		return kubecost.StorageCategory
+	default:
+		return kubecost.OtherCategory
+	}
+}
+
+var parseARNRx = regexp.MustCompile("^.+\\/(.+)?") // Capture "a406f7761142e4ef58a8f2ba478d2db2" from "arn:aws:elasticloadbalancing:us-east-1:297945954695:loadbalancer/a406f7761142e4ef58a8f2ba478d2db2"
+
+func ParseARN(id string) string {
+	match := parseARNRx.FindStringSubmatch(id)
+	if len(match) == 0 {
+		if id != "" {
+			log.DedupedInfof(10, "aws.parseARN: failed to parse %s", id)
+		}
+		return id
+	}
+	return match[len(match)-1]
+}
+
+func GetAthenaQueryFunc(fn func(types.Row)) func(*athena.GetQueryResultsOutput) bool {
+	pageNum := 0
+	processItemQueryResults := func(page *athena.GetQueryResultsOutput) bool {
+		if page == nil {
+			log.Errorf("AthenaQuerier: Athena page is nil")
+			return false
+		} else if page.ResultSet == nil {
+			log.Errorf("AthenaQuerier: Athena page.ResultSet is nil")
+			return false
+		}
+		rows := page.ResultSet.Rows
+		if pageNum == 0 {
+			rows = page.ResultSet.Rows[1:len(page.ResultSet.Rows)]
+		}
+
+		for _, row := range rows {
+			fn(row)
+		}
+		pageNum++
+		return true
+	}
+	return processItemQueryResults
+}

pkg/cloud/aws/authorizer.go

@@ -0,0 +1,251 @@
+package aws
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
+	"github.com/aws/aws-sdk-go-v2/service/sts"
+	"github.com/opencost/opencost/pkg/cloud/config"
+	"github.com/opencost/opencost/pkg/util/json"
+)
+
+const AccessKeyAuthorizerType = "AWSAccessKey"
+const ServiceAccountAuthorizerType = "AWSServiceAccount"
+const AssumeRoleAuthorizerType = "AWSAssumeRole"
+
+// Authorizer implementations provide aws.Config for AWS SDK calls
+type Authorizer interface {
+	config.Authorizer
+	CreateAWSConfig(string) (aws.Config, error)
+}
+
+// SelectAuthorizerByType is an implementation of AuthorizerSelectorFn and acts as a register for Authorizer types
+func SelectAuthorizerByType(typeStr string) (Authorizer, error) {
+	switch typeStr {
+	case AccessKeyAuthorizerType:
+		return &AccessKey{}, nil
+	case ServiceAccountAuthorizerType:
+		return &ServiceAccount{}, nil
+	case AssumeRoleAuthorizerType:
+		return &AssumeRole{}, nil
+	default:
+		return nil, fmt.Errorf("AWS: provider authorizer type '%s' is not valid", typeStr)
+	}
+}
+
+// AccessKey holds AWS credentials and fulfils the awsV2.CredentialsProvider interface
+type AccessKey struct {
+	ID     string `json:"id"`
+	Secret string `json:"secret"`
+}
+
+// MarshalJSON custom json marshalling functions, sets properties as tagged in struct and sets the authorizer type property
+func (ak *AccessKey) MarshalJSON() ([]byte, error) {
+	fmap := make(map[string]any, 3)
+	fmap[config.AuthorizerTypeProperty] = AccessKeyAuthorizerType
+	fmap["id"] = ak.ID
+	fmap["secret"] = ak.Secret
+	return json.Marshal(fmap)
+}
+
+// Retrieve returns a set of awsV2 credentials using the AccessKey's key and secret.
+// This fulfils the awsV2.CredentialsProvider interface contract.
+func (ak *AccessKey) Retrieve(ctx context.Context) (aws.Credentials, error) {
+	return aws.Credentials{
+		AccessKeyID:     ak.ID,
+		SecretAccessKey: ak.Secret,
+	}, nil
+}
+
+func (ak *AccessKey) Validate() error {
+	if ak.ID == "" {
+		return fmt.Errorf("AccessKey: missing ID")
+	}
+	if ak.Secret == "" {
+		return fmt.Errorf("AccessKey: missing Secret")
+	}
+	return nil
+}
+
+func (ak *AccessKey) Equals(config config.Config) bool {
+	if config == nil {
+		return false
+	}
+	thatConfig, ok := config.(*AccessKey)
+	if !ok {
+		return false
+	}
+
+	if ak.ID != thatConfig.ID {
+		return false
+	}
+	if ak.Secret != thatConfig.Secret {
+		return false
+	}
+	return true
+}
+
+func (ak *AccessKey) Sanitize() config.Config {
+	return &AccessKey{
+		ID:     ak.ID,
+		Secret: config.Redacted,
+	}
+}
+
+// CreateAWSConfig creates an AWS SDK V2 Config for the credentials that it contains for the provided region
+func (ak *AccessKey) CreateAWSConfig(region string) (cfg aws.Config, err error) {
+	err = ak.Validate()
+	if err != nil {
+		return cfg, err
+	}
+	// The AWS SDK v2 requires an object fulfilling the CredentialsProvider interface, which cloud.AccessKey does
+	cfg, err = awsconfig.LoadDefaultConfig(context.TODO(), awsconfig.WithCredentialsProvider(ak), awsconfig.WithRegion(region))
+	if err != nil {
+		return cfg, fmt.Errorf("failed to initialize AWS SDK config for region %s: %s", region, err)
+	}
+	return cfg, nil
+}
+
+// ServiceAccount uses pod annotations along with a service account to authenticate integrations
+type ServiceAccount struct{}
+
+// MarshalJSON custom json marshalling functions, sets properties as tagged in struct and sets the authorizer type property
+func (sa *ServiceAccount) MarshalJSON() ([]byte, error) {
+	fmap := make(map[string]any, 1)
+	fmap[config.AuthorizerTypeProperty] = ServiceAccountAuthorizerType
+	return json.Marshal(fmap)
+}
+
+// Check has nothing to check at this level, connection will fail if Pod Annotation and Service Account are not configured correctly
+func (sa *ServiceAccount) Validate() error {
+	return nil
+}
+
+func (sa *ServiceAccount) Equals(config config.Config) bool {
+	if config == nil {
+		return false
+	}
+	_, ok := config.(*ServiceAccount)
+	if !ok {
+		return false
+	}
+
+	return true
+}
+
+func (sa *ServiceAccount) Sanitize() config.Config {
+	return &ServiceAccount{}
+}
+
+func (sa *ServiceAccount) CreateAWSConfig(region string) (aws.Config, error) {
+	cfg, err := awsconfig.LoadDefaultConfig(context.TODO(), awsconfig.WithRegion(region))
+	if err != nil {
+		return cfg, fmt.Errorf("failed to initialize AWS SDK config for region from annotation %s: %s", region, err)
+	}
+	return cfg, nil
+}
+
+// AssumeRole is a wrapper for another Authorizer which adds an assumed role to the configuration
+type AssumeRole struct {
+	Authorizer Authorizer `json:"authorizer"`
+	RoleARN    string     `json:"roleARN"`
+}
+
+// MarshalJSON custom json marshalling functions, sets properties as tagged in struct and sets the authorizer type property
+func (ara *AssumeRole) MarshalJSON() ([]byte, error) {
+	fmap := make(map[string]any, 3)
+	fmap[config.AuthorizerTypeProperty] = AssumeRoleAuthorizerType
+	fmap["roleARN"] = ara.RoleARN
+	fmap["authorizer"] = ara.Authorizer
+	return json.Marshal(fmap)
+}
+
+// UnmarshalJSON is required for AssumeRole because it needs to unmarshal an Authorizer interface
+func (ara *AssumeRole) UnmarshalJSON(b []byte) error {
+	var f interface{}
+	err := json.Unmarshal(b, &f)
+	if err != nil {
+		return err
+	}
+
+	fmap := f.(map[string]interface{})
+
+	roleARN, err := config.GetInterfaceValue[string](fmap, "roleARN")
+	if err != nil {
+		return fmt.Errorf("StorageConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	ara.RoleARN = roleARN
+
+	authAny, ok := fmap["authorizer"]
+	if !ok {
+		return fmt.Errorf("AssumeRole: UnmarshalJSON: missing Authorizer")
+	}
+	authorizer, err := config.AuthorizerFromInterface(authAny, SelectAuthorizerByType)
+	if err != nil {
+		return fmt.Errorf("AssumeRole: UnmarshalJSON: %s", err.Error())
+	}
+	ara.Authorizer = authorizer
+
+	return nil
+}
+
+func (ara *AssumeRole) CreateAWSConfig(region string) (aws.Config, error) {
+	cfg, _ := ara.Authorizer.CreateAWSConfig(region)
+	// Create the credentials from AssumeRoleProvider to assume the role
+	// referenced by the RoleARN.
+	stsSvc := sts.NewFromConfig(cfg)
+	creds := stscreds.NewAssumeRoleProvider(stsSvc, ara.RoleARN)
+	cfg.Credentials = aws.NewCredentialsCache(creds)
+	return cfg, nil
+}
+
+func (ara *AssumeRole) Validate() error {
+	if ara.Authorizer == nil {
+		return fmt.Errorf("AssumeRole: misisng base Authorizer")
+	}
+	err := ara.Authorizer.Validate()
+	if err != nil {
+		return err
+	}
+
+	if ara.RoleARN == "" {
+		return fmt.Errorf("AssumeRole: misisng RoleARN configuration")
+	}
+
+	return nil
+}
+
+func (ara *AssumeRole) Equals(config config.Config) bool {
+	if config == nil {
+		return false
+	}
+	thatConfig, ok := config.(*AssumeRole)
+	if !ok {
+		return false
+	}
+	if ara.Authorizer != nil {
+		if !ara.Authorizer.Equals(thatConfig.Authorizer) {
+			return false
+		}
+	} else {
+		if thatConfig.Authorizer != nil {
+			return false
+		}
+	}
+
+	if ara.RoleARN != thatConfig.RoleARN {
+		return false
+	}
+
+	return true
+}
+
+func (ara *AssumeRole) Sanitize() config.Config {
+	return &AssumeRole{
+		Authorizer: ara.Authorizer.Sanitize().(Authorizer),
+		RoleARN:    ara.RoleARN,
+	}
+}

pkg/cloud/aws/authorizer_test.go

@@ -0,0 +1,67 @@
+package aws
+
+import (
+	"testing"
+
+	"github.com/opencost/opencost/pkg/cloud/config"
+)
+
+func TestAuthorizerJSON_Sanitize(t *testing.T) {
+
+	testCases := map[string]struct {
+		input    Authorizer
+		expected Authorizer
+	}{
+		"Access Key": {
+			input: &AccessKey{
+				ID:     "ID",
+				Secret: "Secret",
+			},
+			expected: &AccessKey{
+				ID:     "ID",
+				Secret: config.Redacted,
+			},
+		},
+		"Service Account": {
+			input:    &ServiceAccount{},
+			expected: &ServiceAccount{},
+		},
+		"Master Payer Access Key": {
+			input: &AssumeRole{
+				Authorizer: &AccessKey{
+					ID:     "ID",
+					Secret: "Secret",
+				},
+				RoleARN: "role arn",
+			},
+			expected: &AssumeRole{
+				Authorizer: &AccessKey{
+					ID:     "ID",
+					Secret: config.Redacted,
+				},
+				RoleARN: "role arn",
+			},
+		},
+		"Master Payer Service Account": {
+			input: &AssumeRole{
+				Authorizer: &ServiceAccount{},
+				RoleARN:    "role arn",
+			},
+			expected: &AssumeRole{
+				Authorizer: &ServiceAccount{},
+				RoleARN:    "role arn",
+			},
+		},
+	}
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			// Convert to AuthorizerJSON for sanitization
+			sanitizedAuthorizer := tc.input.Sanitize()
+
+			if !tc.expected.Equals(sanitizedAuthorizer) {
+				t.Error("Authorizer was not as expected after Sanitization")
+			}
+
+		})
+	}
+}

pkg/cloud/aws/awsprovider.go → pkg/cloud/aws/provider.go

@@ -187,6 +187,7 @@ type AWS struct {
 }
 
 // AWSAccessKey holds AWS credentials and fulfils the awsV2.CredentialsProvider interface
+// Deprecated: v1.104 Use AccessKey instead
 type AWSAccessKey struct {
 	AccessKeyID     string `json:"aws_access_key_id"`
 	SecretAccessKey string `json:"aws_secret_access_key"`
@@ -393,6 +394,7 @@ type AwsSpotFeedInfo struct {
 }
 
 // AwsAthenaInfo contains configuration for CUR integration
+// Deprecated: v1.104 Use AthenaConfiguration instead
 type AwsAthenaInfo struct {
 	AthenaBucketName string `json:"athenaBucketName"`
 	AthenaRegion     string `json:"athenaRegion"`
@@ -532,6 +534,12 @@ func (aws *AWS) UpdateConfig(r io.Reader, updateType string) (*models.CustomPric
 				return err
 			}
 
+			// If the sample nil service key name is set, zero it out so that it is not
+			// misinterpreted as a real service key.
+			if asfi.ServiceKeyName == "AKIXXX" {
+				asfi.ServiceKeyName = ""
+			}
+
 			c.ServiceKeyName = asfi.ServiceKeyName
 			if asfi.ServiceKeySecret != "" {
 				c.ServiceKeySecret = asfi.ServiceKeySecret
@@ -549,6 +557,13 @@ func (aws *AWS) UpdateConfig(r io.Reader, updateType string) (*models.CustomPric
 			if err != nil {
 				return err
 			}
+
+			// If the sample nil service key name is set, zero it out so that it is not
+			// misinterpreted as a real service key.
+			if aai.ServiceKeyName == "AKIXXX" {
+				aai.ServiceKeyName = ""
+			}
+
 			c.AthenaBucketName = aai.AthenaBucketName
 			c.AthenaRegion = aai.AthenaRegion
 			c.AthenaDatabase = aai.AthenaDatabase
@@ -1399,7 +1414,6 @@ func (aws *AWS) ConfigureAuthWith(config *models.CustomPricing) error {
 
 // Gets the aws key id and secret
 func (aws *AWS) getAWSAuth(forceReload bool, cp *models.CustomPricing) (string, string) {
-
 	// 1. Check config values first (set from frontend UI)
 	if cp.ServiceKeyName != "" && cp.ServiceKeySecret != "" {
 		aws.ServiceAccountChecks.Set("hasKey", &models.ServiceAccountCheck{
@@ -1459,6 +1473,12 @@ func (aws *AWS) loadAWSAuthSecret(force bool) (*AWSAccessKey, error) {
 		return nil, err
 	}
 
+	// If the sample nil service key name is set, zero it out so that it is not
+	// misinterpreted as a real service key.
+	if ak.AccessKeyID == "AKIXXX" {
+		ak.AccessKeyID = ""
+	}
+
 	awsSecret = &ak
 	return awsSecret, nil
 }
@@ -1777,7 +1797,17 @@ func (aws *AWS) findCostForDisk(disk *ec2Types.Volume) (*float64, error) {
 
 	key := "us-east-2" + "," + class
 
-	priceStr := aws.Pricing[key].PV.Cost
+	pricing, ok := aws.Pricing[key]
+	if !ok {
+		return nil, fmt.Errorf("no pricing data for key '%s'", key)
+	}
+	if pricing == nil {
+		return nil, fmt.Errorf("nil pricing data for key '%s'", key)
+	}
+	if pricing.PV == nil {
+		return nil, fmt.Errorf("pricing for key '%s' has nil PV", key)
+	}
+	priceStr := pricing.PV.Cost
 
 	price, err := strconv.ParseFloat(priceStr, 64)
 	if err != nil {
@@ -1848,28 +1878,6 @@ func (aws *AWS) QueryAthenaPaginated(ctx context.Context, query string, fn func(
 	return nil
 }
 
-func waitForQueryToComplete(ctx context.Context, client *athena.Client, queryExecutionID *string) error {
-	inp := &athena.GetQueryExecutionInput{
-		QueryExecutionId: queryExecutionID,
-	}
-	isQueryStillRunning := true
-	for isQueryStillRunning {
-		qe, err := client.GetQueryExecution(ctx, inp)
-		if err != nil {
-			return err
-		}
-		if qe.QueryExecution.Status.State == "SUCCEEDED" {
-			isQueryStillRunning = false
-			continue
-		}
-		if qe.QueryExecution.Status.State != "RUNNING" && qe.QueryExecution.Status.State != "QUEUED" {
-			return fmt.Errorf("no query results available for query %s", *queryExecutionID)
-		}
-		time.Sleep(2 * time.Second)
-	}
-	return nil
-}
-
 type SavingsPlanData struct {
 	ResourceID     string
 	EffectiveCost  float64

pkg/cloud/aws/s3configuration.go

@@ -0,0 +1,134 @@
+package aws
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/opencost/opencost/pkg/cloud/config"
+	"github.com/opencost/opencost/pkg/util/json"
+)
+
+type S3Configuration struct {
+	Bucket     string     `json:"bucket"`
+	Region     string     `json:"region"`
+	Account    string     `json:"account"`
+	Authorizer Authorizer `json:"authorizer"`
+}
+
+func (s3c *S3Configuration) Validate() error {
+	// Validate Authorizer
+	if s3c.Authorizer == nil {
+		return fmt.Errorf("S3Configuration: missing Authorizer")
+	}
+
+	err := s3c.Authorizer.Validate()
+	if err != nil {
+		return fmt.Errorf("S3Configuration: %s", err)
+	}
+
+	// Validate base properties
+	if s3c.Bucket == "" {
+		return fmt.Errorf("S3Configuration: missing bucket")
+	}
+
+	if s3c.Region == "" {
+		return fmt.Errorf("S3Configuration: missing region")
+	}
+
+	if s3c.Account == "" {
+		return fmt.Errorf("S3Configuration: missing account")
+	}
+
+	return nil
+}
+
+func (s3c *S3Configuration) Equals(config config.Config) bool {
+	if config == nil {
+		return false
+	}
+	thatConfig, ok := config.(*S3Configuration)
+	if !ok {
+		return false
+	}
+
+	if s3c.Authorizer != nil {
+		if !s3c.Authorizer.Equals(thatConfig.Authorizer) {
+			return false
+		}
+	} else {
+		if thatConfig.Authorizer != nil {
+			return false
+		}
+	}
+
+	if s3c.Bucket != thatConfig.Bucket {
+		return false
+	}
+
+	if s3c.Region != thatConfig.Region {
+		return false
+	}
+
+	if s3c.Account != thatConfig.Account {
+		return false
+	}
+
+	return true
+}
+
+func (s3c *S3Configuration) Sanitize() config.Config {
+	return &S3Configuration{
+		Bucket:     s3c.Bucket,
+		Region:     s3c.Region,
+		Account:    s3c.Account,
+		Authorizer: s3c.Authorizer.Sanitize().(Authorizer),
+	}
+}
+
+func (s3c *S3Configuration) Key() string {
+	return fmt.Sprintf("%s/%s", s3c.Account, s3c.Bucket)
+}
+
+func (s3c *S3Configuration) UnmarshalJSON(b []byte) error {
+	var f interface{}
+	err := json.Unmarshal(b, &f)
+	if err != nil {
+		return err
+	}
+
+	fmap := f.(map[string]interface{})
+
+	bucket, err := config.GetInterfaceValue[string](fmap, "bucket")
+	if err != nil {
+		return fmt.Errorf("S3Configuration: UnmarshalJSON: %s", err.Error())
+	}
+	s3c.Bucket = bucket
+
+	region, err := config.GetInterfaceValue[string](fmap, "region")
+	if err != nil {
+		return fmt.Errorf("S3Configuration: UnmarshalJSON: %s", err.Error())
+	}
+	s3c.Region = region
+
+	account, err := config.GetInterfaceValue[string](fmap, "account")
+	if err != nil {
+		return fmt.Errorf("S3Configuration: UnmarshalJSON: %s", err.Error())
+	}
+	s3c.Account = account
+
+	authAny, ok := fmap["authorizer"]
+	if !ok {
+		return fmt.Errorf("S3Configuration: UnmarshalJSON: missing authorizer")
+	}
+	authorizer, err := config.AuthorizerFromInterface(authAny, SelectAuthorizerByType)
+	if err != nil {
+		return fmt.Errorf("S3Configuration: UnmarshalJSON: %s", err.Error())
+	}
+	s3c.Authorizer = authorizer
+
+	return nil
+}
+
+func (s3c *S3Configuration) CreateAWSConfig() (aws.Config, error) {
+	return s3c.Authorizer.CreateAWSConfig(s3c.Region)
+}

pkg/cloud/aws/s3connection.go

@@ -0,0 +1,40 @@
+package aws
+
+import (
+	"context"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/opencost/opencost/pkg/cloud/config"
+)
+
+type S3Connection struct {
+	S3Configuration
+}
+
+func (s3c *S3Connection) Equals(config config.Config) bool {
+	thatConfig, ok := config.(*S3Connection)
+	if !ok {
+		return false
+	}
+
+	return s3c.S3Configuration.Equals(&thatConfig.S3Configuration)
+}
+
+func (s3c *S3Connection) GetS3Client() (*s3.Client, error) {
+	cfg, err := s3c.CreateAWSConfig()
+	if err != nil {
+		return nil, err
+	}
+	return s3.NewFromConfig(cfg), nil
+}
+
+func (s3c *S3Connection) ListObjects(cli *s3.Client) (*s3.ListObjectsOutput, error) {
+	objs, err := cli.ListObjects(context.TODO(), &s3.ListObjectsInput{
+		Bucket: aws.String(s3c.Bucket),
+	})
+	if err != nil {
+		return nil, err
+	}
+	return objs, err
+}

pkg/cloud/aws/s3connection_test.go

@@ -0,0 +1,387 @@
+package aws
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/opencost/opencost/pkg/cloud/config"
+	"github.com/opencost/opencost/pkg/log"
+	"github.com/opencost/opencost/pkg/util/json"
+)
+
+func TestS3Configuration_Validate(t *testing.T) {
+	testCases := map[string]struct {
+		config   S3Configuration
+		expected error
+	}{
+		"valid config access key": {
+			config: S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			expected: nil,
+		},
+		"valid config service account": {
+			config: S3Configuration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Account:    "account",
+				Authorizer: &ServiceAccount{},
+			},
+			expected: nil,
+		},
+		"access key invalid": {
+			config: S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					ID: "id",
+				},
+			},
+			expected: fmt.Errorf("S3Configuration: AccessKey: missing Secret"),
+		},
+		"missing Authorizer": {
+			config: S3Configuration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Account:    "account",
+				Authorizer: nil,
+			},
+			expected: fmt.Errorf("S3Configuration: missing Authorizer"),
+		},
+		"missing bucket": {
+			config: S3Configuration{
+				Bucket:     "",
+				Region:     "region",
+				Account:    "account",
+				Authorizer: &ServiceAccount{},
+			},
+			expected: fmt.Errorf("S3Configuration: missing bucket"),
+		},
+		"missing region": {
+			config: S3Configuration{
+				Bucket:     "bucket",
+				Region:     "",
+				Account:    "account",
+				Authorizer: &ServiceAccount{},
+			},
+			expected: fmt.Errorf("S3Configuration: missing region"),
+		},
+		"missing account": {
+			config: S3Configuration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Account:    "",
+				Authorizer: &ServiceAccount{},
+			},
+			expected: fmt.Errorf("S3Configuration: missing account"),
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual := testCase.config.Validate()
+			actualString := "nil"
+			if actual != nil {
+				actualString = actual.Error()
+			}
+			expectedString := "nil"
+			if testCase.expected != nil {
+				expectedString = testCase.expected.Error()
+			}
+			if actualString != expectedString {
+				t.Errorf("errors do not match: Actual: '%s', Expected: '%s", actualString, expectedString)
+			}
+		})
+	}
+}
+
+func TestS3Configuration_Equals(t *testing.T) {
+	testCases := map[string]struct {
+		left     S3Configuration
+		right    config.Config
+		expected bool
+	}{
+		"matching config": {
+			left: S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			expected: true,
+		},
+		"different Authorizer": {
+			left: S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &S3Configuration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Account:    "account",
+				Authorizer: &ServiceAccount{},
+			},
+			expected: false,
+		},
+		"missing both Authorizer": {
+			left: S3Configuration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Account:    "account",
+				Authorizer: nil,
+			},
+			right: &S3Configuration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Account:    "account",
+				Authorizer: nil,
+			},
+			expected: true,
+		},
+		"missing left Authorizer": {
+			left: S3Configuration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Account:    "account",
+				Authorizer: nil,
+			},
+			right: &S3Configuration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Account:    "account",
+				Authorizer: &ServiceAccount{},
+			},
+			expected: false,
+		},
+		"missing right Authorizer": {
+			left: S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &S3Configuration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Account:    "account",
+				Authorizer: nil,
+			},
+			expected: false,
+		},
+		"different bucket": {
+			left: S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &S3Configuration{
+				Bucket:  "bucket2",
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			expected: false,
+		},
+		"different region": {
+			left: S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region2",
+				Account: "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			expected: false,
+		},
+		"different account": {
+			left: S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region",
+				Account: "account2",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			expected: false,
+		},
+		"different config": {
+			left: S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+			right: &AccessKey{
+				ID:     "id",
+				Secret: "secret",
+			},
+			expected: false,
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual := testCase.left.Equals(testCase.right)
+			if actual != testCase.expected {
+				t.Errorf("incorrect result: Actual: '%t', Expected: '%t", actual, testCase.expected)
+			}
+		})
+	}
+}
+
+func TestS3Configuration_JSON(t *testing.T) {
+	testCases := map[string]struct {
+		config S3Configuration
+	}{
+		"Empty Config": {
+			config: S3Configuration{},
+		},
+		"AccessKey": {
+			config: S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AccessKey{
+					ID:     "id",
+					Secret: "secret",
+				},
+			},
+		},
+
+		"ServiceAccount": {
+			config: S3Configuration{
+				Bucket:     "bucket",
+				Region:     "region",
+				Account:    "account",
+				Authorizer: &ServiceAccount{},
+			},
+		},
+		"AssumeRole with AccessKey": {
+			config: S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AssumeRole{
+					Authorizer: &AccessKey{
+						ID:     "id",
+						Secret: "secret",
+					},
+					RoleARN: "12345",
+				},
+			},
+		},
+		"AssumeRole with ServiceAccount": {
+			config: S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AssumeRole{
+					Authorizer: &ServiceAccount{},
+					RoleARN:    "12345",
+				},
+			},
+		},
+		"RoleArnNil": {
+			config: S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AssumeRole{
+					Authorizer: nil,
+					RoleARN:    "12345",
+				},
+			},
+		},
+		"AssumeRole with AssumeRole with ServiceAccount": {
+			config: S3Configuration{
+				Bucket:  "bucket",
+				Region:  "region",
+				Account: "account",
+				Authorizer: &AssumeRole{
+					Authorizer: &AssumeRole{
+						RoleARN:    "12345",
+						Authorizer: &ServiceAccount{},
+					},
+					RoleARN: "12345",
+				},
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			// test JSON Marshalling
+			configJSON, err := json.Marshal(testCase.config)
+			if err != nil {
+				t.Errorf("failed to marshal configuration: %s", err.Error())
+			}
+			log.Info(string(configJSON))
+			unmarshalledConfig := &S3Configuration{}
+			err = json.Unmarshal(configJSON, unmarshalledConfig)
+			if err != nil {
+				t.Errorf("failed to unmarshal configuration: %s", err.Error())
+			}
+
+			if !testCase.config.Equals(unmarshalledConfig) {
+				t.Error("config does not equal unmarshalled config")
+			}
+		})
+	}
+}

pkg/cloud/aws/s3selectquerier.go

@@ -0,0 +1,181 @@
+package aws
+
+import (
+	"context"
+	"encoding/csv"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	s3Types "github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/opencost/opencost/pkg/cloud/config"
+	"github.com/opencost/opencost/pkg/util/stringutil"
+)
+
+type S3SelectQuerier struct {
+	S3Connection
+}
+
+func (s3sq *S3SelectQuerier) Equals(config config.Config) bool {
+	thatConfig, ok := config.(*S3SelectQuerier)
+	if !ok {
+		return false
+	}
+
+	return s3sq.S3Connection.Equals(&thatConfig.S3Connection)
+}
+
+func (s3sq *S3SelectQuerier) Query(query string, queryKeys []string, cli *s3.Client, fn func(*csv.Reader) error) error {
+	for _, queryKey := range queryKeys {
+		reader, err2 := s3sq.fetchCSVReader(query, queryKey, cli, s3Types.FileHeaderInfoUse)
+		if err2 != nil {
+			return err2
+		}
+		err2 = fn(reader)
+		if err2 != nil {
+			return err2
+		}
+	}
+
+	return nil
+}
+
+// GetQueryKeys returns a list of s3 object names, where the there are 1 object for each month within the range between
+// start and end
+func (s3sq *S3SelectQuerier) GetQueryKeys(start, end time.Time, client *s3.Client) ([]string, error) {
+	objs, err := s3sq.ListObjects(client)
+	if err != nil {
+		return nil, err
+	}
+
+	monthStrings, err := getMonthStrings(start, end)
+	if err != err {
+		return nil, err
+	}
+
+	var queryKeys []string
+	// Find all matching "csv.gz" files per monthString
+	for _, monthStr := range monthStrings {
+		for _, obj := range objs.Contents {
+			if strings.Contains(*obj.Key, monthStr) && strings.HasSuffix(*obj.Key, ".csv.gz") {
+				queryKeys = append(queryKeys, *obj.Key)
+			}
+		}
+	}
+
+	if len(queryKeys) == 0 {
+		return nil, fmt.Errorf("no CUR files for given time range")
+	}
+
+	return queryKeys, nil
+}
+
+func (s3sq *S3SelectQuerier) fetchCSVReader(query string, queryKey string, client *s3.Client, fileHeaderInfo s3Types.FileHeaderInfo) (*csv.Reader, error) {
+	input := &s3.SelectObjectContentInput{
+		Bucket:         aws.String(s3sq.Bucket),
+		Key:            aws.String(queryKey),
+		Expression:     aws.String(query),
+		ExpressionType: s3Types.ExpressionTypeSql,
+		InputSerialization: &s3Types.InputSerialization{
+			CompressionType: s3Types.CompressionTypeGzip,
+			CSV: &s3Types.CSVInput{
+				FileHeaderInfo: fileHeaderInfo,
+			},
+		},
+		OutputSerialization: &s3Types.OutputSerialization{
+			CSV: &s3Types.CSVOutput{},
+		},
+	}
+
+	res, err := client.SelectObjectContent(context.TODO(), input)
+	if err != nil {
+		return nil, err
+	}
+	resStream := res.GetStream()
+	// todo: this needs work
+	results, resultWriter := io.Pipe()
+	go func() {
+		defer resultWriter.Close()
+		defer resStream.Close()
+		resStream.Events()
+		for event := range resStream.Events() {
+			switch e := event.(type) {
+			case *s3Types.SelectObjectContentEventStreamMemberRecords:
+				resultWriter.Write(e.Value.Payload)
+			case *s3Types.SelectObjectContentEventStreamMemberEnd:
+				break
+			}
+
+		}
+	}()
+
+	if err := resStream.Err(); err != nil {
+		return nil, fmt.Errorf("failed to read from SelectObjectContent EventStream, %v", err)
+	}
+
+	return csv.NewReader(results), nil
+}
+
+func getMonthStrings(start, end time.Time) ([]string, error) {
+	if start.After(end) {
+		return []string{}, fmt.Errorf("start date must be before end date")
+	}
+	if end.After(time.Now()) {
+		end = time.Now()
+	}
+	dateTemplate := "%d%02d01-%d%02d01/"
+	// set to first of the month
+	currMonth := start.AddDate(0, 0, -start.Day()+1)
+	nextMonth := currMonth.AddDate(0, 1, 0)
+	monthStr := fmt.Sprintf(dateTemplate, currMonth.Year(), int(currMonth.Month()), nextMonth.Year(), int(nextMonth.Month()))
+
+	// Create string for end condition
+	endMonth := end.AddDate(0, 0, -end.Day()+1)
+	endNextMonth := endMonth.AddDate(0, 1, 0)
+	endStr := fmt.Sprintf(dateTemplate, endMonth.Year(), int(endMonth.Month()), endNextMonth.Year(), int(endNextMonth.Month()))
+
+	var monthStrs []string
+	monthStrs = append(monthStrs, monthStr)
+
+	for monthStr != endStr {
+		currMonth = nextMonth
+		nextMonth = nextMonth.AddDate(0, 1, 0)
+		monthStr = fmt.Sprintf(dateTemplate, currMonth.Year(), int(currMonth.Month()), nextMonth.Year(), int(nextMonth.Month()))
+		monthStrs = append(monthStrs, monthStr)
+	}
+
+	return monthStrs, nil
+}
+
+// GetCSVRowValue retrieve value from athena row based on column names and used stringutil.Bank() to prevent duplicate
+// allocation of strings
+func GetCSVRowValue(row []string, queryColumnIndexes map[string]int, columnName string) string {
+	if row == nil {
+		return ""
+	}
+	columnIndex, ok := queryColumnIndexes[columnName]
+	if !ok {
+		return ""
+	}
+	return stringutil.Bank(row[columnIndex])
+}
+
+// GetCSVRowValueFloat retrieve value from athena row based on column names and convert to float if possible.
+func GetCSVRowValueFloat(row []string, queryColumnIndexes map[string]int, columnName string) (float64, error) {
+	if row == nil {
+		return 0.0, fmt.Errorf("getCSVRowValueFloat: nil row")
+	}
+	columnIndex, ok := queryColumnIndexes[columnName]
+	if !ok {
+		return 0.0, fmt.Errorf("getCSVRowValueFloat: missing column index: %s", columnName)
+	}
+	cost, err := strconv.ParseFloat(row[columnIndex], 64)
+	if err != nil {
+		return cost, fmt.Errorf("getCSVRowValueFloat: failed to parse %s: '%s': %s", columnName, row[columnIndex], err.Error())
+	}
+	return cost, nil
+}

pkg/cloud/azure/authorizer.go

@@ -0,0 +1,80 @@
+package azure
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/opencost/opencost/pkg/cloud/config"
+)
+
+const AccessKeyAuthorizerType = "AzureAccessKey"
+
+type Authorizer interface {
+	config.Authorizer
+	GetBlobCredentials() (azblob.Credential, error)
+}
+
+// SelectAuthorizerByType is an implementation of AuthorizerSelectorFn and acts as a register for Authorizer types
+func SelectAuthorizerByType(typeStr string) (Authorizer, error) {
+	switch typeStr {
+	case AccessKeyAuthorizerType:
+		return &AccessKey{}, nil
+	default:
+		return nil, fmt.Errorf("azure: provider authorizer type '%s' is not valid", typeStr)
+	}
+}
+
+type AccessKey struct {
+	AccessKey string `json:"accessKey"`
+	Account   string `json:"account"`
+}
+
+func (ak *AccessKey) MarshalJSON() ([]byte, error) {
+	fmap := make(map[string]any, 3)
+	fmap[config.AuthorizerTypeProperty] = AccessKeyAuthorizerType
+	fmap["accessKey"] = ak.AccessKey
+	fmap["account"] = ak.Account
+	return json.Marshal(fmap)
+}
+
+func (ak *AccessKey) Validate() error {
+	if ak.AccessKey == "" {
+		return fmt.Errorf("AccessKey: missing access key")
+	}
+	if ak.Account == "" {
+		return fmt.Errorf("AccessKey: missing account")
+	}
+	return nil
+}
+
+func (ak *AccessKey) Equals(config config.Config) bool {
+	if config == nil {
+		return false
+	}
+	thatConfig, ok := config.(*AccessKey)
+	if !ok {
+		return false
+	}
+
+	if ak.AccessKey != thatConfig.AccessKey {
+		return false
+	}
+	if ak.Account != thatConfig.Account {
+		return false
+	}
+
+	return true
+}
+
+func (ak *AccessKey) Sanitize() config.Config {
+	return &AccessKey{
+		AccessKey: config.Redacted,
+		Account:   ak.Account,
+	}
+}
+
+func (ak *AccessKey) GetBlobCredentials() (azblob.Credential, error) {
+	// Create a default request pipeline using your storage account name and account key.
+	return azblob.NewSharedKeyCredential(ak.Account, ak.AccessKey)
+}

pkg/cloud/azure/azurestorageintegration.go

@@ -0,0 +1,99 @@
+package azure
+
+import (
+	"strings"
+	"time"
+
+	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/opencost/opencost/pkg/kubecost"
+	"github.com/opencost/opencost/pkg/util/timeutil"
+)
+
+type AzureStorageIntegration struct {
+	AzureStorageBillingParser
+	ConnectionStatus cloud.ConnectionStatus
+}
+
+func (asi *AzureStorageIntegration) GetCloudCost(start, end time.Time) (*kubecost.CloudCostSetRange, error) {
+	ccsr, err := kubecost.NewCloudCostSetRange(start, end, timeutil.Day, asi.Key())
+	if err != nil {
+		return nil, err
+	}
+
+	status, err := asi.ParseBillingData(start, end, func(abv *BillingRowValues) error {
+		s := abv.Date
+		e := abv.Date.Add(timeutil.Day)
+		window := kubecost.NewWindow(&s, &e)
+
+		k8sPtc := 0.0
+		if AzureIsK8s(abv.Tags) {
+			k8sPtc = 1.0
+		}
+
+		// Create CloudCost
+		// Using the NetCost as a 'placeholder' for Invoiced and Amortized Net costs now,
+		// until we can revisit and spend the time to do the calculations correctly
+		cc := &kubecost.CloudCost{
+			Properties: &kubecost.CloudCostProperties{
+				ProviderID:      AzureSetProviderID(abv),
+				Provider:        kubecost.AzureProvider,
+				AccountID:       abv.SubscriptionID,
+				InvoiceEntityID: abv.InvoiceEntityID,
+				Service:         abv.Service,
+				Category:        SelectAzureCategory(abv.MeterCategory),
+				Labels:          abv.Tags,
+			},
+			Window: window,
+			AmortizedNetCost: kubecost.CostMetric{
+				Cost:              abv.NetCost,
+				KubernetesPercent: k8sPtc,
+			},
+			InvoicedCost: kubecost.CostMetric{
+				Cost:              abv.NetCost,
+				KubernetesPercent: k8sPtc,
+			},
+			ListCost: kubecost.CostMetric{
+				Cost:              abv.Cost,
+				KubernetesPercent: k8sPtc,
+			},
+			NetCost: kubecost.CostMetric{
+				Cost:              abv.NetCost,
+				KubernetesPercent: k8sPtc,
+			},
+			// NOTE: on Azure, there is no "AmortizedCost" per se, so we use
+			// AmortizedNetCost, or NetCost, instead.
+			AmortizedCost: kubecost.CostMetric{
+				Cost:              abv.NetCost,
+				KubernetesPercent: k8sPtc,
+			},
+		}
+
+		// Check if Item
+		if abv.IsCompute(cc.Properties.Category) {
+			// TODO: Will need to split VMSS for other features
+			ccsr.LoadCloudCost(cc)
+		}
+		return nil
+	})
+	if err != nil {
+		asi.ConnectionStatus = status
+		return nil, err
+	}
+	return ccsr, nil
+}
+
+// Check for the presence of k8s labels
+func AzureIsK8s(labels map[string]string) bool {
+	for key := range labels {
+		if strings.HasPrefix(key, "aks-managed-") {
+			return true
+		}
+		if strings.HasPrefix(key, "kubernetes.io-created-") {
+			return true
+		}
+		if strings.HasPrefix(key, "k8s-azure-created-") {
+			return true
+		}
+	}
+	return false
+}

pkg/cloud/azure/azurestorageintegration_test.go

@@ -0,0 +1,69 @@
+package azure
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/opencost/opencost/pkg/util/json"
+	"github.com/opencost/opencost/pkg/util/timeutil"
+)
+
+func GetCloudCost_Test(t *testing.T) {
+	azureConfigPath := os.Getenv("AZURE_CONFIGURATION")
+	if azureConfigPath == "" {
+		t.Skip("skipping integration test, set environment variable AZURE_CONFIGURATION")
+	}
+	azureConfigBin, err := os.ReadFile(azureConfigPath)
+	if err != nil {
+		t.Fatalf("failed to read config file: %s", err.Error())
+	}
+	var azureConfig StorageConfiguration
+	err = json.Unmarshal(azureConfigBin, &azureConfig)
+	if err != nil {
+		t.Fatalf("failed to unmarshal config from JSON: %s", err.Error())
+	}
+	testCases := map[string]struct {
+		integration *AzureStorageIntegration
+		start       time.Time
+		end         time.Time
+		expected    bool
+	}{
+		// No CUR data is expected within 2 days of now
+		"too_recent_window": {
+			integration: &AzureStorageIntegration{
+				AzureStorageBillingParser: AzureStorageBillingParser{
+					StorageConnection: StorageConnection{
+						StorageConfiguration: azureConfig,
+					},
+				},
+			},
+			end:      time.Now(),
+			start:    time.Now().Add(-timeutil.Day),
+			expected: true,
+		},
+		// CUR data should be available
+		"last week window": {
+			integration: &AzureStorageIntegration{
+				AzureStorageBillingParser: AzureStorageBillingParser{
+					StorageConnection: StorageConnection{
+						StorageConfiguration: azureConfig,
+					},
+				},
+			},
+			end:      time.Now().Add(-7 * timeutil.Day),
+			start:    time.Now().Add(-8 * timeutil.Day),
+			expected: false,
+		},
+	}
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual, err := testCase.integration.GetCloudCost(testCase.start, testCase.end)
+			if err != nil {
+				t.Errorf("Other error during testing %s", err)
+			} else if actual.IsEmpty() != testCase.expected {
+				t.Errorf("Incorrect result, actual emptiness: %t, expected: %t", actual.IsEmpty(), testCase.expected)
+			}
+		})
+	}
+}

pkg/cloud/azure/billingexportparser.go

@@ -0,0 +1,322 @@
+package azure
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/opencost/opencost/pkg/kubecost"
+	"github.com/opencost/opencost/pkg/log"
+	"github.com/opencost/opencost/pkg/util/json"
+)
+
+const azureDateLayout = "2006-01-02"
+const AzureEnterpriseDateLayout = "01/02/2006"
+
+var groupRegex = regexp.MustCompile("(/[^/]+)")
+
+// BillingRowValues holder for Azure Billing Values
+type BillingRowValues struct {
+	Date            time.Time
+	MeterCategory   string
+	SubscriptionID  string
+	InvoiceEntityID string
+	InstanceID      string
+	Service         string
+	Tags            map[string]string
+	AdditionalInfo  map[string]any
+	Cost            float64
+	NetCost         float64
+}
+
+func (brv *BillingRowValues) IsCompute(category string) bool {
+	if category == kubecost.ComputeCategory {
+		return true
+	}
+
+	if category == kubecost.StorageCategory || category == kubecost.NetworkCategory {
+		if brv.Service == "Microsoft.Compute" {
+			return true
+		}
+	}
+	if category == kubecost.NetworkCategory && brv.MeterCategory == "Virtual Network" {
+		return true
+	}
+	return false
+}
+
+// BillingExportParser holds indexes of relevent fields in Azure Billing CSV in addition to the correct data format
+type BillingExportParser struct {
+	Date            int
+	MeterCategory   int
+	InvoiceEntityID int
+	SubscriptionID  int
+	InstanceID      int
+	Service         int
+	Tags            int
+	AdditionalInfo  int
+	Cost            int
+	NetCost         int
+	DateFormat      string
+}
+
+// match "SubscriptionGuid" in "Abonnement-GUID (SubscriptionGuid)"
+var getParenContentRegEx = regexp.MustCompile("\\((.*?)\\)")
+
+func NewBillingParseSchema(headers []string) (*BillingExportParser, error) {
+	// clear BOM from headers
+	if len(headers) != 0 {
+		headers[0] = strings.TrimPrefix(headers[0], "\xEF\xBB\xBF")
+	}
+
+	headerIndexes := map[string]int{}
+	for i, header := range headers {
+		// Azure Headers in different regions will have english headers in parentheses
+		match := getParenContentRegEx.FindStringSubmatch(header)
+		if len(match) != 0 {
+			header = match[len(match)-1]
+		}
+		headerIndexes[strings.ToLower(header)] = i
+	}
+
+	abp := &BillingExportParser{}
+
+	// Set Date Column and Date Format
+	if i, ok := headerIndexes["usagedatetime"]; ok {
+		abp.Date = i
+		abp.DateFormat = azureDateLayout
+	} else if j, ok2 := headerIndexes["date"]; ok2 {
+		abp.Date = j
+		abp.DateFormat = AzureEnterpriseDateLayout
+	} else {
+		return nil, fmt.Errorf("NewBillingParseSchema: failed to find Date field")
+	}
+
+	// set Subscription ID
+	if i, ok := headerIndexes["subscriptionid"]; ok {
+		abp.SubscriptionID = i
+	} else if j, ok2 := headerIndexes["subscriptionguid"]; ok2 {
+		abp.SubscriptionID = j
+	} else {
+		return nil, fmt.Errorf("NewBillingParseSchema: failed to find Subscription ID field")
+	}
+
+	// Set Billing ID
+	if i, ok := headerIndexes["billingaccountid"]; ok {
+		abp.InvoiceEntityID = i
+	} else if j, ok2 := headerIndexes["billingaccountname"]; ok2 {
+		abp.InvoiceEntityID = j
+	} else {
+		// if no billing ID column is present use subscription ID
+		abp.InvoiceEntityID = abp.SubscriptionID
+	}
+
+	// Set Instance ID
+	if i, ok := headerIndexes["instanceid"]; ok {
+		abp.InstanceID = i
+	} else if j, ok2 := headerIndexes["instancename"]; ok2 {
+		abp.InstanceID = j
+	} else if k, ok3 := headerIndexes["resourceid"]; ok3 {
+		abp.InstanceID = k
+	} else {
+		return nil, fmt.Errorf("NewBillingParseSchema: failed to find Instance ID field")
+	}
+
+	// Set Meter Category
+	if i, ok := headerIndexes["metercategory"]; ok {
+		abp.MeterCategory = i
+	} else {
+		return nil, fmt.Errorf("NewBillingParseSchema: failed to find Meter Category field")
+	}
+
+	// Set Tags
+	if i, ok := headerIndexes["tags"]; ok {
+		abp.Tags = i
+	} else {
+		return nil, fmt.Errorf("NewBillingParseSchema: failed to find Tags field")
+	}
+
+	// Set Additional Info
+	if i, ok := headerIndexes["additionalinfo"]; ok {
+		abp.AdditionalInfo = i
+	} else {
+		return nil, fmt.Errorf("NewBillingParseSchema: failed to find Additional Info field")
+	}
+
+	// Set Service
+	if i, ok := headerIndexes["consumedservice"]; ok {
+		abp.Service = i
+	} else {
+		return nil, fmt.Errorf("NewBillingParseSchema: failed to find Service field")
+	}
+
+	// Set Net Cost
+	if i, ok := headerIndexes["costinbillingcurrency"]; ok {
+		abp.NetCost = i
+	} else if j, ok2 := headerIndexes["pretaxcost"]; ok2 {
+		abp.NetCost = j
+	} else if k, ok3 := headerIndexes["cost"]; ok3 {
+		abp.NetCost = k
+	} else {
+		return nil, fmt.Errorf("NewBillingParseSchema: failed to find Net Cost field")
+	}
+
+	// Set Cost
+	if i, ok := headerIndexes["paygcostinbillingcurrency"]; ok {
+		abp.Cost = i
+	} else {
+		// if no Cost column is present use Net Cost column
+		abp.Cost = abp.NetCost
+	}
+
+	return abp, nil
+}
+
+func (bep *BillingExportParser) ParseRow(start, end time.Time, record []string) *BillingRowValues {
+	usageDate, err := time.Parse(bep.DateFormat, record[bep.Date])
+	if err != nil {
+		// try other format, and switch if successful
+		if bep.DateFormat == azureDateLayout {
+			bep.DateFormat = AzureEnterpriseDateLayout
+		} else {
+			bep.DateFormat = azureDateLayout
+		}
+		usageDate, err = time.Parse(bep.DateFormat, record[bep.Date])
+		// If parse still fails then return line
+		if err != nil {
+			log.Errorf("failed to parse usage date: '%s'", record[bep.Date])
+			return nil
+		}
+	}
+
+	// skip if usage data isn't in subject window
+	if usageDate.Before(start) || !usageDate.Before(end) {
+		return nil
+	}
+
+	cost, err := strconv.ParseFloat(record[bep.Cost], 64)
+	if err != nil {
+		log.Errorf("failed to parse cost: '%s'", record[bep.Cost])
+		return nil
+	}
+
+	netCost, err := strconv.ParseFloat(record[bep.NetCost], 64)
+	if err != nil {
+		log.Errorf("failed to parse net cost: '%s'", record[bep.NetCost])
+		return nil
+	}
+
+	additionalInfo := make(map[string]any)
+	additionalInfoJson := encloseInBrackets(record[bep.AdditionalInfo])
+	if additionalInfoJson != "" {
+		err = json.Unmarshal([]byte(additionalInfoJson), &additionalInfo)
+		if err != nil {
+			log.Errorf("Could not parse additional information %s, with Error: %s", additionalInfoJson, err.Error())
+		}
+	}
+
+	tags := make(map[string]string)
+	tagJson := encloseInBrackets(record[bep.Tags])
+	if tagJson != "" {
+		tagsAny := make(map[string]any)
+		err = json.Unmarshal([]byte(tagJson), &tagsAny)
+		if err != nil {
+			log.Errorf("Could not parse tags: %v, with Error: %s", tagJson, err.Error())
+		}
+
+		for name, value := range tagsAny {
+			if valueStr, ok := value.(string); ok && valueStr != "" {
+				tags[name] = valueStr
+			}
+		}
+	}
+
+	return &BillingRowValues{
+		Date:            usageDate,
+		MeterCategory:   record[bep.MeterCategory],
+		SubscriptionID:  record[bep.SubscriptionID],
+		InvoiceEntityID: record[bep.InvoiceEntityID],
+		InstanceID:      record[bep.InstanceID],
+		Service:         record[bep.Service],
+		Tags:            tags,
+		AdditionalInfo:  additionalInfo,
+		Cost:            cost,
+		NetCost:         netCost,
+	}
+}
+
+// enclose json strings in brackets if they are missing
+func encloseInBrackets(jsonString string) string {
+	if jsonString == "" || (jsonString[0] == '{' && jsonString[len(jsonString)-1] == '}') {
+		return jsonString
+	}
+	return fmt.Sprintf("{%s}", jsonString)
+}
+
+func AzureSetProviderID(abv *BillingRowValues) string {
+	category := SelectAzureCategory(abv.MeterCategory)
+	if value, ok := abv.AdditionalInfo["VMName"]; ok {
+		return "azure://" + resourceGroupToLowerCase(abv.InstanceID) + getVMNumberForVMSS(fmt.Sprintf("%v", value))
+	} else if value, ok := abv.AdditionalInfo["VmName"]; ok {
+		return "azure://" + resourceGroupToLowerCase(abv.InstanceID) + getVMNumberForVMSS(fmt.Sprintf("%v", value))
+	} else if value2, ook := abv.AdditionalInfo["IpAddress"]; ook && abv.MeterCategory == "Virtual Network" {
+		return fmt.Sprintf("%v", value2)
+	}
+
+	if category == kubecost.StorageCategory {
+		if value2, ok2 := abv.Tags["creationSource"]; ok2 {
+			creationSource := fmt.Sprintf("%v", value2)
+			return strings.TrimPrefix(creationSource, "aks-")
+		} else if value2, ok2 := abv.Tags["aks-managed-creationSource"]; ok2 {
+			creationSource := fmt.Sprintf("%v", value2)
+			return strings.TrimPrefix(creationSource, "vmssclient-")
+		} else {
+			return getSubStringAfterFinalSlash(abv.InstanceID)
+		}
+	}
+	return "azure://" + resourceGroupToLowerCase(abv.InstanceID)
+}
+
+func SelectAzureCategory(meterCategory string) string {
+	if meterCategory == "Virtual Machines" {
+		return kubecost.ComputeCategory
+	} else if meterCategory == "Storage" {
+		return kubecost.StorageCategory
+	} else if meterCategory == "Load Balancer" || meterCategory == "Bandwidth" || meterCategory == "Virtual Network" {
+		return kubecost.NetworkCategory
+	} else {
+		return kubecost.OtherCategory
+	}
+}
+
+func resourceGroupToLowerCase(providerID string) string {
+	var sb strings.Builder
+	for matchNum, group := range groupRegex.FindAllString(providerID, -1) {
+		if matchNum == 3 {
+			sb.WriteString(strings.ToLower(group))
+		} else {
+			sb.WriteString(group)
+		}
+	}
+	return sb.String()
+}
+
+// Returns the substring after the final "/" in a string
+func getSubStringAfterFinalSlash(id string) string {
+	index := strings.LastIndex(id, "/")
+	if index == -1 {
+		log.DedupedInfof(5, "azure.getSubStringAfterFinalSlash: failed to parse %s", id)
+		return id
+	}
+	return id[index+1:]
+}
+
+func getVMNumberForVMSS(vmName string) string {
+	vmNameSplit := strings.Split(vmName, "_")
+	if len(vmNameSplit) > 1 {
+		return "/virtualMachines/" + vmNameSplit[1]
+	}
+	return ""
+}

pkg/cloud/azure/billingexportparser_test.go

@@ -0,0 +1,194 @@
+package azure
+
+import (
+	"encoding/csv"
+	"os"
+	"testing"
+	"time"
+)
+
+const billingExportPath = "./resources/billingexports/"
+const headerSetPath = billingExportPath + "headersets/"
+const valueCasesPath = billingExportPath + "values/"
+
+type TestCSVRetriever struct {
+	CSVName string
+}
+
+func (tcr TestCSVRetriever) getCSVReaders(start, end time.Time) ([]*csv.Reader, error) {
+	csvFile, err := os.Open(tcr.CSVName)
+	if err != nil {
+		return nil, err
+	}
+	reader := csv.NewReader(csvFile)
+	return append([]*csv.Reader{}, reader), nil
+}
+
+func Test_NewBillingExportParser(t *testing.T) {
+	loc, _ := time.LoadLocation("UTC")
+	start := time.Date(2021, 2, 1, 00, 00, 00, 00, loc)
+	end := time.Date(2021, 2, 3, 00, 00, 00, 00, loc)
+	tests := map[string]struct {
+		input    string
+		expected BillingExportParser
+	}{
+		"English Headers": {
+			input: "PayAsYouGo.csv",
+			expected: BillingExportParser{
+				Date:            3,
+				MeterCategory:   4,
+				InvoiceEntityID: 0,
+				SubscriptionID:  0,
+				InstanceID:      14,
+				Service:         12,
+				Tags:            15,
+				AdditionalInfo:  17,
+				Cost:            11,
+				NetCost:         11,
+				DateFormat:      azureDateLayout,
+			},
+		},
+		"Enterprise Camel Headers": {
+			input: "EnterpriseCamel.csv",
+			expected: BillingExportParser{
+				Date:            11,
+				MeterCategory:   18,
+				InvoiceEntityID: 0,
+				SubscriptionID:  23,
+				InstanceID:      29,
+				Service:         15,
+				Tags:            45,
+				AdditionalInfo:  44,
+				Cost:            38,
+				NetCost:         38,
+				DateFormat:      AzureEnterpriseDateLayout,
+			},
+		},
+		"Enterprise Headers": {
+			input: "Enterprise.csv",
+			expected: BillingExportParser{
+				Date:            7,
+				MeterCategory:   9,
+				InvoiceEntityID: 39,
+				SubscriptionID:  3,
+				InstanceID:      20,
+				Service:         19,
+				Tags:            21,
+				AdditionalInfo:  23,
+				Cost:            17,
+				NetCost:         17,
+				DateFormat:      AzureEnterpriseDateLayout,
+			},
+		},
+		"German Headers": {
+			input: "German.csv",
+			expected: BillingExportParser{
+				Date:            3,
+				MeterCategory:   4,
+				InvoiceEntityID: 0,
+				SubscriptionID:  0,
+				InstanceID:      14,
+				Service:         12,
+				Tags:            15,
+				AdditionalInfo:  17,
+				Cost:            11,
+				NetCost:         11,
+				DateFormat:      azureDateLayout,
+			},
+		},
+		"YA Headers": {
+			input: "YA.csv",
+			expected: BillingExportParser{
+				Date:            3,
+				MeterCategory:   4,
+				InvoiceEntityID: 0,
+				SubscriptionID:  0,
+				InstanceID:      14,
+				Service:         12,
+				Tags:            15,
+				AdditionalInfo:  17,
+				Cost:            11,
+				NetCost:         11,
+				DateFormat:      AzureEnterpriseDateLayout,
+			},
+		},
+		"BOM Prefixed Headers": {
+			input: "BOM.csv",
+			expected: BillingExportParser{
+				Date:            3,
+				MeterCategory:   4,
+				InvoiceEntityID: 0,
+				SubscriptionID:  0,
+				InstanceID:      14,
+				Service:         12,
+				Tags:            15,
+				AdditionalInfo:  17,
+				Cost:            11,
+				NetCost:         11,
+				DateFormat:      azureDateLayout,
+			},
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			csvRetriever := TestCSVRetriever{
+				CSVName: headerSetPath + tc.input,
+			}
+			csvs, err := csvRetriever.getCSVReaders(start, end)
+			if err != nil {
+				t.Errorf("Failed to read specified CSV: %s", err.Error())
+			}
+			reader := csvs[0]
+			headers, _ := reader.Read()
+			abp, err := NewBillingParseSchema(headers)
+			if err != nil {
+				t.Errorf("failed to create Azure Billing Parser from headers with error: %s", err.Error())
+			}
+
+			if abp.DateFormat != tc.expected.DateFormat {
+				t.Errorf("Azure Billing Parser does not have expected DateFormat index. Expected: %s, Actual: %s", tc.expected.DateFormat, abp.DateFormat)
+			}
+
+			if abp.Date != tc.expected.Date {
+				t.Errorf("Azure Billing Parser does not have expected Date index. Expected: %d, Actual: %d", tc.expected.Date, abp.Date)
+			}
+
+			if abp.MeterCategory != tc.expected.MeterCategory {
+				t.Errorf("Azure Billing Parser does not have expected MeterCategory index. Expected: %d, Actual: %d", tc.expected.MeterCategory, abp.MeterCategory)
+			}
+
+			if abp.InvoiceEntityID != tc.expected.InvoiceEntityID {
+				t.Errorf("Azure Billing Parser does not have expected InvoiceEntityID index. Expected: %d, Actual: %d", tc.expected.InvoiceEntityID, abp.InvoiceEntityID)
+			}
+
+			if abp.SubscriptionID != tc.expected.SubscriptionID {
+				t.Errorf("Azure Billing Parser does not have expected SubscriptionID index. Expected: %d, Actual: %d", tc.expected.SubscriptionID, abp.SubscriptionID)
+			}
+
+			if abp.InstanceID != tc.expected.InstanceID {
+				t.Errorf("Azure Billing Parser does not have expected InstanceID index. Expected: %d, Actual: %d", tc.expected.InstanceID, abp.InstanceID)
+			}
+
+			if abp.Service != tc.expected.Service {
+				t.Errorf("Azure Billing Parser does not have expected Service index. Expected: %d, Actual: %d", tc.expected.Service, abp.Service)
+			}
+
+			if abp.Tags != tc.expected.Tags {
+				t.Errorf("Azure Billing Parser does not have expected Tags index. Expected: %d, Actual: %d", tc.expected.Tags, abp.Tags)
+			}
+
+			if abp.AdditionalInfo != tc.expected.AdditionalInfo {
+				t.Errorf("Azure Billing Parser does not have expected AdditionalInfo index. Expected: %d, Actual: %d", tc.expected.AdditionalInfo, abp.AdditionalInfo)
+			}
+
+			if abp.Cost != tc.expected.Cost {
+				t.Errorf("Azure Billing Parser does not have expected Cost index. Expected: %d, Actual: %d", tc.expected.Cost, abp.Cost)
+			}
+
+			if abp.NetCost != tc.expected.NetCost {
+				t.Errorf("Azure Billing Parser does not have expected NetCost index. Expected: %d, Actual: %d", tc.expected.NetCost, abp.NetCost)
+			}
+		})
+	}
+}

pkg/cloud/azure/azureprovider.go → pkg/cloud/azure/provider.go

@@ -489,6 +489,7 @@ func (k *azureKey) GetGPUCount() string {
 }
 
 // AzureStorageConfig Represents an azure storage config
+// Deprecated: v1.104 Use StorageConfiguration instead
 type AzureStorageConfig struct {
 	SubscriptionId string `json:"azureSubscriptionID"`
 	AccountName    string `json:"azureStorageAccount"`
@@ -517,7 +518,8 @@ type AzureAppKey struct {
 	Tenant      string `json:"tenant"`
 }
 
-// Azure service key for a specific subscription
+// AzureServiceKey service key for a specific subscription
+// Deprecated: v1.104 Use ServiceKey instead
 type AzureServiceKey struct {
 	SubscriptionID string       `json:"subscriptionId"`
 	ServiceKey     *AzureAppKey `json:"serviceKey"`

pkg/cloud/azure/resources/billingexports/headersets/BOM.csv

@@ -0,0 +1,2 @@
+SubscriptionGuid,ResourceGroup,ResourceLocation,UsageDateTime,MeterCategory,MeterSubcategory,MeterId,MeterName,MeterRegion,UsageQuantity,ResourceRate,PreTaxCost,ConsumedService,ResourceType,InstanceId,Tags,OfferId,AdditionalInfo,ServiceInfo1,ServiceInfo2,ServiceName,ServiceTier,Currency,UnitOfMeasure
+,,,2022-11-03,,,,,,,,,,,,,,,,,,,,

pkg/cloud/azure/resources/billingexports/headersets/Enterprise.csv

@@ -0,0 +1,2 @@
+InvoiceSectionName,AccountName,AccountOwnerId,SubscriptionId,SubscriptionName,ResourceGroup,ResourceLocation,Date,ProductName,MeterCategory,MeterSubCategory,MeterId,MeterName,MeterRegion,UnitOfMeasure,Quantity,EffectivePrice,CostInBillingCurrency,CostCenter,ConsumedService,ResourceId,Tags,OfferId,AdditionalInfo,ServiceInfo1,ServiceInfo2,ResourceName,ReservationId,ReservationName,UnitPrice,ProductOrderId,ProductOrderName,Term,PublisherType,PublisherName,ChargeType,Frequency,PricingModel,AvailabilityZone,BillingAccountId,BillingAccountName,BillingCurrencyCode,BillingPeriodStartDate,BillingPeriodEndDate,BillingProfileId,BillingProfileName,InvoiceSectionId,IsAzureCreditEligible,PartNumber,PayGPrice,PlanName,ServiceFamily,CostAllocationRuleName
+Unassigned,Azure Service,email@email.com,11111111-12ab-34dc-56ef-123456abcdef,Example-Subscription,Example-Resource-Group,canadacentral,02/02/2021,Virtual Machines Ev3/ESv3 Series - E4 v3/E4s v3 - CA Central,Virtual Machines,Ev3/ESv3 Series,3dbc3a0c-32b6-4c4d-adbb-3ee577aaba4d,E4 v3/E4s v3,CA Central,10 Hours,10,1.2,0,,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-defaultpool-12345678-vmss,"""createOperationID"": ""11111111-12ab-34dc-56ef-123456abcdef"",""creationSource"": ""vmssclient-aks-defaultpool-12345678-vmss"",""orchestrator"": ""Kubernetes:1.19.9"",""poolName"": ""defaultpool"",""resourceNameSuffix"": ""12345678""",MS-AZR-0017P,"{""UsageType"":""ComputeHR"",""ImageType"":""Canonical"",""ServiceType"":""Standard_E4s_v3"",""VMName"":""aks-defaultpool-12345678-vmss_2"",""VMProperties"":null,""VCPUs"":4,""CPUs"":0,""ReservationOrderId"":""11111111-12ab-34dc-56ef-123456abcdef"",""ReservationId"":""4f18e7c9-9ae8-4251-886b-8bd942a41bdf"",""ConsumptionMeter"":""11111111-12ab-34dc-56ef-123456abcdef"",""RINormalizationRatio"":2.0}",,Canonical,aks-defaultpool-12345678-vmss,11111111-12ab-34dc-56ef-123456abcdef,ExampleReservationName,0.1,b13f2808-a13e-49a3-a899-06d83b8f5d32,"Reserved VM Instance, Standard_E2s_v3, CA Central, 3 Years",36,Azure,,Usage,UsageBased,,,12345678,Example Company,CAD,05/01/2021,05/31/2021,12345678,Example Company,,TRUE,ABC-12345,0,,Compute,

pkg/cloud/azure/resources/billingexports/headersets/EnterpriseCamel.csv

@@ -0,0 +1,2 @@
+billingAccountName,partnerName,resellerName,resellerMpnId,customerTenantId,customerName,costCenter,billingPeriodEndDate,billingPeriodStartDate,servicePeriodEndDate,servicePeriodStartDate,date,serviceFamily,productOrderId,productOrderName,consumedService,meterId,meterName,meterCategory,meterSubCategory,meterRegion,ProductId,ProductName,SubscriptionId,subscriptionName,publisherType,publisherId,publisherName,resourceGroupName,ResourceId,resourceLocation,location,effectivePrice,quantity,unitOfMeasure,chargeType,billingCurrency,pricingCurrency,costInBillingCurrency,costInUsd,exchangeRatePricingToBilling,exchangeRateDate,serviceInfo1,serviceInfo2,additionalInfo,tags,PayGPrice,frequency,term,reservationId,reservationName,pricingModel
+,PartnerName,,,11111111-1111-1111-1111-123456789012,Customer Name,,,,02/01/2021,02/01/2021,02/02/2021,Networking,11111111-1111-1111-1111-123456789012,Azure plan,Microsoft.Network,11111111-1111-1111-1111-123456789012,Dynamic Public IP,Virtual Network,IP Addresses,,DZH318Z0BNXN0032,IP Addresses - Basic,11111111-1111-1111-1111-123456789012,Microsoft Azure,Azure,,Microsoft,databricks,/subscriptions/11111111-1111-1111-1111-123456789012/resourceGroups/testspot/providers/Microsoft.Storage/storageAccounts/storename,WESTUS,US West,0.004,3,1 Hour,Usage,USD,USD,0.012,0.012,1,3/1/21,,,,"{  ""ClusterId"": ""0103-212455-stash756"",  ""ServiceType"": ""DataAnalysis"",  ""ClusterName"": ""SrgExtractsPartDeux"",  ""databricks-instance-name"": ""0c1ef59764casdf0c0e094e1cc"",  ""Creator"": ""email@email.com"",  ""Vendor"": ""Databricks"",  ""DatabricksEnvironment"": ""workerenv-6448504491843616""}",0.004,UsageBased,,,,

pkg/cloud/azure/resources/billingexports/headersets/German.csv

@@ -0,0 +1,2 @@
+Abonnement-GUID (SubscriptionGuid),Ressourcengruppe (ResourceGroup),Ressourcenstandort (ResourceLocation),UsageDateTime (UsageDateTime),Kategorie der Verbrauchseinheit (MeterCategory),MeterSubcategory (MeterSubcategory),ID der Verbrauchseinheit (MeterId),Name der Verbrauchseinheit (MeterName),Region der Verbrauchseinheit (MeterRegion),UsageQuantity (UsageQuantity),Ressourcensatz (ResourceRate),PreTaxCost (PreTaxCost),Genutzter Dienst (ConsumedService),ResourceType (ResourceType),InstanceId (InstanceId),Tags (Tags),OfferId (OfferId),Zusätzliche Informationen (AdditionalInfo),Dienstinformation 1 (ServiceInfo1),Dienstinformation 2 (ServiceInfo2),ServiceName,ServiceTier,Currency,Maßeinheit (UnitOfMeasure)
+11111111-12ab-34dc-56ef-123456abcdef,Example-Resource-Group,US East,2021-02-02,Load Balancer,Standard,27827eb0-7f60-4928-940b-f5fe15e7a4cb,Included LB Rules and Outbound Rules,,3,0.025,0.075,Microsoft.Network,Microsoft.Network/loadBalancers,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/loadBalancers/kubernetes,,,,,,Load Balancer,Std Load Balancer,USD,100 Hours

pkg/cloud/azure/resources/billingexports/headersets/PayAsYouGo.csv

@@ -0,0 +1,2 @@
+SubscriptionGuid,ResourceGroup,ResourceLocation,UsageDateTime,MeterCategory,MeterSubcategory,MeterId,MeterName,MeterRegion,UsageQuantity,ResourceRate,PreTaxCost,ConsumedService,ResourceType,InstanceId,Tags,OfferId,AdditionalInfo,ServiceInfo1,ServiceInfo2,ServiceName,ServiceTier,Currency,UnitOfMeasure
+11111111-12ab-34dc-56ef-123456abcdef,Example-Resource-Group,US East,2021-02-02,Load Balancer,Standard,27827eb0-7f60-4928-940b-f5fe15e7a4cb,Included LB Rules and Outbound Rules,,3,0.025,0.075,Microsoft.Network,Microsoft.Network/loadBalancers,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/loadBalancers/kubernetes,,,,,,Load Balancer,Std Load Balancer,USD,100 Hours

pkg/cloud/azure/resources/billingexports/headersets/YA.csv

@@ -0,0 +1,2 @@
+subscriptionId,Ressourcengruppe (ResourceGroup),Ressourcenstandort (ResourceLocation),date,meterCategory,MeterSubcategory (MeterSubcategory),ID der Verbrauchseinheit (MeterId),Name der Verbrauchseinheit (MeterName),Region der Verbrauchseinheit (MeterRegion),UsageQuantity (UsageQuantity),Ressourcensatz (ResourceRate),costInBillingCurrency,consumedService,ResourceType (ResourceType),InstanceName,tags,OfferId (OfferId),additionalInfo,Dienstinformation 1 (ServiceInfo1),Dienstinformation 2 (ServiceInfo2),ServiceName,ServiceTier,Currency,Maßeinheit (UnitOfMeasure)
+11111111-12ab-34dc-56ef-123456abcdef,Example-Resource-Group,US East,02/02/2021,Load Balancer,Standard,27827eb0-7f60-4928-940b-f5fe15e7a4cb,Included LB Rules and Outbound Rules,,3,0.025,0.075,Microsoft.Network,Microsoft.Network/loadBalancers,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/loadBalancers/kubernetes,,,,,,Load Balancer,Std Load Balancer,USD,100 Hours

pkg/cloud/azure/resources/billingexports/values/MissingBrackets.csv

@@ -0,0 +1,2 @@
+subscriptionid,billingaccountid,UsageDateTime,MeterCategory,costinbillingcurrency,paygcostinbillingcurrency,ConsumedService,InstanceId,Tags,AdditionalInfo
+11111111-12ab-34dc-56ef-123456abcdef,11111111-12ab-34dc-56ef-123456abcdef,2021-02-01,Virtual Machines,4,5,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss""","""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": ""aks-nodepool1-12345678-vmss_0"",  ""VCPUs"": 2"

pkg/cloud/azure/resources/billingexports/values/Template.csv

@@ -0,0 +1,88 @@
+subscriptionid,billingaccountid,UsageDateTime,MeterCategory,costinbillingcurrency,paygcostinbillingcurrency,ConsumedService,InstanceId,Tags,AdditionalInfo
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Load Balancer,0.075,0.075,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/loadBalancers/kubernetes,,
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Virtual Machines,3.504,3.504,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"{""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss"",""orchestrator"":""Kubernetes:1.15.7"",""poolName"":""nodepool1""}","{  ""UsageType"": ""ComputeHR"",  ""ImageType"": ""Canonical"",  ""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": ""aks-nodepool1-12345678-vmss_0"",  ""VMProperties"": ""Microsoft.AKS.Compute.AKS.Linux.Billing"",  ""VCPUs"": 2,  ""CPUs"": 0}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.0000045,0.0000045,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd03,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-pushgateway"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd03"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.0013392,0.0013392,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd01,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-alertmanager"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd01"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Virtual Machines,0,0,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachines/aks-nodepool1-34567890-0,"{""resourceNameSuffix"":""34567890"",""aksEngineVersion"":""v0.47.0-aks-gomod-55-aks"",""creationSource"":""aks-aks-nodepool1-34567890-0"",""orchestrator"":""Kubernetes:1.12.8"",""poolName"":""nodepool1""}","{  ""UsageType"": ""ComputeHR"",  ""ImageType"": ""Canonical"",  ""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": null,  ""VMProperties"": ""Microsoft.AKS.Compute.AKS.Linux.Billing"",  ""VCPUs"": 2,  ""CPUs"": 0,  ""ReservationOrderId"": ""689aadb1-13ea-40bb-a8f9-e705dbe57543"",  ""ReservationId"": ""770228a7-62da-4155-802b-0422e1c62efc"",  ""ConsumptionMeter"": ""14fc9a21-4919-4cb1-b495-5666966556bc""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Log Analytics,0,0,microsoft.operationalinsights,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourcegroups/defaultresourcegroup-eus/providers/microsoft.operationalinsights/workspaces/defaultworkspace-11111111-12ab-34dc-56ef-123456abcdef-eus,,
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.0000045,0.0000045,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd02,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-server"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd02"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Virtual Machines,0.146,0.146,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-agentpool-23456789-vmss,"{""resourceNameSuffix"":""23456789"",""aksEngineVersion"":""v0.47.0-aks-gomod-81-aks"",""creationSource"":""aks-aks-agentpool-23456789-vmss"",""orchestrator"":""Kubernetes:1.16.10"",""poolName"":""agentpool""}","{  ""UsageType"": ""ComputeHR"",  ""ImageType"": ""Canonical"",  ""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": ""aks-agentpool-23456789-vmss_0"",  ""VMProperties"": ""Microsoft.AKS.Compute.AKS.Linux.Billing"",  ""VCPUs"": 2,  ""CPUs"": 0}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Log Analytics,0,0,microsoft.operationalinsights,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourcegroups/defaultresourcegroup-eus/providers/microsoft.operationalinsights/workspaces/defaultworkspace-11111111-12ab-34dc-56ef-123456abcdef-eus,,
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.00003615,0.00003615,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd05,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-alertmanager"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd05"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.052568064,0.052568064,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd08,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-cost-analyzer"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd08"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.08798544,0.08798544,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/aks-nodepool1-192133aks-nodepool1-1921336OS__1_0a5e4b97e5ca4c2ab46328ca392a02f5,"{""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss"",""orchestrator"":""Kubernetes:1.15.7"",""poolName"":""nodepool1""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Load Balancer,0.001301934407093,0.001301934407093,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/loadBalancers/kubernetes,,
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Virtual Network,0.0828,0.0828,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/publicIPAddresses/kubernetes-aef001b536d4711ea86115a2af700dc9,"{""service"":""kubecost/kubecost-frontend-test""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Virtual Machines,0.146,0.146,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachines/aks-agentpool-45678901-0,"{""resourceNameSuffix"":""45678901"",""aksEngineVersion"":""v0.35.3-aks"",""creationSource"":""aks-aks-agentpool-45678901-0"",""orchestrator"":""Kubernetes:1.12.8"",""poolName"":""agentpool""}","{  ""UsageType"": ""ComputeHR"",  ""ImageType"": ""Canonical"",  ""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": null,  ""VMProperties"": ""Microsoft.AKS.Compute.AKS.Linux.Billing"",  ""VCPUs"": 2,  ""CPUs"": 0}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Virtual Machines,3.504,3.504,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-agentpool-23456789-vmss,"{""resourceNameSuffix"":""23456789"",""aksEngineVersion"":""v0.47.0-aks-gomod-81-aks"",""creationSource"":""aks-aks-agentpool-23456789-vmss"",""orchestrator"":""Kubernetes:1.16.10"",""poolName"":""agentpool""}","{  ""UsageType"": ""ComputeHR"",  ""ImageType"": ""Canonical"",  ""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": ""aks-agentpool-23456789-vmss_0"",  ""VMProperties"": ""Microsoft.AKS.Compute.AKS.Linux.Billing"",  ""VCPUs"": 2,  ""CPUs"": 0}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.052568064,0.052568064,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd05,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-alertmanager"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd05"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Virtual Network,0.09,0.09,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/publicIPAddresses/kubernetes-a173cf24babf311e98b7f8e5ecb03810,"{""service"":""kubecost/kubecost-frontend""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.006856704,0.006856704,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd07,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-cost-analyzer"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd07"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.0015896,0.0015896,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd00,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-cost-analyzer"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd00"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.052568064,0.052568064,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd03,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-pushgateway"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd03"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.0000362,0.0000362,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd07,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-cost-analyzer"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd07"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Load Balancer,0.01236783717759,0.01236783717759,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/loadBalancers/kubernetes,,
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Bandwidth,0.00000204,0.00000204,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-agentpool-23456789-vmss,"{""resourceNameSuffix"":""23456789"",""aksEngineVersion"":""v0.47.0-aks-gomod-81-aks"",""creationSource"":""aks-aks-agentpool-23456789-vmss"",""orchestrator"":""Kubernetes"",""poolName"":""agentpool""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""CH1"",  ""ContainerId"": ""1c8bb337-451e-487c-ac06-9f83cf69751f"",  ""CRPVMId"": ""2936d707-afda-4ba7-9166-9cac60faba7c""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Bandwidth,0,0,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachines/aks-agentpool-45678901-0,"{""resourceNameSuffix"":""45678901"",""aksEngineVersion"":""v0.35.3-aks"",""creationSource"":""aks-aks-agentpool-45678901-0"",""orchestrator"":""Kubernetes"",""poolName"":""agentpool""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""External"",  ""ContainerId"": ""e5c201c1-7acd-43c3-af5e-3480998c0776"",  ""CRPVMId"": ""0255b3e6-f280-4cb3-9664-ccbe86990e85""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.0000045,0.0000045,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd07,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-cost-analyzer"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd07"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.006856704,0.006856704,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd02,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-server"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd02"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.0102672,0.0102672,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd06,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-cost-analyzer"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd06"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.0821376,0.0821376,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd04,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-server"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd04"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.67455504,0.67455504,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/aks-agentpool-229217aks-agentpool-2292178OS__1_7fcada7aa38e4d5ca6d15257b8998b7a,"{""resourceNameSuffix"":""23456789"",""aksEngineVersion"":""v0.47.0-aks-gomod-81-aks"",""creationSource"":""aks-aks-agentpool-23456789-vmss"",""orchestrator"":""Kubernetes:1.16.10"",""poolName"":""agentpool""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Virtual Network,0.005,0.005,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/publicIPAddresses/bc6b73c3-5689-4f72-9a15-103d0c48d98f,"{""owner"":""kubernetes"",""type"":""aks-slb-managed-outbound-ip""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Bandwidth,0.000000060000000000000000000,0.000000060000000000000000000,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachines/aks-agentpool-45678901-0,"{""resourceNameSuffix"":""45678901"",""aksEngineVersion"":""v0.35.3-aks"",""creationSource"":""aks-aks-agentpool-45678901-0"",""orchestrator"":""Kubernetes"",""poolName"":""agentpool""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""BY1"",  ""ContainerId"": ""e5c201c1-7acd-43c3-af5e-3480998c0776"",  ""CRPVMId"": ""0255b3e6-f280-4cb3-9664-ccbe86990e85""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Bandwidth,0.000000140000000000000000,0.000000140000000000000000,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-agentpool-23456789-vmss,"{""resourceNameSuffix"":""23456789"",""aksEngineVersion"":""v0.47.0-aks-gomod-81-aks"",""creationSource"":""aks-aks-agentpool-23456789-vmss"",""orchestrator"":""Kubernetes"",""poolName"":""agentpool""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""CH1"",  ""ContainerId"": ""1c8bb337-451e-487c-ac06-9f83cf69751f"",  ""CRPVMId"": ""2936d707-afda-4ba7-9166-9cac60faba7c""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Bandwidth,0.000000020000000000000000000,0.000000020000000000000000000,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachines/aks-agentpool-45678901-0,"{""resourceNameSuffix"":""45678901"",""aksEngineVersion"":""v0.35.3-aks"",""creationSource"":""aks-aks-agentpool-45678901-0"",""orchestrator"":""Kubernetes"",""poolName"":""agentpool""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""BY1"",  ""ContainerId"": ""e5c201c1-7acd-43c3-af5e-3480998c0776"",  ""CRPVMId"": ""0255b3e6-f280-4cb3-9664-ccbe86990e85""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.0013522,0.0013522,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd06,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-cost-analyzer"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd06"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Bandwidth,0.000000160000000000000000,0.000000160000000000000000,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-agentpool-23456789-vmss,"{""resourceNameSuffix"":""23456789"",""aksEngineVersion"":""v0.47.0-aks-gomod-81-aks"",""creationSource"":""aks-aks-agentpool-23456789-vmss"",""orchestrator"":""Kubernetes"",""poolName"":""agentpool""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""BY1"",  ""ContainerId"": ""1c8bb337-451e-487c-ac06-9f83cf69751f"",  ""CRPVMId"": ""2936d707-afda-4ba7-9166-9cac60faba7c""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Bandwidth,0.000000100000000000000000,0.000000100000000000000000,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"{""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss"",""orchestrator"":""Kubernetes"",""poolName"":""nodepool1""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""BY1"",  ""ContainerId"": ""ec16b946-8778-49a4-8b9b-283bc90319ed"",  ""CRPVMId"": ""5163cb2c-2a32-4421-ab69-2a75ca69cf16""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Load Balancer,0.001686412831768,0.001686412831768,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/loadBalancers/kubernetes,,
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Virtual Network,0.005,0.005,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/publicIPAddresses/kubernetes-a4969d597c5674b4480ec987cc6b24a1,"{""service"":""kubecost/kubecost-frontend"",""kubernetes-cluster-name"":""kubernetes""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.08798544,0.08798544,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/aks-nodepool1-34567890-0_OsDisk_1_c523fe080d784f55a7cd3868bf989fde,"{""resourceNameSuffix"":""34567890"",""aksEngineVersion"":""v0.47.0-aks-gomod-55-aks"",""creationSource"":""aks-aks-nodepool1-34567890-0"",""orchestrator"":""Kubernetes:1.12.8"",""poolName"":""nodepool1""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Virtual Machines,3.504,3.504,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachines/aks-agentpool-45678901-0,"{""resourceNameSuffix"":""45678901"",""aksEngineVersion"":""v0.35.3-aks"",""creationSource"":""aks-aks-agentpool-45678901-0"",""orchestrator"":""Kubernetes:1.12.8"",""poolName"":""agentpool""}","{  ""UsageType"": ""ComputeHR"",  ""ImageType"": ""Canonical"",  ""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": null,  ""VMProperties"": ""Microsoft.AKS.Compute.AKS.Linux.Billing"",  ""VCPUs"": 2,  ""CPUs"": 0}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Virtual Network,0.125,0.125,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/publicIPAddresses/7b21b77b-4ed1-474b-b068-6ab6d1ecf549,"{""owner"":""kubernetes"",""type"":""aks-slb-managed-outbound-ip""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Bandwidth,0,0,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"{""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss"",""orchestrator"":""Kubernetes"",""poolName"":""nodepool1""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""External"",  ""ContainerId"": ""ec16b946-8778-49a4-8b9b-283bc90319ed"",  ""CRPVMId"": ""5163cb2c-2a32-4421-ab69-2a75ca69cf16""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.006856704,0.006856704,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd03,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-pushgateway"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd03"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.0004494,0.0004494,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd04,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-server"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd04"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Bandwidth,0,0,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachines/aks-agentpool-45678901-0,"{""resourceNameSuffix"":""45678901"",""aksEngineVersion"":""v0.35.3-aks"",""creationSource"":""aks-aks-agentpool-45678901-0"",""orchestrator"":""Kubernetes"",""poolName"":""agentpool""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""External"",  ""ContainerId"": ""e5c201c1-7acd-43c3-af5e-3480998c0776"",  ""CRPVMId"": ""0255b3e6-f280-4cb3-9664-ccbe86990e85""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Bandwidth,0.00000154,0.00000154,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachines/aks-nodepool1-34567890-0,"{""resourceNameSuffix"":""34567890"",""aksEngineVersion"":""v0.47.0-aks-gomod-55-aks"",""creationSource"":""aks-aks-nodepool1-34567890-0"",""orchestrator"":""Kubernetes"",""poolName"":""nodepool1""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""CH1"",  ""ContainerId"": ""ff90fab7-1094-4325-89db-9c12a140131a"",  ""CRPVMId"": ""93b04f9b-4950-42cc-a42e-d72bc852d1e4""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.67455504,0.67455504,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/aks-agentpool-45678901-0_OsDisk_1_6bb726d077d84b238780857a380772ea,"{""resourceNameSuffix"":""45678901"",""aksEngineVersion"":""v0.35.3-aks"",""creationSource"":""aks-aks-agentpool-45678901-0"",""orchestrator"":""Kubernetes:1.12.8"",""poolName"":""agentpool""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Virtual Network,0.15,0.15,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/publicIPAddresses/bc6b73c3-5689-4f72-9a15-103d0c48d98f,"{""owner"":""kubernetes"",""type"":""aks-slb-managed-outbound-ip""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Virtual Machines,0,0,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachines/aks-nodepool1-34567890-0,"{""resourceNameSuffix"":""34567890"",""aksEngineVersion"":""v0.47.0-aks-gomod-55-aks"",""creationSource"":""aks-aks-nodepool1-34567890-0"",""orchestrator"":""Kubernetes:1.12.8"",""poolName"":""nodepool1""}","{  ""UsageType"": ""ComputeHR"",  ""ImageType"": ""Canonical"",  ""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": null,  ""VMProperties"": ""Microsoft.AKS.Compute.AKS.Linux.Billing"",  ""VCPUs"": 2,  ""CPUs"": 0,  ""ReservationOrderId"": ""689aadb1-13ea-40bb-a8f9-e705dbe57543"",  ""ReservationId"": ""770228a7-62da-4155-802b-0422e1c62efc"",  ""ConsumptionMeter"": ""14fc9a21-4919-4cb1-b495-5666966556bc""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Bandwidth,0,0,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"{""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss"",""orchestrator"":""Kubernetes"",""poolName"":""nodepool1""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""External"",  ""ContainerId"": ""ec16b946-8778-49a4-8b9b-283bc90319ed"",  ""CRPVMId"": ""5163cb2c-2a32-4421-ab69-2a75ca69cf16""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Bandwidth,0.000000040000000000000000000,0.000000040000000000000000000,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachines/aks-nodepool1-34567890-0,"{""resourceNameSuffix"":""34567890"",""aksEngineVersion"":""v0.47.0-aks-gomod-55-aks"",""creationSource"":""aks-aks-nodepool1-34567890-0"",""orchestrator"":""Kubernetes"",""poolName"":""nodepool1""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""BY1"",  ""ContainerId"": ""ff90fab7-1094-4325-89db-9c12a140131a"",  ""CRPVMId"": ""93b04f9b-4950-42cc-a42e-d72bc852d1e4""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.0002082,0.0002082,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd00,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-cost-analyzer"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd00"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.0102672,0.0102672,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd01,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-alertmanager"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd01"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.0013392,0.0013392,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd00,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-cost-analyzer"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd00"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.052568064,0.052568064,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd02,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-server"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd02"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.00003615,0.00003615,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd03,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-pushgateway"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd03"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.000177,0.000177,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd06,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-cost-analyzer"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd06"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.67455504,0.67455504,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/aks-nodepool1-192133aks-nodepool1-1921336OS__1_0a5e4b97e5ca4c2ab46328ca392a02f5,"{""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss"",""orchestrator"":""Kubernetes:1.15.7"",""poolName"":""nodepool1""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.0107136,0.0107136,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd04,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-server"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd04"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.0032604,0.0032604,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd04,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-server"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd04"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Bandwidth,0,0,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachines/aks-nodepool1-34567890-0,"{""resourceNameSuffix"":""34567890"",""aksEngineVersion"":""v0.47.0-aks-gomod-55-aks"",""creationSource"":""aks-aks-nodepool1-34567890-0"",""orchestrator"":""Kubernetes"",""poolName"":""nodepool1""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""External"",  ""ContainerId"": ""ff90fab7-1094-4325-89db-9c12a140131a"",  ""CRPVMId"": ""93b04f9b-4950-42cc-a42e-d72bc852d1e4""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.0013392,0.0013392,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd06,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-cost-analyzer"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd06"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Virtual Machines,0.146,0.146,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"{""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss"",""orchestrator"":""Kubernetes:1.15.7"",""poolName"":""nodepool1""}","{  ""UsageType"": ""ComputeHR"",  ""ImageType"": ""Canonical"",  ""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": ""aks-nodepool1-12345678-vmss_0"",  ""VMProperties"": ""Microsoft.AKS.Compute.AKS.Linux.Billing"",  ""VCPUs"": 2,  ""CPUs"": 0}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Virtual Network,0.0072,0.0072,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/publicIPAddresses/kubernetes-aef001b536d4711ea86115a2af700dc9,"{""service"":""kubecost/kubecost-frontend-test""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.00000445,0.00000445,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd05,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-alertmanager"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd05"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Load Balancer,0.575,0.575,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/loadBalancers/kubernetes,,
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Load Balancer,0.00992768780794,0.00992768780794,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/loadBalancers/kubernetes,,
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Bandwidth,0,0,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachines/aks-nodepool1-34567890-0,"{""resourceNameSuffix"":""34567890"",""aksEngineVersion"":""v0.47.0-aks-gomod-55-aks"",""creationSource"":""aks-aks-nodepool1-34567890-0"",""orchestrator"":""Kubernetes"",""poolName"":""nodepool1""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""External"",  ""ContainerId"": ""ff90fab7-1094-4325-89db-9c12a140131a"",  ""CRPVMId"": ""93b04f9b-4950-42cc-a42e-d72bc852d1e4""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.0102672,0.0102672,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd00,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-cost-analyzer"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd00"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Virtual Network,0.14,0.14,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/publicIPAddresses/kubernetes-a4969d597c5674b4480ec987cc6b24a1,"{""service"":""kubecost/kubecost-frontend"",""kubernetes-cluster-name"":""kubernetes""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Bandwidth,0,0,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-agentpool-23456789-vmss,"{""resourceNameSuffix"":""23456789"",""aksEngineVersion"":""v0.47.0-aks-gomod-81-aks"",""creationSource"":""aks-aks-agentpool-23456789-vmss"",""orchestrator"":""Kubernetes"",""poolName"":""agentpool""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""External"",  ""ContainerId"": ""1c8bb337-451e-487c-ac06-9f83cf69751f"",  ""CRPVMId"": ""2936d707-afda-4ba7-9166-9cac60faba7c""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.052568064,0.052568064,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd07,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-cost-analyzer"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd07"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.006856704,0.006856704,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd08,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-cost-analyzer"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd08"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.000191,0.000191,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd01,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-alertmanager"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd01"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.0014714,0.0014714,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd01,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-alertmanager"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd01"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Load Balancer,0.575,0.575,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/loadBalancers/kubernetes,,
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Virtual Network,0.0144,0.0144,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/publicIPAddresses/kubernetes-a173cf24babf311e98b7f8e5ecb03810,"{""service"":""kubecost/kubecost-frontend""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Virtual Network,0.015,0.015,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/publicIPAddresses/7b21b77b-4ed1-474b-b068-6ab6d1ecf549,"{""owner"":""kubernetes"",""type"":""aks-slb-managed-outbound-ip""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Bandwidth,0,0,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-agentpool-23456789-vmss,"{""resourceNameSuffix"":""23456789"",""aksEngineVersion"":""v0.47.0-aks-gomod-81-aks"",""creationSource"":""aks-aks-agentpool-23456789-vmss"",""orchestrator"":""Kubernetes"",""poolName"":""agentpool""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""External"",  ""ContainerId"": ""1c8bb337-451e-487c-ac06-9f83cf69751f"",  ""CRPVMId"": ""2936d707-afda-4ba7-9166-9cac60faba7c""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.67455504,0.67455504,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/aks-nodepool1-34567890-0_OsDisk_1_c523fe080d784f55a7cd3868bf989fde,"{""resourceNameSuffix"":""34567890"",""aksEngineVersion"":""v0.47.0-aks-gomod-55-aks"",""creationSource"":""aks-aks-nodepool1-34567890-0"",""orchestrator"":""Kubernetes:1.12.8"",""poolName"":""nodepool1""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-01,Storage,0.00003615,0.00003615,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd02,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-server"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd02"",""created-by"":""kubernetes-azure-dd""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Bandwidth,0.000000280000000000000000,0.000000280000000000000000,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachines/aks-nodepool1-34567890-0,"{""resourceNameSuffix"":""34567890"",""aksEngineVersion"":""v0.47.0-aks-gomod-55-aks"",""creationSource"":""aks-aks-nodepool1-34567890-0"",""orchestrator"":""Kubernetes"",""poolName"":""nodepool1""}","{  ""ResourceType"": ""Bandwidth"",  ""PipelineType"": ""v2"",  ""DataTransferDirection"": ""DataTrOut"",  ""DataCenter"": ""MNZ20"",  ""NetworkBucket"": ""CH1"",  ""ContainerId"": ""ff90fab7-1094-4325-89db-9c12a140131a"",  ""CRPVMId"": ""93b04f9b-4950-42cc-a42e-d72bc852d1e4""}"
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.08798544,0.08798544,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/aks-agentpool-229217aks-agentpool-2292178OS__1_7fcada7aa38e4d5ca6d15257b8998b7a,"{""resourceNameSuffix"":""23456789"",""aksEngineVersion"":""v0.47.0-aks-gomod-81-aks"",""creationSource"":""aks-aks-agentpool-23456789-vmss"",""orchestrator"":""Kubernetes:1.16.10"",""poolName"":""agentpool""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Load Balancer,0.075,0.075,Microsoft.Network,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Network/loadBalancers/kubernetes,,
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.08798544,0.08798544,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/aks-agentpool-45678901-0_OsDisk_1_6bb726d077d84b238780857a380772ea,"{""resourceNameSuffix"":""45678901"",""aksEngineVersion"":""v0.35.3-aks"",""creationSource"":""aks-aks-agentpool-45678901-0"",""orchestrator"":""Kubernetes:1.12.8"",""poolName"":""agentpool""}",
+11111111-12ab-34dc-56ef-123456abcdef,0bd50fdf-c923-4e1e-850c-196dd3dcc123,2021-02-02,Storage,0.006856704,0.006856704,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/disks/kubernetes-dynamic-pvc-1234abcd-ab12-cd34-ef56-123456abcd05,"{""kubernetes.io-created-for-pvc-namespace"":""kubecost"",""kubernetes.io-created-for-pvc-name"":""kubecost-prometheus-alertmanager"",""kubernetes.io-created-for-pv-name"":""pvc-1234abcd-ab12-cd34-ef56-123456abcd05"",""created-by"":""kubernetes-azure-dd""}",

pkg/cloud/azure/resources/billingexports/values/VirtualMachine.csv

@@ -0,0 +1,2 @@
+subscriptionid,billingaccountid,UsageDateTime,MeterCategory,costinbillingcurrency,paygcostinbillingcurrency,ConsumedService,InstanceId,Tags,AdditionalInfo
+11111111-12ab-34dc-56ef-123456abcdef,11111111-12ab-34dc-56ef-123456billing,2021-02-01,Virtual Machines,4,5,Microsoft.Compute,/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss,"{""resourceNameSuffix"":""12345678"",""aksEngineVersion"":""aks-release-v0.47.0-1-aks"",""creationSource"":""aks-aks-nodepool1-12345678-vmss""}","{ ""ServiceType"": ""Standard_DS2_v2"",  ""VMName"": ""aks-nodepool1-12345678-vmss_0"",  ""VCPUs"": 2  }"

pkg/cloud/azure/storagebillingparser.go

@@ -0,0 +1,170 @@
+package azure
+
+import (
+	"bytes"
+	"context"
+	"encoding/csv"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/opencost/opencost/pkg/cloud"
+	cloudconfig "github.com/opencost/opencost/pkg/cloud/config"
+	"github.com/opencost/opencost/pkg/log"
+)
+
+// AzureStorageBillingParser accesses billing data stored in CSV files in Azure Storage
+type AzureStorageBillingParser struct {
+	StorageConnection
+}
+
+func (asbp *AzureStorageBillingParser) Equals(config cloudconfig.Config) bool {
+	thatConfig, ok := config.(*AzureStorageBillingParser)
+	if !ok {
+		return false
+	}
+	return asbp.StorageConnection.Equals(&thatConfig.StorageConnection)
+}
+
+type AzureBillingResultFunc func(*BillingRowValues) error
+
+func (asbp *AzureStorageBillingParser) ParseBillingData(start, end time.Time, resultFn AzureBillingResultFunc) (cloud.ConnectionStatus, error) {
+	err := asbp.Validate()
+	if err != nil {
+		return cloud.InvalidConfiguration, err
+	}
+
+	containerURL, err := asbp.getContainer()
+	if err != nil {
+		return cloud.FailedConnection, err
+	}
+	ctx := context.Background()
+	blobNames, err := asbp.getMostRecentBlobs(start, end, containerURL, ctx)
+	if err != nil {
+		return cloud.FailedConnection, err
+	}
+	for _, blobName := range blobNames {
+		blobBytes, err2 := asbp.DownloadBlob(blobName, containerURL, ctx)
+		if err2 != nil {
+			return cloud.FailedConnection, err2
+		}
+		err2 = asbp.parseCSV(start, end, csv.NewReader(bytes.NewReader(blobBytes)), resultFn)
+		if err2 != nil {
+			return cloud.ParseError, err2
+		}
+
+	}
+	return cloud.SuccessfulConnection, nil
+}
+
+func (asbp *AzureStorageBillingParser) parseCSV(start, end time.Time, reader *csv.Reader, resultFn AzureBillingResultFunc) error {
+	headers, err := reader.Read()
+	if err != nil {
+		return err
+	}
+	abp, err := NewBillingParseSchema(headers)
+	if err != nil {
+		return err
+	}
+	for {
+		var record, err = reader.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		abv := abp.ParseRow(start, end, record)
+		if abv == nil {
+			continue
+		}
+
+		err = resultFn(abv)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (asbp *AzureStorageBillingParser) getMostRecentBlobs(start, end time.Time, containerURL *azblob.ContainerURL, ctx context.Context) ([]string, error) {
+	log.Infof("Azure Storage: retrieving most recent reports from: %v - %v", start, end)
+
+	// Get list of month substrings for months contained in the start to end range
+	monthStrs, err := asbp.getMonthStrings(start, end)
+	if err != nil {
+		return nil, err
+	}
+	mostResentBlobs := make(map[string]azblob.BlobItemInternal)
+	for marker := (azblob.Marker{}); marker.NotDone(); {
+		// Get a result segment starting with the blob indicated by the current Marker.
+		listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{})
+		if err != nil {
+			return nil, err
+		}
+
+		// ListBlobs returns the start of the next segment; you MUST use this to get
+		// the next segment (after processing the current result segment).
+		marker = listBlob.NextMarker
+
+		// Using the list of months strings find the most resent blob for each month in the range
+		for _, blobInfo := range listBlob.Segment.BlobItems {
+			for _, month := range monthStrs {
+				if strings.Contains(blobInfo.Name, month) {
+					// If Container Path configuration exists, check if it is in the blobs name
+					if asbp.Path != "" && !strings.Contains(blobInfo.Name, asbp.Path) {
+						continue
+					}
+
+					if prevBlob, ok := mostResentBlobs[month]; ok {
+						if prevBlob.Properties.CreationTime.After(*blobInfo.Properties.CreationTime) {
+							continue
+						}
+					}
+					mostResentBlobs[month] = blobInfo
+				}
+			}
+		}
+	}
+
+	// convert blob names into blob urls and move from map into ordered list of blob names
+	var blobNames []string
+	for _, month := range monthStrs {
+		if blob, ok := mostResentBlobs[month]; ok {
+			blobNames = append(blobNames, blob.Name)
+		}
+	}
+
+	return blobNames, nil
+}
+
+func (asbp *AzureStorageBillingParser) getMonthStrings(start, end time.Time) ([]string, error) {
+	if start.After(end) {
+		return []string{}, fmt.Errorf("start date must be before end date")
+	}
+	if end.After(time.Now()) {
+		end = time.Now()
+	}
+	var monthStrs []string
+	monthStr := asbp.timeToMonthString(start)
+	endStr := asbp.timeToMonthString(end)
+	monthStrs = append(monthStrs, monthStr)
+	currMonth := start.AddDate(0, 0, -start.Day()+1)
+	for monthStr != endStr {
+		currMonth = currMonth.AddDate(0, 1, 0)
+		monthStr = asbp.timeToMonthString(currMonth)
+		monthStrs = append(monthStrs, monthStr)
+	}
+
+	return monthStrs, nil
+}
+
+func (asbp *AzureStorageBillingParser) timeToMonthString(input time.Time) string {
+	format := "20060102"
+	startOfMonth := input.AddDate(0, 0, -input.Day()+1)
+	endOfMonth := input.AddDate(0, 1, -input.Day())
+	return startOfMonth.Format(format) + "-" + endOfMonth.Format(format)
+}

pkg/cloud/azure/storagebillingparser_test.go

@@ -0,0 +1,204 @@
+package azure
+
+import (
+	"testing"
+	"time"
+)
+
+func TestAzureStorageBillingParser_getMonthStrings(t *testing.T) {
+	asbp := AzureStorageBillingParser{}
+	loc, _ := time.LoadLocation("UTC")
+	testCases := map[string]struct {
+		start    time.Time
+		end      time.Time
+		expected []string
+	}{
+		"Single Month": {
+			start: time.Date(2021, 2, 1, 00, 00, 00, 00, loc),
+			end:   time.Date(2021, 2, 3, 00, 00, 00, 00, loc),
+			expected: []string{
+				"20210201-20210228",
+			},
+		},
+		"Two Month": {
+			start: time.Date(2021, 2, 1, 00, 00, 00, 00, loc),
+			end:   time.Date(2021, 3, 3, 00, 00, 00, 00, loc),
+			expected: []string{
+				"20210201-20210228",
+				"20210301-20210331",
+			},
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			months, err := asbp.getMonthStrings(tc.start, tc.end)
+			if err != nil {
+				t.Errorf("Could not retrieve month strings %v", err)
+			}
+
+			if len(months) != len(tc.expected) {
+				t.Errorf("Did not create the expected number of month strings. Expected: %d, Actual: %d", len(tc.expected), len(months))
+			}
+
+			for i, monthStr := range months {
+				if monthStr != tc.expected[i] {
+					t.Errorf("Incorrect month string at index %d. Expected: %s, Actual: %s", i, tc.expected[i], monthStr)
+				}
+			}
+		})
+	}
+}
+
+func TestAzureStorageBillingParser_parseCSV(t *testing.T) {
+	loc, _ := time.LoadLocation("UTC")
+	start := time.Date(2021, 2, 1, 00, 00, 00, 00, loc)
+	end := time.Date(2021, 2, 3, 00, 00, 00, 00, loc)
+	tests := map[string]struct {
+		input    string
+		expected []BillingRowValues
+	}{
+		"Virtual Machine": {
+			input: "VirtualMachine.csv",
+			expected: []BillingRowValues{
+				{
+					Date:            start,
+					MeterCategory:   "Virtual Machines",
+					SubscriptionID:  "11111111-12ab-34dc-56ef-123456abcdef",
+					InvoiceEntityID: "11111111-12ab-34dc-56ef-123456billing",
+					InstanceID:      "/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss",
+					Service:         "Microsoft.Compute",
+					Tags: map[string]string{
+						"resourceNameSuffix": "12345678",
+						"aksEngineVersion":   "aks-release-v0.47.0-1-aks",
+						"creationSource":     "aks-aks-nodepool1-12345678-vmss",
+					},
+					AdditionalInfo: map[string]any{
+						"ServiceType": "Standard_DS2_v2",
+						"VMName":      "aks-nodepool1-12345678-vmss_0",
+						"VCPUs":       2.0,
+					},
+					Cost:    5,
+					NetCost: 4,
+				},
+			},
+		},
+		"Missing Brackets": {
+			input: "MissingBrackets.csv",
+			expected: []BillingRowValues{
+				{
+					Date:            start,
+					MeterCategory:   "Virtual Machines",
+					SubscriptionID:  "11111111-12ab-34dc-56ef-123456abcdef",
+					InvoiceEntityID: "11111111-12ab-34dc-56ef-123456abcdef",
+					InstanceID:      "/subscriptions/11111111-12ab-34dc-56ef-123456abcdef/resourceGroups/Example-Resource-Group/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-12345678-vmss",
+					Service:         "Microsoft.Compute",
+					Tags: map[string]string{
+						"resourceNameSuffix": "12345678",
+						"aksEngineVersion":   "aks-release-v0.47.0-1-aks",
+						"creationSource":     "aks-aks-nodepool1-12345678-vmss",
+					},
+					AdditionalInfo: map[string]any{
+						"ServiceType": "Standard_DS2_v2",
+						"VMName":      "aks-nodepool1-12345678-vmss_0",
+						"VCPUs":       2.0,
+					},
+					Cost:    5,
+					NetCost: 4,
+				},
+			},
+		},
+	}
+	asbp := &AzureStorageBillingParser{}
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			csvRetriever := &TestCSVRetriever{
+				CSVName: valueCasesPath + tc.input,
+			}
+			csvs, err := csvRetriever.getCSVReaders(start, end)
+			if err != nil {
+				t.Errorf("Failed to read specified CSV: %s", err.Error())
+			}
+			reader := csvs[0]
+
+			var actual []*BillingRowValues
+			resultFn := func(abv *BillingRowValues) error {
+				actual = append(actual, abv)
+				return nil
+			}
+
+			err = asbp.parseCSV(start, end, reader, resultFn)
+			if err != nil {
+				t.Errorf("Error generating BillingRowValues: %s", err.Error())
+			}
+
+			if len(actual) != len(tc.expected) {
+				t.Errorf("Actual output length did not match expected. Expected: %d, Actual: %d", len(tc.expected), len(actual))
+			}
+
+			for i, this := range actual {
+				that := tc.expected[i]
+
+				if !this.Date.Equal(that.Date) {
+					t.Errorf("Parsed data at index %d has incorrect Date value. Expected: %s, Actual: %s", i, this.Date.String(), that.Date.String())
+				}
+
+				if this.MeterCategory != that.MeterCategory {
+					t.Errorf("Parsed data at index %d has incorrect MeterCategroy value. Expected: %s, Actual: %s", i, this.MeterCategory, that.MeterCategory)
+				}
+
+				if this.SubscriptionID != that.SubscriptionID {
+					t.Errorf("Parsed data at index %d has incorrect SubscriptionID value. Expected: %s, Actual: %s", i, this.SubscriptionID, that.SubscriptionID)
+				}
+
+				if this.InvoiceEntityID != that.InvoiceEntityID {
+					t.Errorf("Parsed data at index %d has incorrect InvoiceEntityID value. Expected: %s, Actual: %s", i, this.InvoiceEntityID, that.InvoiceEntityID)
+				}
+
+				if this.InstanceID != that.InstanceID {
+					t.Errorf("Parsed data at index %d has incorrect InstanceID value. Expected: %s, Actual: %s", i, this.InstanceID, that.InstanceID)
+				}
+
+				if this.Service != that.Service {
+					t.Errorf("Parsed data at index %d has incorrect Service value. Expected: %s, Actual: %s", i, this.Service, that.Service)
+				}
+
+				if this.Cost != that.Cost {
+					t.Errorf("Parsed data at index %d has incorrect Cost value. Expected: %f, Actual: %f", i, this.Cost, that.Cost)
+				}
+
+				if this.NetCost != that.NetCost {
+					t.Errorf("Parsed data at index %d has incorrect NetCost value. Expected: %f, Actual: %f", i, this.NetCost, that.NetCost)
+				}
+
+				if len(this.Tags) != len(that.Tags) {
+					t.Errorf("Parsed data at index %d did not have the expected number of tags. Expected: %d, Actual: %d", i, len(that.Tags), len(this.Tags))
+				}
+
+				for key, thisTag := range this.Tags {
+					thatTag, ok := that.Tags[key]
+					if !ok {
+						t.Errorf("Parsed data at index %d is has unexpected entry in Tags with key: %s", i, key)
+					}
+
+					if thisTag != thatTag {
+						t.Errorf("Parsed data at index %d is has unexpected value in Tags for key: %s. Expected: %s, Actual: %s", i, key, thatTag, thisTag)
+					}
+				}
+
+				for key, thisAI := range this.AdditionalInfo {
+					thatAI, ok := that.AdditionalInfo[key]
+					if !ok {
+						t.Errorf("Parsed data at index %d is has unexpected entry in Additional Inforamation with key: %s", i, key)
+					}
+
+					if thisAI != thatAI {
+						t.Errorf("Parsed data at index %d is has unexpected value in Tags for key: %s. Expected: %v, Actual: %v", i, key, thisAI, thatAI)
+					}
+				}
+			}
+
+		})
+
+	}
+}

pkg/cloud/azure/storageconfiguration.go

@@ -0,0 +1,179 @@
+package azure
+
+import (
+	"fmt"
+
+	"github.com/opencost/opencost/pkg/cloud/config"
+	"github.com/opencost/opencost/pkg/util/json"
+)
+
+type StorageConfiguration struct {
+	SubscriptionID string     `json:"subscriptionID"`
+	Account        string     `json:"account"`
+	Container      string     `json:"container"`
+	Path           string     `json:"path"`
+	Cloud          string     `json:"cloud"`
+	Authorizer     Authorizer `json:"authorizer"`
+}
+
+// Check ensures that all required fields are set, and throws an error if they are not
+func (sc *StorageConfiguration) Validate() error {
+
+	if sc.Authorizer == nil {
+		return fmt.Errorf("StorageConfiguration: missing authorizer")
+	}
+
+	err := sc.Authorizer.Validate()
+	if err != nil {
+		return err
+	}
+
+	if sc.SubscriptionID == "" {
+		return fmt.Errorf("StorageConfiguration: missing Subcription ID")
+	}
+
+	if sc.Account == "" {
+		return fmt.Errorf("StorageConfiguration: missing Account")
+	}
+
+	if sc.Container == "" {
+		return fmt.Errorf("StorageConfiguration: missing Container")
+	}
+
+	return nil
+}
+
+func (sc *StorageConfiguration) Equals(config config.Config) bool {
+	if config == nil {
+		return false
+	}
+	thatConfig, ok := config.(*StorageConfiguration)
+	if !ok {
+		return false
+	}
+
+	if sc.Authorizer != nil {
+		if !sc.Authorizer.Equals(thatConfig.Authorizer) {
+			return false
+		}
+	} else {
+		if thatConfig.Authorizer != nil {
+			return false
+		}
+	}
+
+	if sc.SubscriptionID != thatConfig.SubscriptionID {
+		return false
+	}
+
+	if sc.Account != thatConfig.Account {
+		return false
+	}
+
+	if sc.Container != thatConfig.Container {
+		return false
+	}
+
+	if sc.Path != thatConfig.Path {
+		return false
+	}
+
+	if sc.Cloud != thatConfig.Cloud {
+		return false
+	}
+
+	return true
+}
+
+func (sc *StorageConfiguration) Sanitize() config.Config {
+	return &StorageConfiguration{
+		SubscriptionID: sc.SubscriptionID,
+		Account:        sc.Account,
+		Container:      sc.Container,
+		Path:           sc.Path,
+		Cloud:          sc.Cloud,
+		Authorizer:     sc.Authorizer.Sanitize().(Authorizer),
+	}
+}
+
+func (sc *StorageConfiguration) Key() string {
+	key := fmt.Sprintf("%s/%s", sc.SubscriptionID, sc.Container)
+	// append container path to key if it exists
+	if sc.Path != "" {
+		key = fmt.Sprintf("%s/%s", key, sc.Path)
+	}
+	return key
+}
+
+func (sc *StorageConfiguration) UnmarshalJSON(b []byte) error {
+	var f interface{}
+	err := json.Unmarshal(b, &f)
+	if err != nil {
+		return err
+	}
+
+	fmap := f.(map[string]interface{})
+
+	subscriptionID, err := config.GetInterfaceValue[string](fmap, "subscriptionID")
+	if err != nil {
+		return fmt.Errorf("StorageConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	sc.SubscriptionID = subscriptionID
+
+	account, err := config.GetInterfaceValue[string](fmap, "account")
+	if err != nil {
+		return fmt.Errorf("StorageConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	sc.Account = account
+
+	container, err := config.GetInterfaceValue[string](fmap, "container")
+	if err != nil {
+		return fmt.Errorf("StorageConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	sc.Container = container
+
+	path, err := config.GetInterfaceValue[string](fmap, "path")
+	if err != nil {
+		return fmt.Errorf("StorageConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	sc.Path = path
+
+	cloud, err := config.GetInterfaceValue[string](fmap, "cloud")
+	if err != nil {
+		return fmt.Errorf("StorageConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	sc.Cloud = cloud
+
+	authAny, ok := fmap["authorizer"]
+	if !ok {
+		return fmt.Errorf("StorageConfiguration: UnmarshalJSON: missing authorizer")
+	}
+	authorizer, err := config.AuthorizerFromInterface(authAny, SelectAuthorizerByType)
+	if err != nil {
+		return fmt.Errorf("StorageConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	sc.Authorizer = authorizer
+
+	return nil
+}
+
+func ConvertAzureStorageConfigToConfig(asc AzureStorageConfig) config.KeyedConfig {
+	if asc.IsEmpty() {
+		return nil
+	}
+
+	var authorizer Authorizer
+	authorizer = &AccessKey{
+		AccessKey: asc.AccessKey,
+		Account:   asc.AccountName,
+	}
+
+	return &StorageConfiguration{
+		SubscriptionID: asc.SubscriptionId,
+		Account:        asc.AccountName,
+		Container:      asc.ContainerName,
+		Path:           asc.ContainerPath,
+		Cloud:          asc.AzureCloud,
+		Authorizer:     authorizer,
+	}
+}

pkg/cloud/azure/storageconfiguration_test.go

@@ -0,0 +1,446 @@
+package azure
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/opencost/opencost/pkg/cloud/config"
+	"github.com/opencost/opencost/pkg/log"
+	"github.com/opencost/opencost/pkg/util/json"
+)
+
+func TestStorageConfiguration_Validate(t *testing.T) {
+	testCases := map[string]struct {
+		config   StorageConfiguration
+		expected error
+	}{
+		"valid config Azure AccessKey": {
+			config: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			expected: nil,
+		},
+		"access key invalid": {
+			config: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					Account: "account",
+				},
+			},
+			expected: fmt.Errorf("AccessKey: missing access key"),
+		},
+		"missing authorizer": {
+			config: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer:     nil,
+			},
+			expected: fmt.Errorf("StorageConfiguration: missing authorizer"),
+		},
+		"missing subscriptionID": {
+			config: StorageConfiguration{
+				SubscriptionID: "",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			expected: fmt.Errorf("StorageConfiguration: missing Subcription ID"),
+		},
+		"missing account": {
+			config: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			expected: fmt.Errorf("StorageConfiguration: missing Account"),
+		},
+		"missing container": {
+			config: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			expected: fmt.Errorf("StorageConfiguration: missing Container"),
+		},
+		"missing path": {
+			config: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			expected: nil,
+		},
+		"missing cloud": {
+			config: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			expected: nil,
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual := testCase.config.Validate()
+			actualString := "nil"
+			if actual != nil {
+				actualString = actual.Error()
+			}
+			expectedString := "nil"
+			if testCase.expected != nil {
+				expectedString = testCase.expected.Error()
+			}
+			if actualString != expectedString {
+				t.Errorf("errors do not match: Actual: '%s', Expected: '%s", actualString, expectedString)
+			}
+		})
+	}
+}
+
+func TestStorageConfiguration_Equals(t *testing.T) {
+	testCases := map[string]struct {
+		left     StorageConfiguration
+		right    config.Config
+		expected bool
+	}{
+		"matching config": {
+			left: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			right: &StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			expected: true,
+		},
+
+		"missing both authorizer": {
+			left: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer:     nil,
+			},
+			right: &StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer:     nil,
+			},
+			expected: true,
+		},
+		"missing left authorizer": {
+			left: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer:     nil,
+			},
+			right: &StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			expected: false,
+		},
+		"missing right authorizer": {
+			left: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			right: &StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer:     nil,
+			},
+			expected: false,
+		},
+		"different subscriptionID": {
+			left: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			right: &StorageConfiguration{
+				SubscriptionID: "subscriptionID2",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			expected: false,
+		},
+		"different account": {
+			left: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			right: &StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account2",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			expected: false,
+		},
+		"different container": {
+			left: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			right: &StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container2",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			expected: false,
+		},
+		"different path": {
+			left: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			right: &StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path2",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			expected: false,
+		},
+		"different cloud": {
+			left: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			right: &StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud2",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			expected: false,
+		},
+		"different config": {
+			left: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+			right: &AccessKey{
+				AccessKey: "accessKey",
+				Account:   "account",
+			},
+			expected: false,
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual := testCase.left.Equals(testCase.right)
+			if actual != testCase.expected {
+				t.Errorf("incorrect result: Actual: '%t', Expected: '%t", actual, testCase.expected)
+			}
+		})
+	}
+}
+
+func TestStorageConfiguration_JSON(t *testing.T) {
+	testCases := map[string]struct {
+		config StorageConfiguration
+	}{
+		"Empty Config": {
+			config: StorageConfiguration{},
+		},
+		"Nil Authorizer": {
+			config: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer:     nil,
+			},
+		},
+		"AccessKey Authorizer": {
+			config: StorageConfiguration{
+				SubscriptionID: "subscriptionID",
+				Account:        "account",
+				Container:      "container",
+				Path:           "path",
+				Cloud:          "cloud",
+				Authorizer: &AccessKey{
+					AccessKey: "accessKey",
+					Account:   "account",
+				},
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			// test JSON Marshalling
+			configJSON, err := json.Marshal(testCase.config)
+			if err != nil {
+				t.Errorf("failed to marshal configuration: %s", err.Error())
+			}
+			log.Info(string(configJSON))
+			unmarshalledConfig := &StorageConfiguration{}
+			err = json.Unmarshal(configJSON, unmarshalledConfig)
+			if err != nil {
+				t.Errorf("failed to unmarshal configuration: %s", err.Error())
+			}
+
+			if !testCase.config.Equals(unmarshalledConfig) {
+				t.Error("config does not equal unmarshalled config")
+			}
+		})
+	}
+}

pkg/cloud/azure/storageconnection.go

@@ -0,0 +1,77 @@
+package azure
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	cloudconfig "github.com/opencost/opencost/pkg/cloud/config"
+	"github.com/opencost/opencost/pkg/log"
+)
+
+// StorageConnection provides access to Azure Storage
+type StorageConnection struct {
+	StorageConfiguration
+}
+
+func (sc *StorageConnection) Equals(config cloudconfig.Config) bool {
+	thatConfig, ok := config.(*StorageConnection)
+	if !ok {
+		return false
+	}
+
+	return sc.StorageConfiguration.Equals(&thatConfig.StorageConfiguration)
+}
+
+func (sc *StorageConnection) getContainer() (*azblob.ContainerURL, error) {
+
+	credential, err := sc.Authorizer.GetBlobCredentials()
+	if err != nil {
+		return nil, err
+	}
+
+	p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
+
+	// From the Azure portal, get your storage account blob service URL endpoint.
+	URL, _ := url.Parse(
+		fmt.Sprintf(sc.getBlobURLTemplate(), sc.Account, sc.Container))
+
+	// Create a ContainerURL object that wraps the container URL and a request
+	// pipeline to make requests.
+	containerURL := azblob.NewContainerURL(*URL, p)
+	return &containerURL, nil
+}
+
+// getBlobURLTemplate returns the correct BlobUrl for whichever Cloud storage account is specified by the AzureCloud configuration
+// defaults to the Public Cloud template
+func (sc *StorageConnection) getBlobURLTemplate() string {
+	// Use gov cloud blob url if gov is detected in AzureCloud
+	if strings.Contains(strings.ToLower(sc.Cloud), "gov") {
+		return "https://%s.blob.core.usgovcloudapi.net/%s"
+	}
+	// default to Public Cloud template
+	return "https://%s.blob.core.windows.net/%s"
+}
+
+func (sc *StorageConnection) DownloadBlob(blobName string, containerURL *azblob.ContainerURL, ctx context.Context) ([]byte, error) {
+	log.Infof("Azure Storage: retrieving blob: %v", blobName)
+
+	blobURL := containerURL.NewBlobURL(blobName)
+	downloadResponse, err := blobURL.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	if err != nil {
+		return nil, err
+	}
+	// NOTE: automatically retries are performed if the connection fails
+	bodyStream := downloadResponse.Body(azblob.RetryReaderOptions{MaxRetryRequests: 20})
+
+	// read the body into a buffer
+	downloadedData := bytes.Buffer{}
+	_, err = downloadedData.ReadFrom(bodyStream)
+	if err != nil {
+		return nil, err
+	}
+	return downloadedData.Bytes(), nil
+}

pkg/cloud/cloudcostintegration.go

@@ -0,0 +1,12 @@
+package cloud
+
+import (
+	"time"
+
+	"github.com/opencost/opencost/pkg/kubecost"
+)
+
+// CloudCostIntegration is an interface for retrieving daily granularity CloudCost data for a given range
+type CloudCostIntegration interface {
+	GetCloudCost(time.Time, time.Time) (*kubecost.CloudCostSetRange, error)
+}

pkg/cloud/config/authorizer.go

@@ -0,0 +1,53 @@
+package config
+
+import (
+	"fmt"
+
+	"github.com/opencost/opencost/pkg/util/json"
+)
+
+// AuthorizerTypeProperty is the property where the id of an Authorizer should be placed in its custom MarshalJSON function
+const AuthorizerTypeProperty = "authorizerType"
+
+type Authorizer interface {
+	Config
+	json.Marshaler
+}
+
+// AuthorizerSelectorFn implementations of this function should be a simple switch
+// and acts as a register for the Authorizer types, returned Authorizer should be empty
+// except for its default type property and will have other values marshalled into it
+type AuthorizerSelectorFn[T Authorizer] func(string) (T, error)
+
+// AuthorizerFromInterface this generic function provides Authorizer unmarshalling for all providers
+func AuthorizerFromInterface[T Authorizer](f any, authSelectFn AuthorizerSelectorFn[T]) (T, error) {
+	var emptyAuth T
+	if f == nil {
+		return emptyAuth, nil
+	}
+	fmap, ok := f.(map[string]interface{})
+	if !ok {
+		return emptyAuth, fmt.Errorf("AuthorizerFromInterface: could not cast interface as map")
+	}
+
+	authType, err := GetInterfaceValue[string](fmap, AuthorizerTypeProperty)
+	if err != nil {
+		return emptyAuth, fmt.Errorf("AuthorizerFromInterface: could not retrieve type property: %w", err)
+	}
+	authorizer, err := authSelectFn(authType)
+	if err != nil {
+		return emptyAuth, fmt.Errorf("AuthorizerFromInterface: %w", err)
+	}
+
+	// convert the interface back to a []Byte so that it can be unmarshalled into the correct type
+	fBin, err := json.Marshal(f)
+	if err != nil {
+		return emptyAuth, fmt.Errorf("AuthorizerFromInterface: could not marshal value %v: %w", f, err)
+	}
+
+	err = json.Unmarshal(fBin, authorizer)
+	if err != nil {
+		return emptyAuth, fmt.Errorf("AuthorizerFromInterface: failed to unmarshal into Authorizer type %T from value %v: %w", authorizer, f, err)
+	}
+	return authorizer, nil
+}

pkg/cloud/config/config.go

@@ -0,0 +1,37 @@
+package config
+
+import (
+	"fmt"
+)
+
+const Redacted = "REDACTED"
+
+// Config allows for nested configurations which encapsulate their functionality to be validated and compared easily
+type Config interface {
+	Validate() error
+	Sanitize() Config
+	Equals(Config) bool
+}
+
+// KeyedConfig is a top level Config which uses its public values as a unique identifier allowing duplicates to be identified
+type KeyedConfig interface {
+	Config
+	Key() string
+}
+
+type KeyedConfigWatcher interface {
+	GetConfigs() []KeyedConfig
+}
+
+func GetInterfaceValue[T any](fmap map[string]interface{}, key string) (T, error) {
+	var value T
+	interfaceValue, ok := fmap[key]
+	if !ok {
+		return value, fmt.Errorf("FromInterface: missing '%s' property", key)
+	}
+	typedValue, ok := interfaceValue.(T)
+	if !ok {
+		return value, fmt.Errorf("GetInterfaceValue: property '%s' had expected type '%T' but did not match", key, value)
+	}
+	return typedValue, nil
+}

pkg/cloud/connectionstatus.go

@@ -0,0 +1,47 @@
+package cloud
+
+// ConnectionStatus communicates the status of a cloud connection in a way that is general enough to apply to each
+// Cloud Provider, but still give actionable information on how to trouble shoot one the four failing statuses.
+type ConnectionStatus string
+
+const (
+	// InitialStatus is the zero value of CloudConnectionStatus and means that cloud connection is untested. Once
+	// CloudConnection Status has been changed in should not return to this value. This status is assigned on creation
+	// to the cloud provider
+	InitialStatus ConnectionStatus = "No Connection"
+
+	// InvalidConfiguration means that Cloud Configuration is missing required values to connect to cloud provider.
+	// This status is assigned during failures in the provider implementation of getCloudConfig()
+	InvalidConfiguration = "Invalid Configuration"
+
+	// FailedConnection means that all required Cloud Configuration values are filled in, but a connection with the
+	// Cloud Provider cannot be established. This is indicative of a typo in one of the Cloud Configuration values or an
+	// issue in how the connection was set up in the Cloud Provider's Console. The assignment of this status varies
+	// between implementations, but should happen if an error is thrown when an interaction with an object from
+	// the Cloud Service Provider's sdk occurs.
+	FailedConnection = "Failed Connection"
+
+	// ParseError indicates an issue with our functions which parse responses
+	ParseError = "Parse Error"
+
+	// MissingData means that the Cloud Integration is properly configured, but the cloud provider is not returning
+	// billing/cost and usage data. This status is indicative of the billing/cost and usage data export of the Cloud Provider
+	// being incorrectly set up or the export being set up in the last 48 hours and not having started populating data yet.
+	// This status is set when a query has been successfully made but the results come back empty. If the cloud provider,
+	// already has a SUCCESSFUL_CONNECTION status then this status should not be set, because this indicates that the specific
+	// query made may have been empty.
+	MissingData = "Data Missing"
+
+	// SuccessfulConnection means that the Cloud Integration is properly configured and returning data. This status is
+	// set on any successful query where data is returned
+	SuccessfulConnection = "Connection Successful"
+)
+
+func (cs ConnectionStatus) String() string {
+	return string(cs)
+}
+
+// EmptyChecker provides an interface for to check if a result is empty which can be useful for setting a MissingData status
+type EmptyChecker interface {
+	IsEmpty() bool
+}

pkg/cloud/gcp/authorizer.go

@@ -0,0 +1,132 @@
+package gcp
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/opencost/opencost/pkg/cloud/config"
+	"google.golang.org/api/option"
+)
+
+const ServiceAccountKeyAuthorizerType = "GCPServiceAccountKey"
+const WorkloadIdentityAuthorizerType = "GCPWorkloadIdentity"
+
+// Authorizer provide a []option.ClientOption which is used in when creating clients in the GCP SDK
+type Authorizer interface {
+	config.Authorizer
+	CreateGCPClientOptions() ([]option.ClientOption, error)
+}
+
+// SelectAuthorizerByType is an implementation of AuthorizerSelectorFn and acts as a register for Authorizer types
+func SelectAuthorizerByType(typeStr string) (Authorizer, error) {
+	switch typeStr {
+	case ServiceAccountKeyAuthorizerType:
+		return &ServiceAccountKey{}, nil
+	case WorkloadIdentityAuthorizerType:
+		return &WorkloadIdentity{}, nil
+	default:
+		return nil, fmt.Errorf("GCP: provider authorizer type '%s' is not valid", typeStr)
+	}
+}
+
+type ServiceAccountKey struct {
+	Key map[string]string `json:"key"`
+}
+
+// MarshalJSON custom json marshalling functions, sets properties as tagged in struct and sets the authorizer type property
+func (gkc *ServiceAccountKey) MarshalJSON() ([]byte, error) {
+	fmap := make(map[string]any, 2)
+	fmap[config.AuthorizerTypeProperty] = ServiceAccountKeyAuthorizerType
+	fmap["key"] = gkc.Key
+	return json.Marshal(fmap)
+}
+
+func (gkc *ServiceAccountKey) Validate() error {
+	if gkc.Key == nil || len(gkc.Key) == 0 {
+		return fmt.Errorf("ServiceAccountKey: missing Key")
+	}
+
+	return nil
+}
+
+func (gkc *ServiceAccountKey) Equals(config config.Config) bool {
+	if config == nil {
+		return false
+	}
+	thatConfig, ok := config.(*ServiceAccountKey)
+	if !ok {
+		return false
+	}
+
+	if len(gkc.Key) != len(thatConfig.Key) {
+		return false
+	}
+
+	for k, v := range gkc.Key {
+		if thatConfig.Key[k] != v {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (gkc *ServiceAccountKey) Sanitize() config.Config {
+	redactedMap := make(map[string]string, len(gkc.Key))
+	for key, _ := range gkc.Key {
+		redactedMap[key] = config.Redacted
+	}
+	return &ServiceAccountKey{
+		Key: redactedMap,
+	}
+}
+
+func (gkc *ServiceAccountKey) CreateGCPClientOptions() ([]option.ClientOption, error) {
+	err := gkc.Validate()
+	if err != nil {
+		return nil, err
+	}
+
+	b, err := json.Marshal(gkc.Key)
+	if err != nil {
+		return nil, fmt.Errorf("Key: failed to marshal Key: %s", err.Error())
+	}
+	clientOption := option.WithCredentialsJSON(b)
+
+	// The creation of the BigQuery Client is where FAILED_CONNECTION CloudConnectionStatus is recorded for GCP
+	return []option.ClientOption{clientOption}, nil
+}
+
+// WorkloadIdentity passes an empty slice of client options which causes the GCP SDK to check for the workload identity in the environment
+type WorkloadIdentity struct{}
+
+// MarshalJSON custom json marshalling functions, sets properties as tagged in struct and sets the authorizer type property
+func (wi *WorkloadIdentity) MarshalJSON() ([]byte, error) {
+	fmap := make(map[string]any, 1)
+	fmap[config.AuthorizerTypeProperty] = WorkloadIdentityAuthorizerType
+	return json.Marshal(fmap)
+}
+
+func (wi *WorkloadIdentity) Validate() error {
+	return nil
+}
+
+func (wi *WorkloadIdentity) Equals(config config.Config) bool {
+	if config == nil {
+		return false
+	}
+	_, ok := config.(*WorkloadIdentity)
+	if !ok {
+		return false
+	}
+
+	return true
+}
+
+func (wi *WorkloadIdentity) Sanitize() config.Config {
+	return &WorkloadIdentity{}
+}
+
+func (wi *WorkloadIdentity) CreateGCPClientOptions() ([]option.ClientOption, error) {
+	return []option.ClientOption{}, nil
+}

pkg/cloud/gcp/bigqueryconfiguration.go

@@ -0,0 +1,172 @@
+package gcp
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"cloud.google.com/go/bigquery"
+	"github.com/opencost/opencost/pkg/cloud/config"
+	"github.com/opencost/opencost/pkg/util/json"
+)
+
+type BigQueryConfiguration struct {
+	ProjectID  string     `json:"projectID"`
+	Dataset    string     `json:"dataset"`
+	Table      string     `json:"table"`
+	Authorizer Authorizer `json:"authorizer"`
+}
+
+func (bqc *BigQueryConfiguration) Validate() error {
+
+	if bqc.Authorizer == nil {
+		return fmt.Errorf("BigQueryConfig: missing configurer")
+	}
+
+	err := bqc.Authorizer.Validate()
+	if err != nil {
+		return fmt.Errorf("BigQueryConfig: issue with GCP Authorizer: %s", err.Error())
+	}
+
+	if bqc.ProjectID == "" {
+		return fmt.Errorf("BigQueryConfig: missing ProjectID")
+	}
+
+	if bqc.Dataset == "" {
+		return fmt.Errorf("BigQueryConfig: missing Dataset")
+	}
+
+	if bqc.Table == "" {
+		return fmt.Errorf("BigQueryConfig: missing Table")
+	}
+
+	return nil
+}
+
+func (bqc *BigQueryConfiguration) Equals(config config.Config) bool {
+	if config == nil {
+		return false
+	}
+	thatConfig, ok := config.(*BigQueryConfiguration)
+	if !ok {
+		return false
+	}
+
+	if bqc.Authorizer != nil {
+		if !bqc.Authorizer.Equals(thatConfig.Authorizer) {
+			return false
+		}
+	} else {
+		if thatConfig.Authorizer != nil {
+			return false
+		}
+	}
+
+	if bqc.ProjectID != thatConfig.ProjectID {
+		return false
+	}
+
+	if bqc.Dataset != thatConfig.Dataset {
+		return false
+	}
+
+	if bqc.Table != thatConfig.Table {
+		return false
+	}
+
+	return true
+}
+
+func (bqc *BigQueryConfiguration) Sanitize() config.Config {
+	return &BigQueryConfiguration{
+		ProjectID:  bqc.ProjectID,
+		Dataset:    bqc.Dataset,
+		Table:      bqc.Table,
+		Authorizer: bqc.Authorizer.Sanitize().(Authorizer),
+	}
+}
+
+// Key uses the Usage Project Id as the Provider Key for GCP
+func (bqc *BigQueryConfiguration) Key() string {
+	return fmt.Sprintf("%s/%s", bqc.ProjectID, bqc.GetBillingDataDataset())
+}
+
+func (bqc *BigQueryConfiguration) GetBillingDataDataset() string {
+	return fmt.Sprintf("%s.%s", bqc.Dataset, bqc.Table)
+}
+
+func (bqc *BigQueryConfiguration) GetBigQueryClient(ctx context.Context) (*bigquery.Client, error) {
+	clientOpts, err := bqc.Authorizer.CreateGCPClientOptions()
+	if err != nil {
+		return nil, err
+	}
+	return bigquery.NewClient(ctx, bqc.ProjectID, clientOpts...)
+}
+
+// UnmarshalJSON assumes data is save as an BigQueryConfigurationDTO
+func (bqc *BigQueryConfiguration) UnmarshalJSON(b []byte) error {
+	var f interface{}
+	err := json.Unmarshal(b, &f)
+	if err != nil {
+		return err
+	}
+
+	fmap := f.(map[string]interface{})
+
+	projectID, err := config.GetInterfaceValue[string](fmap, "projectID")
+	if err != nil {
+		return fmt.Errorf("BigQueryConfiguration: FromInterface: %s", err.Error())
+	}
+	bqc.ProjectID = projectID
+
+	dataset, err := config.GetInterfaceValue[string](fmap, "dataset")
+	if err != nil {
+		return fmt.Errorf("BigQueryConfiguration: FromInterface: %s", err.Error())
+	}
+	bqc.Dataset = dataset
+
+	table, err := config.GetInterfaceValue[string](fmap, "table")
+	if err != nil {
+		return fmt.Errorf("BigQueryConfiguration: FromInterface: %s", err.Error())
+	}
+	bqc.Table = table
+
+	authAny, ok := fmap["authorizer"]
+	if !ok {
+		return fmt.Errorf("StorageConfiguration: UnmarshalJSON: missing authorizer")
+	}
+	authorizer, err := config.AuthorizerFromInterface(authAny, SelectAuthorizerByType)
+	if err != nil {
+		return fmt.Errorf("StorageConfiguration: UnmarshalJSON: %s", err.Error())
+	}
+	bqc.Authorizer = authorizer
+	return nil
+}
+
+func ConvertBigQueryConfigToConfig(bqc BigQueryConfig) config.KeyedConfig {
+	if bqc.IsEmpty() {
+		return nil
+	}
+
+	BillingDataDataset := strings.Split(bqc.BillingDataDataset, ".")
+	dataset := BillingDataDataset[0]
+	var table string
+	if len(BillingDataDataset) > 1 {
+		table = BillingDataDataset[1]
+	}
+
+	bigQueryConfiguration := &BigQueryConfiguration{
+		ProjectID:  bqc.ProjectID,
+		Dataset:    dataset,
+		Table:      table,
+		Authorizer: &WorkloadIdentity{}, // Default to WorkloadIdentity
+	}
+
+	if len(bqc.Key) != 0 {
+		bigQueryConfiguration.Authorizer = &ServiceAccountKey{
+			Key: bqc.Key,
+		}
+	}
+
+	return bigQueryConfiguration
+}

pkg/cloud/gcp/bigqueryconfiguration_test.go

@@ -0,0 +1,388 @@
+package gcp
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/opencost/opencost/pkg/cloud/config"
+	"github.com/opencost/opencost/pkg/log"
+	"github.com/opencost/opencost/pkg/util/json"
+)
+
+func TestBigQueryConfiguration_Validate(t *testing.T) {
+	testCases := map[string]struct {
+		config   BigQueryConfiguration
+		expected error
+	}{
+		"valid config GCP Key": {
+			config: BigQueryConfiguration{
+				ProjectID: "projectID",
+				Dataset:   "dataset",
+				Table:     "table",
+				Authorizer: &ServiceAccountKey{
+					Key: map[string]string{
+						"Key":  "Key",
+						"key1": "key2",
+					},
+				},
+			},
+			expected: nil,
+		},
+		"valid config WorkloadIdentity": {
+			config: BigQueryConfiguration{
+				ProjectID:  "projectID",
+				Dataset:    "dataset",
+				Table:      "table",
+				Authorizer: &WorkloadIdentity{},
+			},
+			expected: nil,
+		},
+		"access Key invalid": {
+			config: BigQueryConfiguration{
+				ProjectID: "projectID",
+				Dataset:   "dataset",
+				Table:     "table",
+				Authorizer: &ServiceAccountKey{
+					Key: nil,
+				},
+			},
+			expected: fmt.Errorf("BigQueryConfig: issue with GCP Authorizer: ServiceAccountKey: missing Key"),
+		},
+		"missing configurer": {
+			config: BigQueryConfiguration{
+				ProjectID:  "projectID",
+				Dataset:    "dataset",
+				Table:      "table",
+				Authorizer: nil,
+			},
+			expected: fmt.Errorf("BigQueryConfig: missing configurer"),
+		},
+		"missing projectID": {
+			config: BigQueryConfiguration{
+				ProjectID: "",
+				Dataset:   "dataset",
+				Table:     "table",
+				Authorizer: &ServiceAccountKey{
+					Key: map[string]string{
+						"Key":  "Key",
+						"key1": "key2",
+					},
+				},
+			},
+			expected: fmt.Errorf("BigQueryConfig: missing ProjectID"),
+		},
+		"missing dataset": {
+			config: BigQueryConfiguration{
+				ProjectID: "projectID",
+				Dataset:   "",
+				Table:     "table",
+				Authorizer: &ServiceAccountKey{
+					Key: map[string]string{
+						"Key":  "Key",
+						"key1": "key2",
+					},
+				},
+			},
+			expected: fmt.Errorf("BigQueryConfig: missing Dataset"),
+		},
+		"missing table": {
+			config: BigQueryConfiguration{
+				ProjectID: "projectID",
+				Dataset:   "dataset",
+				Table:     "",
+				Authorizer: &ServiceAccountKey{
+					Key: map[string]string{
+						"Key":  "Key",
+						"key1": "key2",
+					},
+				},
+			},
+			expected: fmt.Errorf("BigQueryConfig: missing Table"),
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual := testCase.config.Validate()
+			actualString := "nil"
+			if actual != nil {
+				actualString = actual.Error()
+			}
+			expectedString := "nil"
+			if testCase.expected != nil {
+				expectedString = testCase.expected.Error()
+			}
+			if actualString != expectedString {
+				t.Errorf("errors do not match: Actual: '%s', Expected: '%s", actualString, expectedString)
+			}
+		})
+	}
+}
+
+func TestBigQueryConfiguration_Equals(t *testing.T) {
+	testCases := map[string]struct {
+		left     BigQueryConfiguration
+		right    config.Config
+		expected bool
+	}{
+		"matching config": {
+			left: BigQueryConfiguration{
+				ProjectID: "projectID",
+				Dataset:   "dataset",
+				Table:     "table",
+				Authorizer: &ServiceAccountKey{
+					Key: map[string]string{
+						"Key":  "Key",
+						"key1": "key2",
+					},
+				},
+			},
+			right: &BigQueryConfiguration{
+				ProjectID: "projectID",
+				Dataset:   "dataset",
+				Table:     "table",
+				Authorizer: &ServiceAccountKey{
+					Key: map[string]string{
+						"Key":  "Key",
+						"key1": "key2",
+					},
+				},
+			},
+			expected: true,
+		},
+		"different configurer": {
+			left: BigQueryConfiguration{
+				ProjectID: "projectID",
+				Dataset:   "dataset",
+				Table:     "table",
+				Authorizer: &ServiceAccountKey{
+					Key: map[string]string{
+						"Key":  "Key",
+						"key1": "key2",
+					},
+				},
+			},
+			right: &BigQueryConfiguration{
+				ProjectID:  "projectID",
+				Dataset:    "dataset",
+				Table:      "table",
+				Authorizer: &WorkloadIdentity{},
+			},
+			expected: false,
+		},
+		"missing both configurer": {
+			left: BigQueryConfiguration{
+				ProjectID:  "projectID",
+				Dataset:    "dataset",
+				Table:      "table",
+				Authorizer: nil,
+			},
+			right: &BigQueryConfiguration{
+				ProjectID:  "projectID",
+				Dataset:    "dataset",
+				Table:      "table",
+				Authorizer: nil,
+			},
+			expected: true,
+		},
+		"missing left configurer": {
+			left: BigQueryConfiguration{
+				ProjectID:  "projectID",
+				Dataset:    "dataset",
+				Table:      "table",
+				Authorizer: nil,
+			},
+			right: &BigQueryConfiguration{
+				ProjectID:  "projectID",
+				Dataset:    "dataset",
+				Table:      "table",
+				Authorizer: &WorkloadIdentity{},
+			},
+			expected: false,
+		},
+		"missing right configurer": {
+			left: BigQueryConfiguration{
+				ProjectID: "projectID",
+				Dataset:   "dataset",
+				Table:     "table",
+				Authorizer: &ServiceAccountKey{
+					Key: map[string]string{
+						"Key":  "Key",
+						"key1": "key2",
+					},
+				},
+			},
+			right: &BigQueryConfiguration{
+				ProjectID:  "projectID",
+				Dataset:    "dataset",
+				Table:      "table",
+				Authorizer: nil,
+			},
+			expected: false,
+		},
+		"different projectID": {
+			left: BigQueryConfiguration{
+				ProjectID: "projectID",
+				Dataset:   "dataset",
+				Table:     "table",
+				Authorizer: &ServiceAccountKey{
+					Key: map[string]string{
+						"Key":  "Key",
+						"key1": "key2",
+					},
+				},
+			},
+			right: &BigQueryConfiguration{
+				ProjectID: "projectID2",
+				Dataset:   "dataset",
+				Table:     "table",
+				Authorizer: &ServiceAccountKey{
+					Key: map[string]string{
+						"Key":  "Key",
+						"key1": "key2",
+					},
+				},
+			},
+			expected: false,
+		},
+		"different dataset": {
+			left: BigQueryConfiguration{
+				ProjectID: "projectID",
+				Dataset:   "dataset",
+				Table:     "table",
+				Authorizer: &ServiceAccountKey{
+					Key: map[string]string{
+						"Key":  "Key",
+						"key1": "key2",
+					},
+				},
+			},
+			right: &BigQueryConfiguration{
+				ProjectID: "projectID",
+				Dataset:   "dataset2",
+				Table:     "table",
+				Authorizer: &ServiceAccountKey{
+					Key: map[string]string{
+						"Key":  "Key",
+						"key1": "key2",
+					},
+				},
+			},
+			expected: false,
+		},
+		"different table": {
+			left: BigQueryConfiguration{
+				ProjectID: "projectID",
+				Dataset:   "dataset",
+				Table:     "table",
+				Authorizer: &ServiceAccountKey{
+					Key: map[string]string{
+						"Key":  "Key",
+						"key1": "key2",
+					},
+				},
+			},
+			right: &BigQueryConfiguration{
+				ProjectID: "projectID",
+				Dataset:   "dataset",
+				Table:     "table2",
+				Authorizer: &ServiceAccountKey{
+					Key: map[string]string{
+						"Key":  "Key",
+						"key1": "key2",
+					},
+				},
+			},
+			expected: false,
+		},
+		"different config": {
+			left: BigQueryConfiguration{
+				ProjectID: "projectID",
+				Dataset:   "dataset",
+				Table:     "table",
+				Authorizer: &ServiceAccountKey{
+					Key: map[string]string{
+						"Key":  "Key",
+						"key1": "key2",
+					},
+				},
+			},
+			right: &ServiceAccountKey{
+
+				Key: map[string]string{
+					"Key":  "Key",
+					"key1": "key2",
+				},
+			},
+			expected: false,
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual := testCase.left.Equals(testCase.right)
+			if actual != testCase.expected {
+				t.Errorf("incorrect result: Actual: '%t', Expected: '%t", actual, testCase.expected)
+			}
+		})
+	}
+}
+
+func TestBigQueryConfiguration_JSON(t *testing.T) {
+	testCases := map[string]struct {
+		config BigQueryConfiguration
+	}{
+		"Empty Config": {
+			config: BigQueryConfiguration{},
+		},
+		"Nil Authorizer": {
+			config: BigQueryConfiguration{
+				ProjectID:  "projectID",
+				Dataset:    "dataset",
+				Table:      "table",
+				Authorizer: nil,
+			},
+		},
+		"ServiceAccountKeyConfigurer": {
+			config: BigQueryConfiguration{
+				ProjectID: "projectID",
+				Dataset:   "dataset",
+				Table:     "table",
+				Authorizer: &ServiceAccountKey{
+					Key: map[string]string{
+						"Key":  "Key",
+						"key1": "key2",
+					},
+				},
+			},
+		},
+		"WorkLoadIdentityConfigurer": {
+			config: BigQueryConfiguration{
+				ProjectID:  "projectID",
+				Dataset:    "dataset",
+				Table:      "table",
+				Authorizer: &WorkloadIdentity{},
+			},
+		},
+	}
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+
+			// test JSON Marshalling
+			configJSON, err := json.Marshal(testCase.config)
+			if err != nil {
+				t.Errorf("failed to marshal configuration: %s", err.Error())
+			}
+			log.Info(string(configJSON))
+			unmarshalledConfig := &BigQueryConfiguration{}
+			err = json.Unmarshal(configJSON, unmarshalledConfig)
+			if err != nil {
+				t.Errorf("failed to unmarshal configuration: %s", err.Error())
+			}
+			if !testCase.config.Equals(unmarshalledConfig) {
+				t.Error("config does not equal unmarshalled config")
+			}
+		})
+	}
+}

pkg/cloud/gcp/bigqueryintegration.go

@@ -0,0 +1,369 @@
+package gcp
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+
+	"cloud.google.com/go/bigquery"
+	"github.com/opencost/opencost/pkg/kubecost"
+	"github.com/opencost/opencost/pkg/log"
+	"github.com/opencost/opencost/pkg/util/timeutil"
+	"google.golang.org/api/iterator"
+)
+
+type BigQueryIntegration struct {
+	BigQueryQuerier
+}
+
+const (
+	UsageDateColumnName          = "usage_date"
+	BillingAccountIDColumnName   = "billing_id"
+	ProjectIDColumnName          = "project_id"
+	ServiceDescriptionColumnName = "service"
+	SKUDescriptionColumnName     = "description"
+	LabelsColumnName             = "labels"
+	ResourceNameColumnName       = "resource"
+	CostColumnName               = "cost"
+	CreditsColumnName            = "credits"
+)
+
+const BiqQueryWherePartitionFmt = `DATE(_PARTITIONTIME) >= "%s" AND DATE(_PARTITIONTIME) < "%s"`
+const BiqQueryWhereDateFmt = `usage_start_time >= "%s" AND usage_start_time < "%s"`
+
+func (bqi *BigQueryIntegration) GetCloudCost(start time.Time, end time.Time) (*kubecost.CloudCostSetRange, error) {
+	// Build Query
+
+	selectColumns := []string{
+		fmt.Sprintf("TIMESTAMP_TRUNC(usage_start_time, day) as %s", UsageDateColumnName),
+		fmt.Sprintf("billing_account_id as %s", BillingAccountIDColumnName),
+		fmt.Sprintf("project.id as %s", ProjectIDColumnName),
+		fmt.Sprintf("service.description as %s", ServiceDescriptionColumnName),
+		fmt.Sprintf("sku.description as %s", SKUDescriptionColumnName),
+		fmt.Sprintf("resource.name as %s", ResourceNameColumnName),
+		fmt.Sprintf("TO_JSON_STRING(labels) as %s", LabelsColumnName),
+		fmt.Sprintf("SUM(cost) as %s", CostColumnName),
+		fmt.Sprintf("IFNULL(SUM((Select SUM(amount) FROM bd.credits)),0) as %s", CreditsColumnName),
+	}
+
+	groupByColumns := []string{
+		UsageDateColumnName,
+		BillingAccountIDColumnName,
+		ProjectIDColumnName,
+		ServiceDescriptionColumnName,
+		SKUDescriptionColumnName,
+		LabelsColumnName,
+		ResourceNameColumnName,
+	}
+
+	partitionStart := start
+	partitionEnd := end.AddDate(0, 0, 2)
+	wherePartition := fmt.Sprintf(BiqQueryWherePartitionFmt, partitionStart.Format("2006-01-02"), partitionEnd.Format("2006-01-02"))
+	whereDate := fmt.Sprintf(BiqQueryWhereDateFmt, start.Format("2006-01-02"), end.Format("2006-01-02"))
+
+	whereConjuncts := []string{
+		wherePartition,
+		whereDate,
+	}
+
+	columnStr := strings.Join(selectColumns, ", ")
+	table := fmt.Sprintf(" `%s` bd ", bqi.GetBillingDataDataset())
+	whereClause := strings.Join(whereConjuncts, " AND ")
+	groupByStr := strings.Join(groupByColumns, ", ")
+	queryStr := `
+		SELECT %s
+		FROM %s
+		WHERE %s
+		GROUP BY %s
+	`
+
+	querystr := fmt.Sprintf(queryStr, columnStr, table, whereClause, groupByStr)
+
+	// Perform Query and parse values
+
+	ccsr, err := kubecost.NewCloudCostSetRange(start, end, timeutil.Day, bqi.Key())
+	if err != nil {
+		return ccsr, fmt.Errorf("error creating new CloudCostSetRange: %s", err)
+	}
+
+	iter, err := bqi.Query(context.Background(), querystr)
+	if err != nil {
+		return ccsr, fmt.Errorf("error querying: %s", err)
+	}
+
+	// Parse query into CloudCostSetRange
+	for {
+		var ccl CloudCostLoader
+		err = iter.Next(&ccl)
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			return ccsr, err
+		}
+		if ccl.CloudCost == nil {
+			continue
+		}
+		ccsr.LoadCloudCost(ccl.CloudCost)
+
+	}
+	return ccsr, nil
+
+}
+
+type CloudCostLoader struct {
+	CloudCost *kubecost.CloudCost
+}
+
+// Load populates the fields of a CloudCostValues with bigquery.Value from provided slice
+func (ccl *CloudCostLoader) Load(values []bigquery.Value, schema bigquery.Schema) error {
+
+	// Create Cloud Cost Properties
+	properties := kubecost.CloudCostProperties{
+		Provider: kubecost.GCPProvider,
+	}
+	var window kubecost.Window
+	var description string
+	var listCost float64
+	var credits float64
+
+	for i, field := range schema {
+		if field == nil {
+			log.DedupedErrorf(5, "GCP: BigQuery: found nil field in schema")
+			continue
+		}
+
+		switch field.Name {
+		case UsageDateColumnName:
+			usageDate, ok := values[i].(time.Time)
+			if !ok {
+				// It would be very surprising if an unparsable time came back from the API, so it should be ok to return here.
+				return fmt.Errorf("error parsing usage date: %v", values[0])
+			}
+			// start and end will be the day that the usage occurred on
+			s := usageDate
+			e := s.Add(timeutil.Day)
+			window = kubecost.NewWindow(&s, &e)
+		case BillingAccountIDColumnName:
+			invoiceEntityID, ok := values[i].(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", BillingAccountIDColumnName, values[i])
+				invoiceEntityID = ""
+			}
+			properties.InvoiceEntityID = invoiceEntityID
+		case ProjectIDColumnName:
+			accountID, ok := values[i].(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", ProjectIDColumnName, values[i])
+				accountID = ""
+			}
+			properties.AccountID = accountID
+		case ServiceDescriptionColumnName:
+			service, ok := values[i].(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", ServiceDescriptionColumnName, values[i])
+				service = ""
+			}
+			properties.Service = service
+		case SKUDescriptionColumnName:
+			d, ok := values[i].(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", SKUDescriptionColumnName, values[i])
+				d = ""
+			}
+			description = d
+		case LabelsColumnName:
+			labelJSON, ok := values[i].(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", LabelsColumnName, values[i])
+			}
+			labelList := []map[string]string{}
+			err := json.Unmarshal([]byte(labelJSON), &labelList)
+			if err != nil {
+				log.Warnf("GCP Cloud Assets: error unmarshaling GCP CloudCost labels: %s", err)
+			}
+			labels := map[string]string{}
+			for _, pair := range labelList {
+				key := pair["key"]
+				value := pair["value"]
+				labels[key] = value
+			}
+			properties.Labels = labels
+		case ResourceNameColumnName:
+			resouceNameValue := values[i]
+			if resouceNameValue == nil {
+				properties.ProviderID = ""
+				continue
+			}
+			resource, ok := resouceNameValue.(string)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", ResourceNameColumnName, values[i])
+				properties.ProviderID = ""
+				continue
+			}
+
+			properties.ProviderID = ParseProviderID(resource)
+		case CostColumnName:
+			cost, ok := values[i].(float64)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", CostColumnName, values[i])
+				cost = 0.0
+			}
+			listCost = cost
+		case CreditsColumnName:
+			creditSum, ok := values[i].(float64)
+			if !ok {
+				log.DedupedErrorf(5, "error parsing GCP CloudCost %s: %v", CreditsColumnName, values[i])
+				creditSum = 0.0
+			}
+			credits = creditSum
+		default:
+			log.DedupedErrorf(5, "GCP: BigQuery: found unrecognized column name %s", field.Name)
+		}
+	}
+
+	// Check required Fields
+	if window.IsOpen() {
+		return fmt.Errorf("GCP: BigQuery: error parsing, item had invalid window")
+	}
+
+	// Determine Category
+	properties.Category = SelectCategory(properties.Service, description)
+
+	// sum credit and cost for NetCost
+	netCost := listCost + credits
+
+	// Using the NetCost as a 'placeholder' for these costs now, until we can revisit and spend the time to do
+	// the calculations correctly
+	amortizedCost := netCost
+	amortizedNetCost := netCost
+	invoicedCost := netCost
+
+	// percent k8s is determined by the presence of labels
+	k8sPercent := 0.0
+	if IsK8s(properties.Labels) {
+		k8sPercent = 1.0
+	}
+
+	ccl.CloudCost = &kubecost.CloudCost{
+		Properties: &properties,
+		Window:     window,
+		ListCost: kubecost.CostMetric{
+			Cost:              listCost,
+			KubernetesPercent: k8sPercent,
+		},
+		AmortizedCost: kubecost.CostMetric{
+			Cost:              amortizedCost,
+			KubernetesPercent: k8sPercent,
+		},
+		AmortizedNetCost: kubecost.CostMetric{
+			Cost:              amortizedNetCost,
+			KubernetesPercent: k8sPercent,
+		},
+		InvoicedCost: kubecost.CostMetric{
+			Cost:              invoicedCost,
+			KubernetesPercent: k8sPercent,
+		},
+		NetCost: kubecost.CostMetric{
+			Cost:              netCost,
+			KubernetesPercent: k8sPercent,
+		},
+	}
+
+	return nil
+}
+
+func IsK8s(labels map[string]string) bool {
+	if _, ok := labels["goog-gke-volume"]; ok {
+		return true
+	}
+
+	if _, ok := labels["goog-gke-node"]; ok {
+		return true
+	}
+
+	if _, ok := labels["goog-k8s-cluster-name"]; ok {
+		return true
+	}
+
+	return false
+}
+
+var parseProviderIDRx = regexp.MustCompile("^.+\\/(.+)?") // Capture "gke-cluster-3-default-pool-xxxx-yy" from "projects/###/instances/gke-cluster-3-default-pool-xxxx-yy"
+
+func ParseProviderID(id string) string {
+	match := parseProviderIDRx.FindStringSubmatch(id)
+	if len(match) == 0 {
+		return id
+	}
+	return match[len(match)-1]
+}
+
+func SelectCategory(service, description string) string {
+	s := strings.ToLower(service)
+	d := strings.ToLower(description)
+
+	// Network descriptions
+	if strings.Contains(d, "download") {
+		return kubecost.NetworkCategory
+	}
+	if strings.Contains(d, "network") {
+		return kubecost.NetworkCategory
+	}
+	if strings.Contains(d, "ingress") {
+		return kubecost.NetworkCategory
+	}
+	if strings.Contains(d, "egress") {
+		return kubecost.NetworkCategory
+	}
+	if strings.Contains(d, "static ip") {
+		return kubecost.NetworkCategory
+	}
+	if strings.Contains(d, "external ip") {
+		return kubecost.NetworkCategory
+	}
+	if strings.Contains(d, "load balanced") {
+		return kubecost.NetworkCategory
+	}
+	if strings.Contains(d, "licensing fee") {
+		return kubecost.OtherCategory
+	}
+
+	// Storage Descriptions
+	if strings.Contains(d, "storage") {
+		return kubecost.StorageCategory
+	}
+	if strings.Contains(d, "pd capacity") {
+		return kubecost.StorageCategory
+	}
+	if strings.Contains(d, "pd iops") {
+		return kubecost.StorageCategory
+	}
+	if strings.Contains(d, "pd snapshot") {
+		return kubecost.StorageCategory
+	}
+
+	// Service Defaults
+	if strings.Contains(s, "storage") {
+		return kubecost.StorageCategory
+	}
+	if strings.Contains(s, "compute") {
+		return kubecost.ComputeCategory
+	}
+	if strings.Contains(s, "sql") {
+		return kubecost.StorageCategory
+	}
+	if strings.Contains(s, "bigquery") {
+		return kubecost.StorageCategory
+	}
+	if strings.Contains(s, "kubernetes") {
+		return kubecost.ManagementCategory
+	} else if strings.Contains(s, "pub/sub") {
+		return kubecost.NetworkCategory
+	}
+
+	return kubecost.OtherCategory
+}

pkg/cloud/gcp/bigqueryintegration_test.go

@@ -0,0 +1,58 @@
+package gcp
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/opencost/opencost/pkg/kubecost"
+	"github.com/opencost/opencost/pkg/util/timeutil"
+)
+
+func TestBigQueryIntegration_GetCloudCost(t *testing.T) {
+	bigQueryConfigPath := os.Getenv("BIGQUERY_CONFIGURATION")
+	if bigQueryConfigPath == "" {
+		t.Skip("skipping integration test, set environment variable ATHENA_CONFIGURATION")
+	}
+	bigQueryConfigBin, err := os.ReadFile(bigQueryConfigPath)
+	if err != nil {
+		t.Fatalf("failed to read config file: %s", err.Error())
+	}
+	var bigQueryConfig BigQueryConfiguration
+	err = json.Unmarshal(bigQueryConfigBin, &bigQueryConfig)
+	if err != nil {
+		t.Fatalf("failed to unmarshal config from JSON: %s", err.Error())
+	}
+
+	today := kubecost.RoundBack(time.Now().UTC(), timeutil.Day)
+
+	testCases := map[string]struct {
+		integration *BigQueryIntegration
+		start       time.Time
+		end         time.Time
+		expected    bool
+	}{
+
+		"last week window": {
+			integration: &BigQueryIntegration{
+				BigQueryQuerier: BigQueryQuerier{
+					BigQueryConfiguration: bigQueryConfig,
+				},
+			},
+			end:      today.Add(-7 * timeutil.Day),
+			start:    today.Add(-8 * timeutil.Day),
+			expected: false,
+		},
+	}
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			actual, err := testCase.integration.GetCloudCost(testCase.start, testCase.end)
+			if err != nil {
+				t.Errorf("Other error during testing %s", err)
+			} else if actual.IsEmpty() != testCase.expected {
+				t.Errorf("Incorrect result, actual emptiness: %t, expected: %t", actual.IsEmpty(), testCase.expected)
+			}
+		})
+	}
+}

pkg/cloud/gcp/bigqueryquerier.go

@@ -0,0 +1,41 @@
+package gcp
+
+import (
+	"context"
+
+	"cloud.google.com/go/bigquery"
+	"github.com/opencost/opencost/pkg/cloud"
+	cloudconfig "github.com/opencost/opencost/pkg/cloud/config"
+)
+
+type BigQueryQuerier struct {
+	BigQueryConfiguration
+	ConnectionStatus cloud.ConnectionStatus
+}
+
+func (bqq *BigQueryQuerier) Equals(config cloudconfig.Config) bool {
+	thatConfig, ok := config.(*BigQueryQuerier)
+	if !ok {
+		return false
+	}
+
+	return bqq.BigQueryConfiguration.Equals(&thatConfig.BigQueryConfiguration)
+}
+
+func (bqq *BigQueryQuerier) Query(ctx context.Context, queryStr string) (*bigquery.RowIterator, error) {
+	err := bqq.Validate()
+
+	if err != nil {
+		bqq.ConnectionStatus = cloud.InvalidConfiguration
+		return nil, err
+	}
+
+	client, err := bqq.GetBigQueryClient(ctx)
+	if err != nil {
+		bqq.ConnectionStatus = cloud.FailedConnection
+		return nil, err
+	}
+
+	query := client.Query(queryStr)
+	return query.Read(ctx)
+}

pkg/cloud/gcp/gcpprovider.go → pkg/cloud/gcp/provider.go

@@ -139,11 +139,11 @@ func (gcp *GCP) GetLocalStorageQuery(window, offset time.Duration, rate bool, us
 	fmtOffset := timeutil.DurationToPromOffsetString(offset)
 
 	fmtCumulativeQuery := `sum(
-		sum_over_time(%s{device!="tmpfs", id="/"}[%s:1m]%s)
+		sum_over_time(%s{device!="tmpfs", id="/", %s}[%s:1m]%s)
 	) by (%s) / 60 / 730 / 1024 / 1024 / 1024 * %f`
 
 	fmtMonthlyQuery := `sum(
-		avg_over_time(%s{device!="tmpfs", id="/"}[%s:1m]%s)
+		avg_over_time(%s{device!="tmpfs", id="/", %s}[%s:1m]%s)
 	) by (%s) / 1024 / 1024 / 1024 * %f`
 
 	fmtQuery := fmtCumulativeQuery
@@ -152,7 +152,7 @@ func (gcp *GCP) GetLocalStorageQuery(window, offset time.Duration, rate bool, us
 	}
 	fmtWindow := timeutil.DurationString(window)
 
-	return fmt.Sprintf(fmtQuery, baseMetric, fmtWindow, fmtOffset, env.GetPromClusterLabel(), localStorageCost)
+	return fmt.Sprintf(fmtQuery, env.GetPromClusterFilter(), baseMetric, fmtWindow, fmtOffset, env.GetPromClusterLabel(), localStorageCost)
 }
 
 func (gcp *GCP) GetConfig() (*models.CustomPricing, error) {
@@ -176,6 +176,7 @@ func (gcp *GCP) GetConfig() (*models.CustomPricing, error) {
 }
 
 // BigQueryConfig contain the required config and credentials to access OOC resources for GCP
+// Deprecated: v1.104 Use BigQueryConfiguration instead
 type BigQueryConfig struct {
 	ProjectID          string            `json:"projectID"`
 	BillingDataDataset string            `json:"billingDataDataset"`

pkg/cloud/models/models.go

@@ -60,6 +60,7 @@ type Node struct {
 	Reserved         *ReservedInstanceData `json:"reserved,omitempty"`
 	ProviderID       string                `json:"providerID,omitempty"`
 	PricingType      PricingType           `json:"pricingType,omitempty"`
+	ArchType         string                `json:"archType,omitempty"`
 }
 
 // IsSpot determines whether or not a Node uses spot by usage type

pkg/cloud/csvprovider.go → pkg/cloud/provider/csvprovider.go

@@ -1,4 +1,4 @@
-package cloud
+package provider
 
 import (
 	"encoding/csv"
@@ -308,10 +308,10 @@ func NodeValueFromMapField(m string, n *v1.Node, useRegion bool) string {
 		if mf[1] == "name" {
 			return toReturn + n.Name
 		} else if mf[1] == "labels" {
-			lkey := strings.Join(mf[2:], "")
+			lkey := strings.Join(mf[2:len(mf)], ".")
 			return toReturn + n.Labels[lkey]
 		} else if mf[1] == "annotations" {
-			akey := strings.Join(mf[2:], "")
+			akey := strings.Join(mf[2:len(mf)], ".")
 			return toReturn + n.Annotations[akey]
 		} else {
 			log.Errorf("Unsupported InstanceIDField %s in CSV For Node", m)
@@ -329,10 +329,10 @@ func PVValueFromMapField(m string, n *v1.PersistentVolume) string {
 		if mf[1] == "name" {
 			return n.Name
 		} else if mf[1] == "labels" {
-			lkey := strings.Join(mf[2:], "")
+			lkey := strings.Join(mf[2:len(mf)], "")
 			return n.Labels[lkey]
 		} else if mf[1] == "annotations" {
-			akey := strings.Join(mf[2:], "")
+			akey := strings.Join(mf[2:len(mf)], "")
 			return n.Annotations[akey]
 		} else {
 			log.Errorf("Unsupported InstanceIDField %s in CSV For PV", m)
@@ -346,6 +346,13 @@ func PVValueFromMapField(m string, n *v1.PersistentVolume) string {
 			log.Infof("[ERROR] Unsupported InstanceIDField %s in CSV For PV", m)
 			return ""
 		}
+	} else if len(mf) > 1 && mf[0] == "spec" {
+		if mf[1] == "storageClassName" {
+			return n.Spec.StorageClassName
+		} else {
+			log.Infof("[ERROR] Unsupported InstanceIDField %s in CSV For PV", m)
+			return ""
+		}
 	} else {
 		log.Errorf("Unsupported InstanceIDField %s in CSV For PV", m)
 		return ""

pkg/cloud/customprovider.go → pkg/cloud/provider/customprovider.go

@@ -1,4 +1,4 @@
-package cloud
+package provider
 
 import (
 	"errors"
@@ -33,10 +33,10 @@ type CustomProvider struct {
 	SpotLabelValue          string
 	GPULabel                string
 	GPULabelValue           string
-	clusterRegion           string
-	clusterAccountID        string
+	ClusterRegion           string
+	ClusterAccountID        string
 	DownloadPricingDataLock sync.RWMutex
-	Config                  *ProviderConfig
+	Config                  models.ProviderConfig
 }
 
 var volTypes = map[string]string{
@@ -147,8 +147,8 @@ func (cp *CustomProvider) ClusterInfo() (map[string]string, error) {
 		m["name"] = conf.ClusterName
 	}
 	m["provider"] = kubecost.CustomProvider
-	m["region"] = cp.clusterRegion
-	m["account"] = cp.clusterAccountID
+	m["region"] = cp.ClusterRegion
+	m["account"] = cp.ClusterAccountID
 	m["id"] = env.GetClusterID()
 	return m, nil
 }

pkg/cloud/provider.go → pkg/cloud/provider/provider.go

@@ -1,4 +1,4 @@
-package cloud
+package provider
 
 import (
 	"errors"
@@ -8,10 +8,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/opencost/opencost/pkg/cloud/alibaba"
 	"github.com/opencost/opencost/pkg/cloud/aws"
 	"github.com/opencost/opencost/pkg/cloud/azure"
 	"github.com/opencost/opencost/pkg/cloud/gcp"
 	"github.com/opencost/opencost/pkg/cloud/models"
+	"github.com/opencost/opencost/pkg/cloud/scaleway"
 	"github.com/opencost/opencost/pkg/kubecost"
 
 	"github.com/opencost/opencost/pkg/util"
@@ -160,6 +162,13 @@ func NewProvider(cache clustercache.ClusterCache, apiKey string, config *config.
 		cp.accountID = providerConfig.customPricing.ClusterAccountID
 	}
 
+	providerConfig.Update(func(cp *models.CustomPricing) error {
+		if cp.ServiceKeyName == "AKIXXX" {
+			cp.ServiceKeyName = ""
+		}
+		return nil
+	})
+
 	switch cp.provider {
 	case kubecost.CSVProvider:
 		log.Infof("Using CSV Provider with CSV at %s", env.GetCSVPath())
@@ -167,8 +176,8 @@ func NewProvider(cache clustercache.ClusterCache, apiKey string, config *config.
 			CSVLocation: env.GetCSVPath(),
 			CustomProvider: &CustomProvider{
 				Clientset:        cache,
-				clusterRegion:    cp.region,
-				clusterAccountID: cp.accountID,
+				ClusterRegion:    cp.region,
+				ClusterAccountID: cp.accountID,
 				Config:           NewProviderConfig(config, cp.configFileName),
 			},
 		}, nil
@@ -215,19 +224,19 @@ func NewProvider(cache clustercache.ClusterCache, apiKey string, config *config.
 		}, nil
 	case kubecost.AlibabaProvider:
 		log.Info("Found ProviderID starting with \"alibaba\", using Alibaba Cloud Provider")
-		return &Alibaba{
+		return &alibaba.Alibaba{
 			Clientset:            cache,
 			Config:               NewProviderConfig(config, cp.configFileName),
-			clusterRegion:        cp.region,
-			clusterAccountId:     cp.accountID,
-			serviceAccountChecks: models.NewServiceAccountChecks(),
+			ClusterRegion:        cp.region,
+			ClusterAccountId:     cp.accountID,
+			ServiceAccountChecks: models.NewServiceAccountChecks(),
 		}, nil
 	case kubecost.ScalewayProvider:
 		log.Info("Found ProviderID starting with \"scaleway\", using Scaleway Provider")
-		return &Scaleway{
+		return &scaleway.Scaleway{
 			Clientset:        cache,
-			clusterRegion:    cp.region,
-			clusterAccountID: cp.accountID,
+			ClusterRegion:    cp.region,
+			ClusterAccountID: cp.accountID,
 			Config:           NewProviderConfig(config, cp.configFileName),
 		}, nil
 
@@ -235,8 +244,8 @@ func NewProvider(cache clustercache.ClusterCache, apiKey string, config *config.
 		log.Info("Unsupported provider, falling back to default")
 		return &CustomProvider{
 			Clientset:        cache,
-			clusterRegion:    cp.region,
-			clusterAccountID: cp.accountID,
+			ClusterRegion:    cp.region,
+			ClusterAccountID: cp.accountID,
 			Config:           NewProviderConfig(config, cp.configFileName),
 		}, nil
 	}

pkg/cloud/providerconfig.go → pkg/cloud/provider/providerconfig.go

@@ -1,8 +1,7 @@
-package cloud
+package provider
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	gopath "path"
 	"strconv"
@@ -144,6 +143,12 @@ func (pc *ProviderConfig) loadConfig(writeIfNotExists bool) (*models.CustomPrici
 		pc.customPricing.ShareTenancyCosts = models.DefaultShareTenancyCost
 	}
 
+	// If the sample nil service key name is set, zero it out so that it is not
+	// misinterpreted as a real service key.
+	if pc.customPricing.ServiceKeyName == "AKIXXX" {
+		pc.customPricing.ServiceKeyName = ""
+	}
+
 	return pc.customPricing, nil
 }
 
@@ -277,7 +282,7 @@ func ReturnPricingFromConfigs(filename string) (*models.CustomPricing, error) {
 	if _, err := os.Stat(providerConfigFile); err != nil {
 		return &models.CustomPricing{}, fmt.Errorf("ReturnPricingFromConfigs: unable to find file %s with err: %v", providerConfigFile, err)
 	}
-	configFile, err := ioutil.ReadFile(providerConfigFile)
+	configFile, err := os.ReadFile(providerConfigFile)
 	if err != nil {
 		return &models.CustomPricing{}, fmt.Errorf("ReturnPricingFromConfigs: unable to open file %s with err: %v", providerConfigFile, err)
 	}

pkg/cloud/scalewayprovider.go → pkg/cloud/scaleway/provider.go

@@ -1,4 +1,4 @@
-package cloud
+package scaleway
 
 import (
 	"errors"
@@ -36,10 +36,10 @@ type ScalewayPricing struct {
 
 type Scaleway struct {
 	Clientset               clustercache.ClusterCache
-	Config                  *ProviderConfig
+	Config                  models.ProviderConfig
 	Pricing                 map[string]*ScalewayPricing
-	clusterRegion           string
-	clusterAccountID        string
+	ClusterRegion           string
+	ClusterAccountID        string
 	DownloadPricingDataLock sync.RWMutex
 }
 
@@ -288,8 +288,8 @@ func (scw *Scaleway) ClusterInfo() (map[string]string, error) {
 		m["name"] = c.ClusterName
 	}
 	m["provider"] = kubecost.ScalewayProvider
-	m["region"] = scw.clusterRegion
-	m["account"] = scw.clusterAccountID
+	m["region"] = scw.ClusterRegion
+	m["account"] = scw.ClusterAccountID
 	m["remoteReadEnabled"] = strconv.FormatBool(remoteEnabled)
 	m["id"] = env.GetClusterID()
 	return m, nil

pkg/cmd/agent/agent.go

@@ -7,7 +7,7 @@ import (
 	"path"
 	"time"
 
-	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/opencost/opencost/pkg/cloud/provider"
 	"github.com/opencost/opencost/pkg/clustercache"
 	"github.com/opencost/opencost/pkg/config"
 	"github.com/opencost/opencost/pkg/costmodel"
@@ -157,13 +157,13 @@ func Execute(opts *AgentOpts) error {
 	})
 
 	cloudProviderKey := env.GetCloudProviderAPIKey()
-	cloudProvider, err := cloud.NewProvider(clusterCache, cloudProviderKey, confManager)
+	cloudProvider, err := provider.NewProvider(clusterCache, cloudProviderKey, confManager)
 	if err != nil {
 		panic(err.Error())
 	}
 
 	// Append the pricing config watcher
-	configWatchers.AddWatcher(cloud.ConfigWatcherFor(cloudProvider))
+	configWatchers.AddWatcher(provider.ConfigWatcherFor(cloudProvider))
 	watchConfigFunc := configWatchers.ToWatchFunc()
 	watchedConfigs := configWatchers.GetWatchedConfigs()
 

pkg/costmodel/aggregation.go

@@ -11,10 +11,10 @@ import (
 	"time"
 
 	"github.com/julienschmidt/httprouter"
+	"github.com/opencost/opencost/pkg/cloud/provider"
 	"github.com/patrickmn/go-cache"
 	prometheusClient "github.com/prometheus/client_golang/api"
 
-	"github.com/opencost/opencost/pkg/cloud"
 	"github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/env"
 	"github.com/opencost/opencost/pkg/errors"
@@ -761,7 +761,7 @@ func getPriceVectors(cp models.Provider, costDatum *CostData, rate string, disco
 	if err != nil {
 		log.Errorf("failed to load custom pricing: %s", err)
 	}
-	if cloud.CustomPricesEnabled(cp) && err == nil {
+	if provider.CustomPricesEnabled(cp) && err == nil {
 		var cpuCostStr string
 		var ramCostStr string
 		var gpuCostStr string
@@ -839,7 +839,7 @@ func getPriceVectors(cp models.Provider, costDatum *CostData, rate string, disco
 			cost, _ := strconv.ParseFloat(pvcData.Volume.Cost, 64)
 
 			// override with custom pricing if enabled
-			if cloud.CustomPricesEnabled(cp) {
+			if provider.CustomPricesEnabled(cp) {
 				cost = pvCost
 			}
 
@@ -1768,10 +1768,10 @@ func (a *Accesses) warmAggregateCostModelCache() {
 		aggOpts.NoExpireCache = false
 		aggOpts.ShareSplit = SplitTypeWeighted
 		aggOpts.RemoteEnabled = env.IsRemoteEnabled()
-		aggOpts.AllocateIdle = cloud.AllocateIdleByDefault(a.CloudProvider)
+		aggOpts.AllocateIdle = provider.AllocateIdleByDefault(a.CloudProvider)
 
-		sharedNamespaces := cloud.SharedNamespaces(a.CloudProvider)
-		sharedLabelNames, sharedLabelValues := cloud.SharedLabels(a.CloudProvider)
+		sharedNamespaces := provider.SharedNamespaces(a.CloudProvider)
+		sharedLabelNames, sharedLabelValues := provider.SharedLabels(a.CloudProvider)
 
 		if len(sharedNamespaces) > 0 || len(sharedLabelNames) > 0 {
 			aggOpts.SharedResources = NewSharedResourceInfo(true, sharedNamespaces, sharedLabelNames, sharedLabelValues)
@@ -2242,6 +2242,19 @@ func (a *Accesses) ComputeAllocationHandler(w http.ResponseWriter, r *http.Reque
 
 	// IncludeIdle, if true, uses Asset data to incorporate Idle Allocation
 	includeIdle := qp.GetBool("includeIdle", false)
+	// Accumulate is an optional parameter, defaulting to false, which if true
+	// sums each Set in the Range, producing one Set.
+	accumulate := qp.GetBool("accumulate", false)
+
+	// Accumulate is an optional parameter that accumulates an AllocationSetRange
+	// by the resolution of the given time duration.
+	// Defaults to 0. If a value is not passed then the parameter is not used.
+	accumulateBy := kubecost.AccumulateOption(qp.Get("accumulateBy", ""))
+
+	// if accumulateBy is not explicitly set, and accumulate is true, ensure result is accumulated
+	if accumulateBy == kubecost.AccumulateOptionNone && accumulate {
+		accumulateBy = kubecost.AccumulateOptionAll
+	}
 
 	// IdleByNode, if true, computes idle allocations at the node level.
 	// Otherwise it is computed at the cluster level. (Not relevant if idle
@@ -2252,7 +2265,7 @@ func (a *Accesses) ComputeAllocationHandler(w http.ResponseWriter, r *http.Reque
 	includeProportionalAssetResourceCosts := qp.GetBool("includeProportionalAssetResourceCosts", false)
 
 	// include aggregated labels/annotations if true
-	includeAggregatedMetadata := qp.GetBool("includeAggregatedMetadata", true)
+	includeAggregatedMetadata := qp.GetBool("includeAggregatedMetadata", false)
 
 	asr, err := a.Model.QueryAllocation(window, resolution, step, aggregateBy, includeIdle, idleByNode, includeProportionalAssetResourceCosts, includeAggregatedMetadata)
 	if err != nil {
@@ -2265,6 +2278,16 @@ func (a *Accesses) ComputeAllocationHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
+	// Accumulate, if requested
+	if accumulateBy != kubecost.AccumulateOptionNone {
+		asr, err = asr.Accumulate(accumulateBy)
+		if err != nil {
+			log.Errorf("error accumulating by %v: %s", accumulateBy, err)
+			WriteError(w, InternalServerError(fmt.Errorf("error accumulating by %v: %s", accumulateBy, err).Error()))
+			return
+		}
+	}
+
 	w.Write(WrapData(asr, nil))
 }
 

pkg/costmodel/allocation.go

@@ -13,52 +13,52 @@ import (
 )
 
 const (
-	queryFmtPods                        = `avg(kube_pod_container_status_running{}) by (pod, namespace, %s)[%s:%s]`
-	queryFmtPodsUID                     = `avg(kube_pod_container_status_running{}) by (pod, namespace, uid, %s)[%s:%s]`
-	queryFmtRAMBytesAllocated           = `avg(avg_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!=""}[%s])) by (container, pod, namespace, node, %s, provider_id)`
-	queryFmtRAMRequests                 = `avg(avg_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte", container!="", container!="POD", node!=""}[%s])) by (container, pod, namespace, node, %s)`
-	queryFmtRAMUsageAvg                 = `avg(avg_over_time(container_memory_working_set_bytes{container!="", container_name!="POD", container!="POD"}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
-	queryFmtRAMUsageMax                 = `max(max_over_time(container_memory_working_set_bytes{container!="", container_name!="POD", container!="POD"}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
-	queryFmtCPUCoresAllocated           = `avg(avg_over_time(container_cpu_allocation{container!="", container!="POD", node!=""}[%s])) by (container, pod, namespace, node, %s)`
-	queryFmtCPURequests                 = `avg(avg_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="", container!="POD", node!=""}[%s])) by (container, pod, namespace, node, %s)`
-	queryFmtCPUUsageAvg                 = `avg(rate(container_cpu_usage_seconds_total{container!="", container_name!="POD", container!="POD"}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
-	queryFmtGPUsRequested               = `avg(avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!=""}[%s])) by (container, pod, namespace, node, %s)`
-	queryFmtGPUsAllocated               = `avg(avg_over_time(container_gpu_allocation{container!="", container!="POD", node!=""}[%s])) by (container, pod, namespace, node, %s)`
-	queryFmtNodeCostPerCPUHr            = `avg(avg_over_time(node_cpu_hourly_cost[%s])) by (node, %s, instance_type, provider_id)`
-	queryFmtNodeCostPerRAMGiBHr         = `avg(avg_over_time(node_ram_hourly_cost[%s])) by (node, %s, instance_type, provider_id)`
-	queryFmtNodeCostPerGPUHr            = `avg(avg_over_time(node_gpu_hourly_cost[%s])) by (node, %s, instance_type, provider_id)`
-	queryFmtNodeIsSpot                  = `avg_over_time(kubecost_node_is_spot[%s])`
-	queryFmtPVCInfo                     = `avg(kube_persistentvolumeclaim_info{volumename != ""}) by (persistentvolumeclaim, storageclass, volumename, namespace, %s)[%s:%s]`
-	queryFmtPodPVCAllocation            = `avg(avg_over_time(pod_pvc_allocation[%s])) by (persistentvolume, persistentvolumeclaim, pod, namespace, %s)`
-	queryFmtPVCBytesRequested           = `avg(avg_over_time(kube_persistentvolumeclaim_resource_requests_storage_bytes{}[%s])) by (persistentvolumeclaim, namespace, %s)`
-	queryFmtPVActiveMins                = `count(kube_persistentvolume_capacity_bytes) by (persistentvolume, %s)[%s:%s]`
-	queryFmtPVBytes                     = `avg(avg_over_time(kube_persistentvolume_capacity_bytes[%s])) by (persistentvolume, %s)`
-	queryFmtPVCostPerGiBHour            = `avg(avg_over_time(pv_hourly_cost[%s])) by (volumename, %s)`
-	queryFmtNetZoneGiB                  = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="true"}[%s])) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
-	queryFmtNetZoneCostPerGiB           = `avg(avg_over_time(kubecost_network_zone_egress_cost{}[%s])) by (%s)`
-	queryFmtNetRegionGiB                = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="false"}[%s])) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
-	queryFmtNetRegionCostPerGiB         = `avg(avg_over_time(kubecost_network_region_egress_cost{}[%s])) by (%s)`
-	queryFmtNetInternetGiB              = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="true"}[%s])) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
-	queryFmtNetInternetCostPerGiB       = `avg(avg_over_time(kubecost_network_internet_egress_cost{}[%s])) by (%s)`
-	queryFmtNetReceiveBytes             = `sum(increase(container_network_receive_bytes_total{pod!=""}[%s])) by (pod_name, pod, namespace, %s)`
-	queryFmtNetTransferBytes            = `sum(increase(container_network_transmit_bytes_total{pod!=""}[%s])) by (pod_name, pod, namespace, %s)`
-	queryFmtNodeLabels                  = `avg_over_time(kube_node_labels[%s])`
-	queryFmtNamespaceLabels             = `avg_over_time(kube_namespace_labels[%s])`
-	queryFmtNamespaceAnnotations        = `avg_over_time(kube_namespace_annotations[%s])`
-	queryFmtPodLabels                   = `avg_over_time(kube_pod_labels[%s])`
-	queryFmtPodAnnotations              = `avg_over_time(kube_pod_annotations[%s])`
-	queryFmtServiceLabels               = `avg_over_time(service_selector_labels[%s])`
-	queryFmtDeploymentLabels            = `avg_over_time(deployment_match_labels[%s])`
-	queryFmtStatefulSetLabels           = `avg_over_time(statefulSet_match_labels[%s])`
-	queryFmtDaemonSetLabels             = `sum(avg_over_time(kube_pod_owner{owner_kind="DaemonSet"}[%s])) by (pod, owner_name, namespace, %s)`
-	queryFmtJobLabels                   = `sum(avg_over_time(kube_pod_owner{owner_kind="Job"}[%s])) by (pod, owner_name, namespace ,%s)`
-	queryFmtPodsWithReplicaSetOwner     = `sum(avg_over_time(kube_pod_owner{owner_kind="ReplicaSet"}[%s])) by (pod, owner_name, namespace ,%s)`
-	queryFmtReplicaSetsWithoutOwners    = `avg(avg_over_time(kube_replicaset_owner{owner_kind="<none>", owner_name="<none>"}[%s])) by (replicaset, namespace, %s)`
-	queryFmtReplicaSetsWithRolloutOwner = `avg(avg_over_time(kube_replicaset_owner{owner_kind="Rollout"}[%s])) by (replicaset, namespace, owner_kind, owner_name, %s)`
-	queryFmtLBCostPerHr                 = `avg(avg_over_time(kubecost_load_balancer_cost[%s])) by (namespace, service_name, %s)`
-	queryFmtLBActiveMins                = `count(kubecost_load_balancer_cost) by (namespace, service_name, %s)[%s:%s]`
-	queryFmtOldestSample                = `min_over_time(timestamp(group(node_cpu_hourly_cost))[%s:%s])`
-	queryFmtNewestSample                = `max_over_time(timestamp(group(node_cpu_hourly_cost))[%s:%s])`
+	queryFmtPods                        = `avg(kube_pod_container_status_running{%s}) by (pod, namespace, %s)[%s:%s]`
+	queryFmtPodsUID                     = `avg(kube_pod_container_status_running{%s}) by (pod, namespace, uid, %s)[%s:%s]`
+	queryFmtRAMBytesAllocated           = `avg(avg_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s, provider_id)`
+	queryFmtRAMRequests                 = `avg(avg_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte", container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
+	queryFmtRAMUsageAvg                 = `avg(avg_over_time(container_memory_working_set_bytes{container!="", container_name!="POD", container!="POD", %s}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
+	queryFmtRAMUsageMax                 = `max(max_over_time(container_memory_working_set_bytes{container!="", container_name!="POD", container!="POD", %s}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
+	queryFmtCPUCoresAllocated           = `avg(avg_over_time(container_cpu_allocation{container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
+	queryFmtCPURequests                 = `avg(avg_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
+	queryFmtCPUUsageAvg                 = `avg(rate(container_cpu_usage_seconds_total{container!="", container_name!="POD", container!="POD", %s}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
+	queryFmtGPUsRequested               = `avg(avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
+	queryFmtGPUsAllocated               = `avg(avg_over_time(container_gpu_allocation{container!="", container!="POD", node!="", %s}[%s])) by (container, pod, namespace, node, %s)`
+	queryFmtNodeCostPerCPUHr            = `avg(avg_over_time(node_cpu_hourly_cost{%s}[%s])) by (node, %s, instance_type, provider_id)`
+	queryFmtNodeCostPerRAMGiBHr         = `avg(avg_over_time(node_ram_hourly_cost{%s}[%s])) by (node, %s, instance_type, provider_id)`
+	queryFmtNodeCostPerGPUHr            = `avg(avg_over_time(node_gpu_hourly_cost{%s}[%s])) by (node, %s, instance_type, provider_id)`
+	queryFmtNodeIsSpot                  = `avg_over_time(kubecost_node_is_spot{%s}[%s])`
+	queryFmtPVCInfo                     = `avg(kube_persistentvolumeclaim_info{volumename != "", %s}) by (persistentvolumeclaim, storageclass, volumename, namespace, %s)[%s:%s]`
+	queryFmtPodPVCAllocation            = `avg(avg_over_time(pod_pvc_allocation{%s}[%s])) by (persistentvolume, persistentvolumeclaim, pod, namespace, %s)`
+	queryFmtPVCBytesRequested           = `avg(avg_over_time(kube_persistentvolumeclaim_resource_requests_storage_bytes{%s}[%s])) by (persistentvolumeclaim, namespace, %s)`
+	queryFmtPVActiveMins                = `count(kube_persistentvolume_capacity_bytes{%s}) by (persistentvolume, %s)[%s:%s]`
+	queryFmtPVBytes                     = `avg(avg_over_time(kube_persistentvolume_capacity_bytes{%s}[%s])) by (persistentvolume, %s)`
+	queryFmtPVCostPerGiBHour            = `avg(avg_over_time(pv_hourly_cost{%s}[%s])) by (volumename, %s)`
+	queryFmtNetZoneGiB                  = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="true", %s}[%s])) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
+	queryFmtNetZoneCostPerGiB           = `avg(avg_over_time(kubecost_network_zone_egress_cost{%s}[%s])) by (%s)`
+	queryFmtNetRegionGiB                = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="false", %s}[%s])) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
+	queryFmtNetRegionCostPerGiB         = `avg(avg_over_time(kubecost_network_region_egress_cost{%s}[%s])) by (%s)`
+	queryFmtNetInternetGiB              = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="true", %s}[%s])) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
+	queryFmtNetInternetCostPerGiB       = `avg(avg_over_time(kubecost_network_internet_egress_cost{%s}[%s])) by (%s)`
+	queryFmtNetReceiveBytes             = `sum(increase(container_network_receive_bytes_total{pod!="", %s}[%s])) by (pod_name, pod, namespace, %s)`
+	queryFmtNetTransferBytes            = `sum(increase(container_network_transmit_bytes_total{pod!="", %s}[%s])) by (pod_name, pod, namespace, %s)`
+	queryFmtNodeLabels                  = `avg_over_time(kube_node_labels{%s}[%s])`
+	queryFmtNamespaceLabels             = `avg_over_time(kube_namespace_labels{%s}[%s])`
+	queryFmtNamespaceAnnotations        = `avg_over_time(kube_namespace_annotations{%s}[%s])`
+	queryFmtPodLabels                   = `avg_over_time(kube_pod_labels{%s}[%s])`
+	queryFmtPodAnnotations              = `avg_over_time(kube_pod_annotations{%s}[%s])`
+	queryFmtServiceLabels               = `avg_over_time(service_selector_labels{%s}[%s])`
+	queryFmtDeploymentLabels            = `avg_over_time(deployment_match_labels{%s}[%s])`
+	queryFmtStatefulSetLabels           = `avg_over_time(statefulSet_match_labels{%s}[%s])`
+	queryFmtDaemonSetLabels             = `sum(avg_over_time(kube_pod_owner{owner_kind="DaemonSet", %s}[%s])) by (pod, owner_name, namespace, %s)`
+	queryFmtJobLabels                   = `sum(avg_over_time(kube_pod_owner{owner_kind="Job", %s}[%s])) by (pod, owner_name, namespace ,%s)`
+	queryFmtPodsWithReplicaSetOwner     = `sum(avg_over_time(kube_pod_owner{owner_kind="ReplicaSet", %s}[%s])) by (pod, owner_name, namespace ,%s)`
+	queryFmtReplicaSetsWithoutOwners    = `avg(avg_over_time(kube_replicaset_owner{owner_kind="<none>", owner_name="<none>", %s}[%s])) by (replicaset, namespace, %s)`
+	queryFmtReplicaSetsWithRolloutOwner = `avg(avg_over_time(kube_replicaset_owner{owner_kind="Rollout", %s}[%s])) by (replicaset, namespace, owner_kind, owner_name, %s)`
+	queryFmtLBCostPerHr                 = `avg(avg_over_time(kubecost_load_balancer_cost{%s}[%s])) by (namespace, service_name, %s)`
+	queryFmtLBActiveMins                = `count(kubecost_load_balancer_cost{%s}) by (namespace, service_name, %s)[%s:%s]`
+	queryFmtOldestSample                = `min_over_time(timestamp(group(node_cpu_hourly_cost{%s}))[%s:%s])`
+	queryFmtNewestSample                = `max_over_time(timestamp(group(node_cpu_hourly_cost{%s}))[%s:%s])`
 
 	// Because we use container_cpu_usage_seconds_total to calculate CPU usage
 	// at any given "instant" of time, we need to use an irate or rate. To then
@@ -73,7 +73,7 @@ const (
 	//
 	// If changing the name of the recording rule, make sure to update the
 	// corresponding diagnostic query to avoid confusion.
-	queryFmtCPUUsageMaxRecordingRule = `max(max_over_time(kubecost_container_cpu_usage_irate{}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
+	queryFmtCPUUsageMaxRecordingRule = `max(max_over_time(kubecost_container_cpu_usage_irate{%s}[%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
 	// This is the subquery equivalent of the above recording rule query. It is
 	// more expensive, but does not require the recording rule. It should be
 	// used as a fallback query if the recording rule data does not exist.
@@ -84,7 +84,7 @@ const (
 	// the resolution, to make sure the irate always has two points to query
 	// in case the Prom scrape duration has been reduced to be equal to the
 	// ETL resolution.
-	queryFmtCPUUsageMaxSubquery = `max(max_over_time(irate(container_cpu_usage_seconds_total{container_name!="POD", container_name!=""}[%s])[%s:%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
+	queryFmtCPUUsageMaxSubquery = `max(max_over_time(irate(container_cpu_usage_seconds_total{container_name!="POD", container_name!="", %s}[%s])[%s:%s])) by (container_name, container, pod_name, pod, namespace, instance, %s)`
 )
 
 // Constants for Network Cost Subtype
@@ -287,7 +287,7 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 func (cm *CostModel) DateRange() (time.Time, time.Time, error) {
 	ctx := prom.NewNamedContext(cm.PrometheusClient, prom.AllocationContextName)
 
-	resOldest, _, err := ctx.QuerySync(fmt.Sprintf(queryFmtOldestSample, "90d", "1h"))
+	resOldest, _, err := ctx.QuerySync(fmt.Sprintf(queryFmtOldestSample, env.GetPromClusterFilter(), "90d", "1h"))
 	if err != nil {
 		return time.Time{}, time.Time{}, fmt.Errorf("querying oldest sample: %w", err)
 	}
@@ -296,7 +296,7 @@ func (cm *CostModel) DateRange() (time.Time, time.Time, error) {
 	}
 	oldest := time.Unix(int64(resOldest[0].Values[0].Value), 0)
 
-	resNewest, _, err := ctx.QuerySync(fmt.Sprintf(queryFmtNewestSample, "90d", "1h"))
+	resNewest, _, err := ctx.QuerySync(fmt.Sprintf(queryFmtNewestSample, env.GetPromClusterFilter(), "90d", "1h"))
 	if err != nil {
 		return time.Time{}, time.Time{}, fmt.Errorf("querying newest sample: %w", err)
 	}
@@ -373,28 +373,28 @@ func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Dur
 
 	ctx := prom.NewNamedContext(cm.PrometheusClient, prom.AllocationContextName)
 
-	queryRAMBytesAllocated := fmt.Sprintf(queryFmtRAMBytesAllocated, durStr, env.GetPromClusterLabel())
+	queryRAMBytesAllocated := fmt.Sprintf(queryFmtRAMBytesAllocated, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChRAMBytesAllocated := ctx.QueryAtTime(queryRAMBytesAllocated, end)
 
-	queryRAMRequests := fmt.Sprintf(queryFmtRAMRequests, durStr, env.GetPromClusterLabel())
+	queryRAMRequests := fmt.Sprintf(queryFmtRAMRequests, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChRAMRequests := ctx.QueryAtTime(queryRAMRequests, end)
 
-	queryRAMUsageAvg := fmt.Sprintf(queryFmtRAMUsageAvg, durStr, env.GetPromClusterLabel())
+	queryRAMUsageAvg := fmt.Sprintf(queryFmtRAMUsageAvg, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChRAMUsageAvg := ctx.QueryAtTime(queryRAMUsageAvg, end)
 
-	queryRAMUsageMax := fmt.Sprintf(queryFmtRAMUsageMax, durStr, env.GetPromClusterLabel())
+	queryRAMUsageMax := fmt.Sprintf(queryFmtRAMUsageMax, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChRAMUsageMax := ctx.QueryAtTime(queryRAMUsageMax, end)
 
-	queryCPUCoresAllocated := fmt.Sprintf(queryFmtCPUCoresAllocated, durStr, env.GetPromClusterLabel())
+	queryCPUCoresAllocated := fmt.Sprintf(queryFmtCPUCoresAllocated, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChCPUCoresAllocated := ctx.QueryAtTime(queryCPUCoresAllocated, end)
 
-	queryCPURequests := fmt.Sprintf(queryFmtCPURequests, durStr, env.GetPromClusterLabel())
+	queryCPURequests := fmt.Sprintf(queryFmtCPURequests, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChCPURequests := ctx.QueryAtTime(queryCPURequests, end)
 
-	queryCPUUsageAvg := fmt.Sprintf(queryFmtCPUUsageAvg, durStr, env.GetPromClusterLabel())
+	queryCPUUsageAvg := fmt.Sprintf(queryFmtCPUUsageAvg, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChCPUUsageAvg := ctx.QueryAtTime(queryCPUUsageAvg, end)
 
-	queryCPUUsageMax := fmt.Sprintf(queryFmtCPUUsageMaxRecordingRule, durStr, env.GetPromClusterLabel())
+	queryCPUUsageMax := fmt.Sprintf(queryFmtCPUUsageMaxRecordingRule, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChCPUUsageMax := ctx.QueryAtTime(queryCPUUsageMax, end)
 	resCPUUsageMax, _ := resChCPUUsageMax.Await()
 	// If the recording rule has no data, try to fall back to the subquery.
@@ -404,7 +404,7 @@ func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Dur
 		// in case the Prom scrape duration has been reduced to be equal to the
 		// resolution.
 		doubleResStr := timeutil.DurationString(2 * resolution)
-		queryCPUUsageMax = fmt.Sprintf(queryFmtCPUUsageMaxSubquery, doubleResStr, durStr, resStr, env.GetPromClusterLabel())
+		queryCPUUsageMax = fmt.Sprintf(queryFmtCPUUsageMaxSubquery, env.GetPromClusterFilter(), doubleResStr, durStr, resStr, env.GetPromClusterLabel())
 		resChCPUUsageMax = ctx.QueryAtTime(queryCPUUsageMax, end)
 		resCPUUsageMax, _ = resChCPUUsageMax.Await()
 
@@ -415,112 +415,112 @@ func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Dur
 		}
 	}
 
-	queryGPUsRequested := fmt.Sprintf(queryFmtGPUsRequested, durStr, env.GetPromClusterLabel())
+	queryGPUsRequested := fmt.Sprintf(queryFmtGPUsRequested, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChGPUsRequested := ctx.QueryAtTime(queryGPUsRequested, end)
 
-	queryGPUsAllocated := fmt.Sprintf(queryFmtGPUsAllocated, durStr, env.GetPromClusterLabel())
+	queryGPUsAllocated := fmt.Sprintf(queryFmtGPUsAllocated, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChGPUsAllocated := ctx.QueryAtTime(queryGPUsAllocated, end)
 
-	queryNodeCostPerCPUHr := fmt.Sprintf(queryFmtNodeCostPerCPUHr, durStr, env.GetPromClusterLabel())
+	queryNodeCostPerCPUHr := fmt.Sprintf(queryFmtNodeCostPerCPUHr, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChNodeCostPerCPUHr := ctx.QueryAtTime(queryNodeCostPerCPUHr, end)
 
-	queryNodeCostPerRAMGiBHr := fmt.Sprintf(queryFmtNodeCostPerRAMGiBHr, durStr, env.GetPromClusterLabel())
+	queryNodeCostPerRAMGiBHr := fmt.Sprintf(queryFmtNodeCostPerRAMGiBHr, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChNodeCostPerRAMGiBHr := ctx.QueryAtTime(queryNodeCostPerRAMGiBHr, end)
 
-	queryNodeCostPerGPUHr := fmt.Sprintf(queryFmtNodeCostPerGPUHr, durStr, env.GetPromClusterLabel())
+	queryNodeCostPerGPUHr := fmt.Sprintf(queryFmtNodeCostPerGPUHr, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChNodeCostPerGPUHr := ctx.QueryAtTime(queryNodeCostPerGPUHr, end)
 
-	queryNodeIsSpot := fmt.Sprintf(queryFmtNodeIsSpot, durStr)
+	queryNodeIsSpot := fmt.Sprintf(queryFmtNodeIsSpot, env.GetPromClusterFilter(), durStr)
 	resChNodeIsSpot := ctx.QueryAtTime(queryNodeIsSpot, end)
 
-	queryPVCInfo := fmt.Sprintf(queryFmtPVCInfo, env.GetPromClusterLabel(), durStr, resStr)
+	queryPVCInfo := fmt.Sprintf(queryFmtPVCInfo, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, resStr)
 	resChPVCInfo := ctx.QueryAtTime(queryPVCInfo, end)
 
-	queryPodPVCAllocation := fmt.Sprintf(queryFmtPodPVCAllocation, durStr, env.GetPromClusterLabel())
+	queryPodPVCAllocation := fmt.Sprintf(queryFmtPodPVCAllocation, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChPodPVCAllocation := ctx.QueryAtTime(queryPodPVCAllocation, end)
 
-	queryPVCBytesRequested := fmt.Sprintf(queryFmtPVCBytesRequested, durStr, env.GetPromClusterLabel())
+	queryPVCBytesRequested := fmt.Sprintf(queryFmtPVCBytesRequested, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChPVCBytesRequested := ctx.QueryAtTime(queryPVCBytesRequested, end)
 
-	queryPVActiveMins := fmt.Sprintf(queryFmtPVActiveMins, env.GetPromClusterLabel(), durStr, resStr)
+	queryPVActiveMins := fmt.Sprintf(queryFmtPVActiveMins, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, resStr)
 	resChPVActiveMins := ctx.QueryAtTime(queryPVActiveMins, end)
 
-	queryPVBytes := fmt.Sprintf(queryFmtPVBytes, durStr, env.GetPromClusterLabel())
+	queryPVBytes := fmt.Sprintf(queryFmtPVBytes, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChPVBytes := ctx.QueryAtTime(queryPVBytes, end)
 
-	queryPVCostPerGiBHour := fmt.Sprintf(queryFmtPVCostPerGiBHour, durStr, env.GetPromClusterLabel())
+	queryPVCostPerGiBHour := fmt.Sprintf(queryFmtPVCostPerGiBHour, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChPVCostPerGiBHour := ctx.QueryAtTime(queryPVCostPerGiBHour, end)
 
-	queryNetTransferBytes := fmt.Sprintf(queryFmtNetTransferBytes, durStr, env.GetPromClusterLabel())
+	queryNetTransferBytes := fmt.Sprintf(queryFmtNetTransferBytes, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChNetTransferBytes := ctx.QueryAtTime(queryNetTransferBytes, end)
 
-	queryNetReceiveBytes := fmt.Sprintf(queryFmtNetReceiveBytes, durStr, env.GetPromClusterLabel())
+	queryNetReceiveBytes := fmt.Sprintf(queryFmtNetReceiveBytes, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChNetReceiveBytes := ctx.QueryAtTime(queryNetReceiveBytes, end)
 
-	queryNetZoneGiB := fmt.Sprintf(queryFmtNetZoneGiB, durStr, env.GetPromClusterLabel())
+	queryNetZoneGiB := fmt.Sprintf(queryFmtNetZoneGiB, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChNetZoneGiB := ctx.QueryAtTime(queryNetZoneGiB, end)
 
-	queryNetZoneCostPerGiB := fmt.Sprintf(queryFmtNetZoneCostPerGiB, durStr, env.GetPromClusterLabel())
+	queryNetZoneCostPerGiB := fmt.Sprintf(queryFmtNetZoneCostPerGiB, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChNetZoneCostPerGiB := ctx.QueryAtTime(queryNetZoneCostPerGiB, end)
 
-	queryNetRegionGiB := fmt.Sprintf(queryFmtNetRegionGiB, durStr, env.GetPromClusterLabel())
+	queryNetRegionGiB := fmt.Sprintf(queryFmtNetRegionGiB, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChNetRegionGiB := ctx.QueryAtTime(queryNetRegionGiB, end)
 
-	queryNetRegionCostPerGiB := fmt.Sprintf(queryFmtNetRegionCostPerGiB, durStr, env.GetPromClusterLabel())
+	queryNetRegionCostPerGiB := fmt.Sprintf(queryFmtNetRegionCostPerGiB, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChNetRegionCostPerGiB := ctx.QueryAtTime(queryNetRegionCostPerGiB, end)
 
-	queryNetInternetGiB := fmt.Sprintf(queryFmtNetInternetGiB, durStr, env.GetPromClusterLabel())
+	queryNetInternetGiB := fmt.Sprintf(queryFmtNetInternetGiB, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChNetInternetGiB := ctx.QueryAtTime(queryNetInternetGiB, end)
 
-	queryNetInternetCostPerGiB := fmt.Sprintf(queryFmtNetInternetCostPerGiB, durStr, env.GetPromClusterLabel())
+	queryNetInternetCostPerGiB := fmt.Sprintf(queryFmtNetInternetCostPerGiB, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChNetInternetCostPerGiB := ctx.QueryAtTime(queryNetInternetCostPerGiB, end)
 
 	var resChNodeLabels prom.QueryResultsChan
 	if env.GetAllocationNodeLabelsEnabled() {
-		queryNodeLabels := fmt.Sprintf(queryFmtNodeLabels, durStr)
+		queryNodeLabels := fmt.Sprintf(queryFmtNodeLabels, env.GetPromClusterFilter(), durStr)
 		resChNodeLabels = ctx.QueryAtTime(queryNodeLabels, end)
 	}
 
-	queryNamespaceLabels := fmt.Sprintf(queryFmtNamespaceLabels, durStr)
+	queryNamespaceLabels := fmt.Sprintf(queryFmtNamespaceLabels, env.GetPromClusterFilter(), durStr)
 	resChNamespaceLabels := ctx.QueryAtTime(queryNamespaceLabels, end)
 
-	queryNamespaceAnnotations := fmt.Sprintf(queryFmtNamespaceAnnotations, durStr)
+	queryNamespaceAnnotations := fmt.Sprintf(queryFmtNamespaceAnnotations, env.GetPromClusterFilter(), durStr)
 	resChNamespaceAnnotations := ctx.QueryAtTime(queryNamespaceAnnotations, end)
 
-	queryPodLabels := fmt.Sprintf(queryFmtPodLabels, durStr)
+	queryPodLabels := fmt.Sprintf(queryFmtPodLabels, env.GetPromClusterFilter(), durStr)
 	resChPodLabels := ctx.QueryAtTime(queryPodLabels, end)
 
-	queryPodAnnotations := fmt.Sprintf(queryFmtPodAnnotations, durStr)
+	queryPodAnnotations := fmt.Sprintf(queryFmtPodAnnotations, env.GetPromClusterFilter(), durStr)
 	resChPodAnnotations := ctx.QueryAtTime(queryPodAnnotations, end)
 
-	queryServiceLabels := fmt.Sprintf(queryFmtServiceLabels, durStr)
+	queryServiceLabels := fmt.Sprintf(queryFmtServiceLabels, env.GetPromClusterFilter(), durStr)
 	resChServiceLabels := ctx.QueryAtTime(queryServiceLabels, end)
 
-	queryDeploymentLabels := fmt.Sprintf(queryFmtDeploymentLabels, durStr)
+	queryDeploymentLabels := fmt.Sprintf(queryFmtDeploymentLabels, env.GetPromClusterFilter(), durStr)
 	resChDeploymentLabels := ctx.QueryAtTime(queryDeploymentLabels, end)
 
-	queryStatefulSetLabels := fmt.Sprintf(queryFmtStatefulSetLabels, durStr)
+	queryStatefulSetLabels := fmt.Sprintf(queryFmtStatefulSetLabels, env.GetPromClusterFilter(), durStr)
 	resChStatefulSetLabels := ctx.QueryAtTime(queryStatefulSetLabels, end)
 
-	queryDaemonSetLabels := fmt.Sprintf(queryFmtDaemonSetLabels, durStr, env.GetPromClusterLabel())
+	queryDaemonSetLabels := fmt.Sprintf(queryFmtDaemonSetLabels, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChDaemonSetLabels := ctx.QueryAtTime(queryDaemonSetLabels, end)
 
-	queryPodsWithReplicaSetOwner := fmt.Sprintf(queryFmtPodsWithReplicaSetOwner, durStr, env.GetPromClusterLabel())
+	queryPodsWithReplicaSetOwner := fmt.Sprintf(queryFmtPodsWithReplicaSetOwner, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChPodsWithReplicaSetOwner := ctx.QueryAtTime(queryPodsWithReplicaSetOwner, end)
 
-	queryReplicaSetsWithoutOwners := fmt.Sprintf(queryFmtReplicaSetsWithoutOwners, durStr, env.GetPromClusterLabel())
+	queryReplicaSetsWithoutOwners := fmt.Sprintf(queryFmtReplicaSetsWithoutOwners, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChReplicaSetsWithoutOwners := ctx.QueryAtTime(queryReplicaSetsWithoutOwners, end)
 
-	queryReplicaSetsWithRolloutOwner := fmt.Sprintf(queryFmtReplicaSetsWithRolloutOwner, durStr, env.GetPromClusterLabel())
+	queryReplicaSetsWithRolloutOwner := fmt.Sprintf(queryFmtReplicaSetsWithRolloutOwner, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChReplicaSetsWithRolloutOwner := ctx.QueryAtTime(queryReplicaSetsWithRolloutOwner, end)
 
-	queryJobLabels := fmt.Sprintf(queryFmtJobLabels, durStr, env.GetPromClusterLabel())
+	queryJobLabels := fmt.Sprintf(queryFmtJobLabels, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChJobLabels := ctx.QueryAtTime(queryJobLabels, end)
 
-	queryLBCostPerHr := fmt.Sprintf(queryFmtLBCostPerHr, durStr, env.GetPromClusterLabel())
+	queryLBCostPerHr := fmt.Sprintf(queryFmtLBCostPerHr, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
 	resChLBCostPerHr := ctx.QueryAtTime(queryLBCostPerHr, end)
 
-	queryLBActiveMins := fmt.Sprintf(queryFmtLBActiveMins, env.GetPromClusterLabel(), durStr, resStr)
+	queryLBActiveMins := fmt.Sprintf(queryFmtLBActiveMins, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, resStr)
 	resChLBActiveMins := ctx.QueryAtTime(queryLBActiveMins, end)
 
 	resCPUCoresAllocated, _ := resChCPUCoresAllocated.Await()

pkg/costmodel/allocation_helpers.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/opencost/opencost/pkg/cloud/provider"
 	"github.com/opencost/opencost/pkg/env"
 	"github.com/opencost/opencost/pkg/kubecost"
 	"github.com/opencost/opencost/pkg/log"
@@ -69,9 +69,9 @@ func (cm *CostModel) buildPodMap(window kubecost.Window, resolution, maxBatchSiz
 			var queryPods string
 			// If ingesting UIDs, avg on them
 			if ingestPodUID {
-				queryPods = fmt.Sprintf(queryFmtPodsUID, env.GetPromClusterLabel(), durStr, resStr)
+				queryPods = fmt.Sprintf(queryFmtPodsUID, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, resStr)
 			} else {
-				queryPods = fmt.Sprintf(queryFmtPods, env.GetPromClusterLabel(), durStr, resStr)
+				queryPods = fmt.Sprintf(queryFmtPods, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, resStr)
 			}
 
 			queryProfile := time.Now()
@@ -926,6 +926,11 @@ func applyLabels(podMap map[podKey]*pod, nodeLabels map[nodeKey]map[string]strin
 				allocLabels = make(map[string]string)
 			}
 
+			nsLabels := alloc.Properties.NamespaceLabels
+			if nsLabels == nil {
+				nsLabels = make(map[string]string)
+			}
+
 			// Apply node labels first, then namespace labels, then pod labels
 			// so that pod labels overwrite namespace labels, which overwrite
 			// node labels.
@@ -943,6 +948,7 @@ func applyLabels(podMap map[podKey]*pod, nodeLabels map[nodeKey]map[string]strin
 			if labels, ok := namespaceLabels[nsKey]; ok {
 				for k, v := range labels {
 					allocLabels[k] = v
+					nsLabels[k] = v
 				}
 			}
 
@@ -953,6 +959,8 @@ func applyLabels(podMap map[podKey]*pod, nodeLabels map[nodeKey]map[string]strin
 			}
 
 			alloc.Properties.Labels = allocLabels
+			alloc.Properties.NamespaceLabels = nsLabels
+			
 		}
 	}
 }
@@ -964,11 +972,18 @@ func applyAnnotations(podMap map[podKey]*pod, namespaceAnnotations map[string]ma
 			if allocAnnotations == nil {
 				allocAnnotations = make(map[string]string)
 			}
+
+			nsAnnotations := alloc.Properties.NamespaceAnnotations
+			if nsAnnotations == nil {
+				nsAnnotations = make(map[string]string)
+			}
+
 			// Apply namespace annotations first, then pod annotations so that
 			// pod labels overwrite namespace labels.
 			if labels, ok := namespaceAnnotations[key.Namespace]; ok {
 				for k, v := range labels {
 					allocAnnotations[k] = v
+					nsAnnotations[k] = v
 				}
 			}
 			if labels, ok := podAnnotations[key]; ok {
@@ -978,6 +993,7 @@ func applyAnnotations(podMap map[podKey]*pod, namespaceAnnotations map[string]ma
 			}
 
 			alloc.Properties.Annotations = allocAnnotations
+			alloc.Properties.NamespaceAnnotations = nsAnnotations
 		}
 	}
 }
@@ -1432,7 +1448,7 @@ func applyNodeCostPerCPUHr(nodeMap map[nodeKey]*nodePricing, resNodeCostPerCPUHr
 			nodeMap[key] = &nodePricing{
 				Name:       node,
 				NodeType:   instanceType,
-				ProviderID: cloud.ParseID(providerID),
+				ProviderID: provider.ParseID(providerID),
 			}
 		}
 
@@ -1470,7 +1486,7 @@ func applyNodeCostPerRAMGiBHr(nodeMap map[nodeKey]*nodePricing, resNodeCostPerRA
 			nodeMap[key] = &nodePricing{
 				Name:       node,
 				NodeType:   instanceType,
-				ProviderID: cloud.ParseID(providerID),
+				ProviderID: provider.ParseID(providerID),
 			}
 		}
 
@@ -1508,7 +1524,7 @@ func applyNodeCostPerGPUHr(nodeMap map[nodeKey]*nodePricing, resNodeCostPerGPUHr
 			nodeMap[key] = &nodePricing{
 				Name:       node,
 				NodeType:   instanceType,
-				ProviderID: cloud.ParseID(providerID),
+				ProviderID: provider.ParseID(providerID),
 			}
 		}
 
@@ -1654,7 +1670,7 @@ func (cm *CostModel) getNodePricing(nodeMap map[nodeKey]*nodePricing, nodeKey no
 	if err != nil {
 		log.Warnf("CostModel: failed to load custom pricing: %s", err)
 	}
-	if cloud.CustomPricesEnabled(cm.Provider) && customPricingConfig != nil {
+	if provider.CustomPricesEnabled(cm.Provider) && customPricingConfig != nil {
 		return cm.getCustomNodePricing(node.Preemptible, node.ProviderID)
 	}
 

pkg/costmodel/assets.go

@@ -133,6 +133,17 @@ func (cm *CostModel) ComputeAssets(start, end time.Time) (*kubecost.AssetSet, er
 		node.GPUCost = n.GPUCost
 		node.GPUCount = n.GPUCount
 		node.RAMCost = n.RAMCost
+
+		if n.Overhead != nil {
+			node.Overhead = &kubecost.NodeOverhead{
+				RamOverheadFraction: n.Overhead.RamOverheadFraction,
+				CpuOverheadFraction: n.Overhead.CpuOverheadFraction,
+				OverheadCostFraction: ((n.Overhead.CpuOverheadFraction * n.CPUCost) +
+					(n.Overhead.RamOverheadFraction * n.RAMCost)) / node.TotalCost(),
+			}
+		} else {
+			node.Overhead = &kubecost.NodeOverhead{}
+		}
 		node.Discount = n.Discount
 		if n.Preemptible {
 			node.Preemptible = 1.0

pkg/costmodel/cluster.go

@@ -5,10 +5,10 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/opencost/opencost/pkg/cloud/provider"
 	prometheus "github.com/prometheus/client_golang/api"
 	"golang.org/x/exp/slices"
 
-	"github.com/opencost/opencost/pkg/cloud"
 	"github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/env"
 	"github.com/opencost/opencost/pkg/kubecost"
@@ -19,26 +19,26 @@ import (
 
 const (
 	queryClusterCores = `sum(
-		avg(avg_over_time(kube_node_status_capacity_cpu_cores[%s] %s)) by (node, %s) * avg(avg_over_time(node_cpu_hourly_cost[%s] %s)) by (node, %s) * 730 +
-		avg(avg_over_time(node_gpu_hourly_cost[%s] %s)) by (node, %s) * 730
+		avg(avg_over_time(kube_node_status_capacity_cpu_cores{%s}[%s] %s)) by (node, %s) * avg(avg_over_time(node_cpu_hourly_cost{%s}[%s] %s)) by (node, %s) * 730 +
+		avg(avg_over_time(node_gpu_hourly_cost{%s}[%s] %s)) by (node, %s) * 730
 	  ) by (%s)`
 
 	queryClusterRAM = `sum(
-		avg(avg_over_time(kube_node_status_capacity_memory_bytes[%s] %s)) by (node, %s) / 1024 / 1024 / 1024 * avg(avg_over_time(node_ram_hourly_cost[%s] %s)) by (node, %s) * 730
+		avg(avg_over_time(kube_node_status_capacity_memory_bytes{%s}[%s] %s)) by (node, %s) / 1024 / 1024 / 1024 * avg(avg_over_time(node_ram_hourly_cost{%s}[%s] %s)) by (node, %s) * 730
 	  ) by (%s)`
 
 	queryStorage = `sum(
-		avg(avg_over_time(pv_hourly_cost[%s] %s)) by (persistentvolume, %s) * 730
-		* avg(avg_over_time(kube_persistentvolume_capacity_bytes[%s] %s)) by (persistentvolume, %s) / 1024 / 1024 / 1024
+		avg(avg_over_time(pv_hourly_cost{%s}[%s] %s)) by (persistentvolume, %s) * 730
+		* avg(avg_over_time(kube_persistentvolume_capacity_bytes{%s}[%s] %s)) by (persistentvolume, %s) / 1024 / 1024 / 1024
 	  ) by (%s) %s`
 
-	queryTotal = `sum(avg(node_total_hourly_cost) by (node, %s)) * 730 +
+	queryTotal = `sum(avg(node_total_hourly_cost{%s}) by (node, %s)) * 730 +
 	  sum(
-		avg(avg_over_time(pv_hourly_cost[1h])) by (persistentvolume, %s) * 730
-		* avg(avg_over_time(kube_persistentvolume_capacity_bytes[1h])) by (persistentvolume, %s) / 1024 / 1024 / 1024
+		avg(avg_over_time(pv_hourly_cost{%s}[1h])) by (persistentvolume, %s) * 730
+		* avg(avg_over_time(kube_persistentvolume_capacity_bytes{%s}[1h])) by (persistentvolume, %s) / 1024 / 1024 / 1024
 	  ) by (%s) %s`
 
-	queryNodes = `sum(avg(node_total_hourly_cost) by (node, %s)) * 730 %s`
+	queryNodes = `sum(avg(node_total_hourly_cost{%s}) by (node, %s)) * 730 %s`
 )
 
 const maxLocalDiskSize = 200 // AWS limits root disks to 100 Gi, and occasional metric errors in filesystem size should not contribute to large costs.
@@ -171,19 +171,19 @@ func ClusterDisks(client prometheus.Client, provider models.Provider, start, end
 	costPerGBHr := 0.04 / 730.0
 
 	ctx := prom.NewNamedContext(client, prom.ClusterContextName)
-	queryPVCost := fmt.Sprintf(`avg(avg_over_time(pv_hourly_cost[%s])) by (%s, persistentvolume,provider_id)`, durStr, env.GetPromClusterLabel())
-	queryPVSize := fmt.Sprintf(`avg(avg_over_time(kube_persistentvolume_capacity_bytes[%s])) by (%s, persistentvolume)`, durStr, env.GetPromClusterLabel())
-	queryActiveMins := fmt.Sprintf(`avg(kube_persistentvolume_capacity_bytes) by (%s, persistentvolume)[%s:%dm]`, env.GetPromClusterLabel(), durStr, minsPerResolution)
-	queryPVStorageClass := fmt.Sprintf(`avg(avg_over_time(kubecost_pv_info[%s])) by (%s, persistentvolume, storageclass)`, durStr, env.GetPromClusterLabel())
-	queryPVUsedAvg := fmt.Sprintf(`avg(avg_over_time(kubelet_volume_stats_used_bytes[%s])) by (%s, persistentvolumeclaim, namespace)`, durStr, env.GetPromClusterLabel())
-	queryPVUsedMax := fmt.Sprintf(`max(max_over_time(kubelet_volume_stats_used_bytes[%s])) by (%s, persistentvolumeclaim, namespace)`, durStr, env.GetPromClusterLabel())
-	queryPVCInfo := fmt.Sprintf(`avg(avg_over_time(kube_persistentvolumeclaim_info[%s])) by (%s, volumename, persistentvolumeclaim, namespace)`, durStr, env.GetPromClusterLabel())
-	queryLocalStorageCost := fmt.Sprintf(`sum_over_time(sum(container_fs_limit_bytes{device!="tmpfs", id="/"}) by (instance, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`, env.GetPromClusterLabel(), durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
-	queryLocalStorageUsedCost := fmt.Sprintf(`sum_over_time(sum(container_fs_usage_bytes{device!="tmpfs", id="/"}) by (instance, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`, env.GetPromClusterLabel(), durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
-	queryLocalStorageUsedAvg := fmt.Sprintf(`avg(avg_over_time(container_fs_usage_bytes{device!="tmpfs", id="/"}[%s])) by (instance, %s)`, durStr, env.GetPromClusterLabel())
-	queryLocalStorageUsedMax := fmt.Sprintf(`max(max_over_time(container_fs_usage_bytes{device!="tmpfs", id="/"}[%s])) by (instance, %s)`, durStr, env.GetPromClusterLabel())
-	queryLocalStorageBytes := fmt.Sprintf(`avg_over_time(sum(container_fs_limit_bytes{device!="tmpfs", id="/"}) by (instance, %s)[%s:%dm])`, env.GetPromClusterLabel(), durStr, minsPerResolution)
-	queryLocalActiveMins := fmt.Sprintf(`count(node_total_hourly_cost) by (%s, node)[%s:%dm]`, env.GetPromClusterLabel(), durStr, minsPerResolution)
+	queryPVCost := fmt.Sprintf(`avg(avg_over_time(pv_hourly_cost{%s}[%s])) by (%s, persistentvolume,provider_id)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryPVSize := fmt.Sprintf(`avg(avg_over_time(kube_persistentvolume_capacity_bytes{%s}[%s])) by (%s, persistentvolume)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryActiveMins := fmt.Sprintf(`avg(kube_persistentvolume_capacity_bytes{%s}) by (%s, persistentvolume)[%s:%dm]`, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
+	queryPVStorageClass := fmt.Sprintf(`avg(avg_over_time(kubecost_pv_info{%s}[%s])) by (%s, persistentvolume, storageclass)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryPVUsedAvg := fmt.Sprintf(`avg(avg_over_time(kubelet_volume_stats_used_bytes{%s}[%s])) by (%s, persistentvolumeclaim, namespace)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryPVUsedMax := fmt.Sprintf(`max(max_over_time(kubelet_volume_stats_used_bytes{%s}[%s])) by (%s, persistentvolumeclaim, namespace)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryPVCInfo := fmt.Sprintf(`avg(avg_over_time(kube_persistentvolumeclaim_info{%s}[%s])) by (%s, volumename, persistentvolumeclaim, namespace)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryLocalStorageCost := fmt.Sprintf(`sum_over_time(sum(container_fs_limit_bytes{device!="tmpfs", id="/", %s}) by (instance, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
+	queryLocalStorageUsedCost := fmt.Sprintf(`sum_over_time(sum(container_fs_usage_bytes{device!="tmpfs", id="/", %s}) by (instance, %s)[%s:%dm]) / 1024 / 1024 / 1024 * %f * %f`, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution, hourlyToCumulative, costPerGBHr)
+	queryLocalStorageUsedAvg := fmt.Sprintf(`avg(avg_over_time(container_fs_usage_bytes{device!="tmpfs", id="/", %s}[%s])) by (instance, %s)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryLocalStorageUsedMax := fmt.Sprintf(`max(max_over_time(container_fs_usage_bytes{device!="tmpfs", id="/", %s}[%s])) by (instance, %s)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryLocalStorageBytes := fmt.Sprintf(`avg_over_time(sum(container_fs_limit_bytes{device!="tmpfs", id="/", %s}) by (instance, %s)[%s:%dm])`, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
+	queryLocalActiveMins := fmt.Sprintf(`count(node_total_hourly_cost{%s}) by (%s, node)[%s:%dm]`, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
 
 	resChPVCost := ctx.QueryAtTime(queryPVCost, t)
 	resChPVSize := ctx.QueryAtTime(queryPVSize, t)
@@ -472,6 +472,10 @@ func ClusterDisks(client prometheus.Client, provider models.Provider, start, end
 	return diskMap, nil
 }
 
+type NodeOverhead struct {
+	CpuOverheadFraction float64
+	RamOverheadFraction float64
+}
 type Node struct {
 	Cluster         string
 	Name            string
@@ -494,6 +498,7 @@ type Node struct {
 	CostPerCPUHr    float64
 	CostPerRAMGiBHr float64
 	CostPerGPUHr    float64
+	Overhead        *NodeOverhead
 }
 
 // GKE lies about the number of cores e2 nodes have. This table
@@ -566,24 +571,28 @@ func ClusterNodes(cp models.Provider, client prometheus.Client, start, end time.
 	requiredCtx := prom.NewNamedContext(client, prom.ClusterContextName)
 	optionalCtx := prom.NewNamedContext(client, prom.ClusterOptionalContextName)
 
-	queryNodeCPUHourlyCost := fmt.Sprintf(`avg(avg_over_time(node_cpu_hourly_cost[%s])) by (%s, node, instance_type, provider_id)`, durStr, env.GetPromClusterLabel())
-	queryNodeCPUCores := fmt.Sprintf(`avg(avg_over_time(kube_node_status_capacity_cpu_cores[%s])) by (%s, node)`, durStr, env.GetPromClusterLabel())
-	queryNodeRAMHourlyCost := fmt.Sprintf(`avg(avg_over_time(node_ram_hourly_cost[%s])) by (%s, node, instance_type, provider_id) / 1024 / 1024 / 1024`, durStr, env.GetPromClusterLabel())
-	queryNodeRAMBytes := fmt.Sprintf(`avg(avg_over_time(kube_node_status_capacity_memory_bytes[%s])) by (%s, node)`, durStr, env.GetPromClusterLabel())
-	queryNodeGPUCount := fmt.Sprintf(`avg(avg_over_time(node_gpu_count[%s])) by (%s, node, provider_id)`, durStr, env.GetPromClusterLabel())
-	queryNodeGPUHourlyCost := fmt.Sprintf(`avg(avg_over_time(node_gpu_hourly_cost[%s])) by (%s, node, instance_type, provider_id)`, durStr, env.GetPromClusterLabel())
-	queryNodeCPUModeTotal := fmt.Sprintf(`sum(rate(node_cpu_seconds_total[%s:%dm])) by (kubernetes_node, %s, mode)`, durStr, minsPerResolution, env.GetPromClusterLabel())
-	queryNodeRAMSystemPct := fmt.Sprintf(`sum(sum_over_time(container_memory_working_set_bytes{container_name!="POD",container_name!="",namespace="kube-system"}[%s:%dm])) by (instance, %s) / avg(label_replace(sum(sum_over_time(kube_node_status_capacity_memory_bytes[%s:%dm])) by (node, %s), "instance", "$1", "node", "(.*)")) by (instance, %s)`, durStr, minsPerResolution, env.GetPromClusterLabel(), durStr, minsPerResolution, env.GetPromClusterLabel(), env.GetPromClusterLabel())
-	queryNodeRAMUserPct := fmt.Sprintf(`sum(sum_over_time(container_memory_working_set_bytes{container_name!="POD",container_name!="",namespace!="kube-system"}[%s:%dm])) by (instance, %s) / avg(label_replace(sum(sum_over_time(kube_node_status_capacity_memory_bytes[%s:%dm])) by (node, %s), "instance", "$1", "node", "(.*)")) by (instance, %s)`, durStr, minsPerResolution, env.GetPromClusterLabel(), durStr, minsPerResolution, env.GetPromClusterLabel(), env.GetPromClusterLabel())
-	queryActiveMins := fmt.Sprintf(`avg(node_total_hourly_cost) by (node, %s, provider_id)[%s:%dm]`, env.GetPromClusterLabel(), durStr, minsPerResolution)
-	queryIsSpot := fmt.Sprintf(`avg_over_time(kubecost_node_is_spot[%s:%dm])`, durStr, minsPerResolution)
-	queryLabels := fmt.Sprintf(`count_over_time(kube_node_labels[%s:%dm])`, durStr, minsPerResolution)
+	queryNodeCPUHourlyCost := fmt.Sprintf(`avg(avg_over_time(node_cpu_hourly_cost{%s}[%s])) by (%s, node, instance_type, provider_id)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+  queryNodeCPUCoresCapacity := fmt.Sprintf(`avg(avg_over_time(kube_node_status_capacity_cpu_cores{%s}[%s])) by (%s, node)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryNodeCPUCoresAllocatable := fmt.Sprintf(`avg(avg_over_time(kube_node_status_allocatable_cpu_cores{%s}[%s])) by (%s, node)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryNodeRAMHourlyCost := fmt.Sprintf(`avg(avg_over_time(node_ram_hourly_cost{%s}[%s])) by (%s, node, instance_type, provider_id) / 1024 / 1024 / 1024`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+  queryNodeRAMBytesCapacity := fmt.Sprintf(`avg(avg_over_time(kube_node_status_capacity_memory_bytes{%s}[%s])) by (%s, node)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryNodeRAMBytesAllocatable := fmt.Sprintf(`avg(avg_over_time(kube_node_status_allocatable_memory_bytes{%s}[%s])) by (%s, node)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryNodeGPUCount := fmt.Sprintf(`avg(avg_over_time(node_gpu_count{%s}[%s])) by (%s, node, provider_id)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryNodeGPUHourlyCost := fmt.Sprintf(`avg(avg_over_time(node_gpu_hourly_cost{%s}[%s])) by (%s, node, instance_type, provider_id)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryNodeCPUModeTotal := fmt.Sprintf(`sum(rate(node_cpu_seconds_total{%s}[%s:%dm])) by (kubernetes_node, %s, mode)`, env.GetPromClusterFilter(), durStr, minsPerResolution, env.GetPromClusterLabel())
+	queryNodeRAMSystemPct := fmt.Sprintf(`sum(sum_over_time(container_memory_working_set_bytes{container_name!="POD",container_name!="",namespace="kube-system", %s}[%s:%dm])) by (instance, %s) / avg(label_replace(sum(sum_over_time(kube_node_status_capacity_memory_bytes{%s}[%s:%dm])) by (node, %s), "instance", "$1", "node", "(.*)")) by (instance, %s)`, env.GetPromClusterFilter(), durStr, minsPerResolution, env.GetPromClusterLabel(), env.GetPromClusterFilter(), durStr, minsPerResolution, env.GetPromClusterLabel(), env.GetPromClusterLabel())
+	queryNodeRAMUserPct := fmt.Sprintf(`sum(sum_over_time(container_memory_working_set_bytes{container_name!="POD",container_name!="",namespace!="kube-system", %s}[%s:%dm])) by (instance, %s) / avg(label_replace(sum(sum_over_time(kube_node_status_capacity_memory_bytes{%s}[%s:%dm])) by (node, %s), "instance", "$1", "node", "(.*)")) by (instance, %s)`, env.GetPromClusterFilter(), durStr, minsPerResolution, env.GetPromClusterLabel(), env.GetPromClusterFilter(), durStr, minsPerResolution, env.GetPromClusterLabel(), env.GetPromClusterLabel())
+	queryActiveMins := fmt.Sprintf(`avg(node_total_hourly_cost{%s}) by (node, %s, provider_id)[%s:%dm]`, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
+	queryIsSpot := fmt.Sprintf(`avg_over_time(kubecost_node_is_spot{%s}[%s:%dm])`, env.GetPromClusterFilter(), durStr, minsPerResolution)
+	queryLabels := fmt.Sprintf(`count_over_time(kube_node_labels{%s}[%s:%dm])`, env.GetPromClusterFilter(), durStr, minsPerResolution)
 
 	// Return errors if these fail
 	resChNodeCPUHourlyCost := requiredCtx.QueryAtTime(queryNodeCPUHourlyCost, t)
-	resChNodeCPUCores := requiredCtx.QueryAtTime(queryNodeCPUCores, t)
+	resChNodeCPUCoresCapacity := requiredCtx.QueryAtTime(queryNodeCPUCoresCapacity, t)
+	resChNodeCPUCoresAllocatable := requiredCtx.QueryAtTime(queryNodeCPUCoresAllocatable, t)
 	resChNodeRAMHourlyCost := requiredCtx.QueryAtTime(queryNodeRAMHourlyCost, t)
-	resChNodeRAMBytes := requiredCtx.QueryAtTime(queryNodeRAMBytes, t)
+	resChNodeRAMBytesCapacity := requiredCtx.QueryAtTime(queryNodeRAMBytesCapacity, t)
+	resChNodeRAMBytesAllocatable := requiredCtx.QueryAtTime(queryNodeRAMBytesAllocatable, t)
 	resChNodeGPUCount := requiredCtx.QueryAtTime(queryNodeGPUCount, t)
 	resChNodeGPUHourlyCost := requiredCtx.QueryAtTime(queryNodeGPUHourlyCost, t)
 	resChActiveMins := requiredCtx.QueryAtTime(queryActiveMins, t)
@@ -596,11 +605,13 @@ func ClusterNodes(cp models.Provider, client prometheus.Client, start, end time.
 	resChLabels := optionalCtx.QueryAtTime(queryLabels, t)
 
 	resNodeCPUHourlyCost, _ := resChNodeCPUHourlyCost.Await()
-	resNodeCPUCores, _ := resChNodeCPUCores.Await()
+	resNodeCPUCoresCapacity, _ := resChNodeCPUCoresCapacity.Await()
+	resNodeCPUCoresAllocatable, _ := resChNodeCPUCoresAllocatable.Await()
 	resNodeGPUCount, _ := resChNodeGPUCount.Await()
 	resNodeGPUHourlyCost, _ := resChNodeGPUHourlyCost.Await()
 	resNodeRAMHourlyCost, _ := resChNodeRAMHourlyCost.Await()
-	resNodeRAMBytes, _ := resChNodeRAMBytes.Await()
+	resNodeRAMBytesCapacity, _ := resChNodeRAMBytesCapacity.Await()
+	resNodeRAMBytesAllocatable, _ := resChNodeRAMBytesAllocatable.Await()
 	resIsSpot, _ := resChIsSpot.Await()
 	resNodeCPUModeTotal, _ := resChNodeCPUModeTotal.Await()
 	resNodeRAMSystemPct, _ := resChNodeRAMSystemPct.Await()
@@ -633,8 +644,12 @@ func ClusterNodes(cp models.Provider, client prometheus.Client, start, end time.
 	clusterAndNameToTypeIntermediate := mergeTypeMaps(clusterAndNameToType1, clusterAndNameToType2)
 	clusterAndNameToType := mergeTypeMaps(clusterAndNameToTypeIntermediate, clusterAndNameToType3)
 
-	cpuCoresMap := buildCPUCoresMap(resNodeCPUCores)
-	ramBytesMap := buildRAMBytesMap(resNodeRAMBytes)
+	cpuCoresCapacityMap := buildCPUCoresMap(resNodeCPUCoresCapacity)
+	ramBytesCapacityMap := buildRAMBytesMap(resNodeRAMBytesCapacity)
+
+	cpuCoresAllocatableMap := buildCPUCoresMap(resNodeCPUCoresAllocatable)
+	ramBytesAllocatableMap := buildRAMBytesMap(resNodeRAMBytesAllocatable)
+	overheadMap := buildOverheadMap(ramBytesCapacityMap, ramBytesAllocatableMap, cpuCoresCapacityMap, cpuCoresAllocatableMap)
 
 	ramUserPctMap := buildRAMUserPctMap(resNodeRAMUserPct)
 	ramSystemPctMap := buildRAMSystemPctMap(resNodeRAMSystemPct)
@@ -643,13 +658,13 @@ func ClusterNodes(cp models.Provider, client prometheus.Client, start, end time.
 
 	labelsMap := buildLabelsMap(resLabels)
 
-	costTimesMinuteAndCount(activeDataMap, cpuCostMap, cpuCoresMap)
-	costTimesMinuteAndCount(activeDataMap, ramCostMap, ramBytesMap)
+	costTimesMinuteAndCount(activeDataMap, cpuCostMap, cpuCoresCapacityMap)
+	costTimesMinuteAndCount(activeDataMap, ramCostMap, ramBytesCapacityMap)
 	costTimesMinute(activeDataMap, gpuCostMap) // there's no need to do a weird "nodeIdentifierNoProviderID" type match since gpuCounts have a providerID
 
 	nodeMap := buildNodeMap(
 		cpuCostMap, ramCostMap, gpuCostMap, gpuCountMap,
-		cpuCoresMap, ramBytesMap, ramUserPctMap,
+		cpuCoresCapacityMap, ramBytesCapacityMap, ramUserPctMap,
 		ramSystemPctMap,
 		cpuBreakdownMap,
 		activeDataMap,
@@ -657,6 +672,7 @@ func ClusterNodes(cp models.Provider, client prometheus.Client, start, end time.
 		labelsMap,
 		clusterAndNameToType,
 		resolution,
+		overheadMap,
 	)
 
 	c, err := cp.GetConfig()
@@ -726,8 +742,8 @@ func ClusterLoadBalancers(client prometheus.Client, start, end time.Time) (map[L
 
 	ctx := prom.NewNamedContext(client, prom.ClusterContextName)
 
-	queryLBCost := fmt.Sprintf(`avg(avg_over_time(kubecost_load_balancer_cost[%s])) by (namespace, service_name, %s, ingress_ip)`, durStr, env.GetPromClusterLabel())
-	queryActiveMins := fmt.Sprintf(`avg(kubecost_load_balancer_cost) by (namespace, service_name, %s, ingress_ip)[%s:%dm]`, env.GetPromClusterLabel(), durStr, minsPerResolution)
+	queryLBCost := fmt.Sprintf(`avg(avg_over_time(kubecost_load_balancer_cost{%s}[%s])) by (namespace, service_name, %s, ingress_ip)`, env.GetPromClusterFilter(), durStr, env.GetPromClusterLabel())
+	queryActiveMins := fmt.Sprintf(`avg(kubecost_load_balancer_cost{%s}) by (namespace, service_name, %s, ingress_ip)[%s:%dm]`, env.GetPromClusterFilter(), env.GetPromClusterLabel(), durStr, minsPerResolution)
 
 	resChLBCost := ctx.QueryAtTime(queryLBCost, t)
 	resChActiveMins := ctx.QueryAtTime(queryActiveMins, t)
@@ -779,7 +795,7 @@ func ClusterLoadBalancers(client prometheus.Client, start, end time.Time) (map[L
 				Cluster:    cluster,
 				Namespace:  namespace,
 				Name:       fmt.Sprintf("%s/%s", namespace, name), // TODO:ETL this is kept for backwards-compatibility, but not good
-				ProviderID: cloud.ParseLBID(providerID),
+				ProviderID: provider.ParseLBID(providerID),
 			}
 		}
 
@@ -862,49 +878,49 @@ func (a *Accesses) ComputeClusterCosts(client prometheus.Client, provider models
 	hourlyToCumulative := float64(minsPerResolution) * (1.0 / 60.0)
 
 	const fmtQueryDataCount = `
-		count_over_time(sum(kube_node_status_capacity_cpu_cores) by (%s)[%s:%dm]%s) * %d
+		count_over_time(sum(kube_node_status_capacity_cpu_cores{%s}) by (%s)[%s:%dm]%s) * %d
 	`
 
 	const fmtQueryTotalGPU = `
 		sum(
-			sum_over_time(node_gpu_hourly_cost[%s:%dm]%s) * %f
+			sum_over_time(node_gpu_hourly_cost{%s}[%s:%dm]%s) * %f
 		) by (%s)
 	`
 
 	const fmtQueryTotalCPU = `
 		sum(
-			sum_over_time(avg(kube_node_status_capacity_cpu_cores) by (node, %s)[%s:%dm]%s) *
-			avg(avg_over_time(node_cpu_hourly_cost[%s:%dm]%s)) by (node, %s) * %f
+			sum_over_time(avg(kube_node_status_capacity_cpu_cores{%s}) by (node, %s)[%s:%dm]%s) *
+			avg(avg_over_time(node_cpu_hourly_cost{%s}[%s:%dm]%s)) by (node, %s) * %f
 		) by (%s)
 	`
 
 	const fmtQueryTotalRAM = `
 		sum(
-			sum_over_time(avg(kube_node_status_capacity_memory_bytes) by (node, %s)[%s:%dm]%s) / 1024 / 1024 / 1024 *
-			avg(avg_over_time(node_ram_hourly_cost[%s:%dm]%s)) by (node, %s) * %f
+			sum_over_time(avg(kube_node_status_capacity_memory_bytes{%s}) by (node, %s)[%s:%dm]%s) / 1024 / 1024 / 1024 *
+			avg(avg_over_time(node_ram_hourly_cost{%s}[%s:%dm]%s)) by (node, %s) * %f
 		) by (%s)
 	`
 
 	const fmtQueryTotalStorage = `
 		sum(
-			sum_over_time(avg(kube_persistentvolume_capacity_bytes) by (persistentvolume, %s)[%s:%dm]%s) / 1024 / 1024 / 1024 *
-			avg(avg_over_time(pv_hourly_cost[%s:%dm]%s)) by (persistentvolume, %s) * %f
+			sum_over_time(avg(kube_persistentvolume_capacity_bytes{%s}) by (persistentvolume, %s)[%s:%dm]%s) / 1024 / 1024 / 1024 *
+			avg(avg_over_time(pv_hourly_cost{%s}[%s:%dm]%s)) by (persistentvolume, %s) * %f
 		) by (%s)
 	`
 
 	const fmtQueryCPUModePct = `
-		sum(rate(node_cpu_seconds_total[%s]%s)) by (%s, mode) / ignoring(mode)
-		group_left sum(rate(node_cpu_seconds_total[%s]%s)) by (%s)
+		sum(rate(node_cpu_seconds_total{%s}[%s]%s)) by (%s, mode) / ignoring(mode)
+		group_left sum(rate(node_cpu_seconds_total{%s}[%s]%s)) by (%s)
 	`
 
 	const fmtQueryRAMSystemPct = `
-		sum(sum_over_time(container_memory_usage_bytes{container_name!="",namespace="kube-system"}[%s:%dm]%s)) by (%s)
-		/ sum(sum_over_time(kube_node_status_capacity_memory_bytes[%s:%dm]%s)) by (%s)
+		sum(sum_over_time(container_memory_usage_bytes{container_name!="",namespace="kube-system", %s}[%s:%dm]%s)) by (%s)
+		/ sum(sum_over_time(kube_node_status_capacity_memory_bytes{%s}[%s:%dm]%s)) by (%s)
 	`
 
 	const fmtQueryRAMUserPct = `
-		sum(sum_over_time(kubecost_cluster_memory_working_set_bytes[%s:%dm]%s)) by (%s)
-		/ sum(sum_over_time(kube_node_status_capacity_memory_bytes[%s:%dm]%s)) by (%s)
+		sum(sum_over_time(kubecost_cluster_memory_working_set_bytes{%s}[%s:%dm]%s)) by (%s)
+		/ sum(sum_over_time(kube_node_status_capacity_memory_bytes{%s}[%s:%dm]%s)) by (%s)
 	`
 
 	// TODO niko/clustercost metric "kubelet_volume_stats_used_bytes" was deprecated in 1.12, then seems to have come back in 1.17
@@ -920,11 +936,11 @@ func (a *Accesses) ComputeClusterCosts(client prometheus.Client, provider models
 
 	fmtOffset := timeutil.DurationToPromOffsetString(offset)
 
-	queryDataCount := fmt.Sprintf(fmtQueryDataCount, env.GetPromClusterLabel(), windowStr, minsPerResolution, fmtOffset, minsPerResolution)
-	queryTotalGPU := fmt.Sprintf(fmtQueryTotalGPU, windowStr, minsPerResolution, fmtOffset, hourlyToCumulative, env.GetPromClusterLabel())
-	queryTotalCPU := fmt.Sprintf(fmtQueryTotalCPU, env.GetPromClusterLabel(), windowStr, minsPerResolution, fmtOffset, windowStr, minsPerResolution, fmtOffset, env.GetPromClusterLabel(), hourlyToCumulative, env.GetPromClusterLabel())
-	queryTotalRAM := fmt.Sprintf(fmtQueryTotalRAM, env.GetPromClusterLabel(), windowStr, minsPerResolution, fmtOffset, windowStr, minsPerResolution, fmtOffset, env.GetPromClusterLabel(), hourlyToCumulative, env.GetPromClusterLabel())
-	queryTotalStorage := fmt.Sprintf(fmtQueryTotalStorage, env.GetPromClusterLabel(), windowStr, minsPerResolution, fmtOffset, windowStr, minsPerResolution, fmtOffset, env.GetPromClusterLabel(), hourlyToCumulative, env.GetPromClusterLabel())
+	queryDataCount := fmt.Sprintf(fmtQueryDataCount, env.GetPromClusterFilter(), env.GetPromClusterLabel(), windowStr, minsPerResolution, fmtOffset, minsPerResolution)
+	queryTotalGPU := fmt.Sprintf(fmtQueryTotalGPU, env.GetPromClusterFilter(), windowStr, minsPerResolution, fmtOffset, hourlyToCumulative, env.GetPromClusterLabel())
+	queryTotalCPU := fmt.Sprintf(fmtQueryTotalCPU, env.GetPromClusterFilter(), env.GetPromClusterLabel(), windowStr, minsPerResolution, fmtOffset, env.GetPromClusterFilter(), windowStr, minsPerResolution, fmtOffset, env.GetPromClusterLabel(), hourlyToCumulative, env.GetPromClusterLabel())
+	queryTotalRAM := fmt.Sprintf(fmtQueryTotalRAM, env.GetPromClusterFilter(), env.GetPromClusterLabel(), windowStr, minsPerResolution, fmtOffset, env.GetPromClusterFilter(), windowStr, minsPerResolution, fmtOffset, env.GetPromClusterLabel(), hourlyToCumulative, env.GetPromClusterLabel())
+	queryTotalStorage := fmt.Sprintf(fmtQueryTotalStorage, env.GetPromClusterFilter(), env.GetPromClusterLabel(), windowStr, minsPerResolution, fmtOffset, env.GetPromClusterFilter(), windowStr, minsPerResolution, fmtOffset, env.GetPromClusterLabel(), hourlyToCumulative, env.GetPromClusterLabel())
 
 	ctx := prom.NewNamedContext(client, prom.ClusterContextName)
 
@@ -946,9 +962,9 @@ func (a *Accesses) ComputeClusterCosts(client prometheus.Client, provider models
 	}
 
 	if withBreakdown {
-		queryCPUModePct := fmt.Sprintf(fmtQueryCPUModePct, windowStr, fmtOffset, env.GetPromClusterLabel(), windowStr, fmtOffset, env.GetPromClusterLabel())
-		queryRAMSystemPct := fmt.Sprintf(fmtQueryRAMSystemPct, windowStr, minsPerResolution, fmtOffset, env.GetPromClusterLabel(), windowStr, minsPerResolution, fmtOffset, env.GetPromClusterLabel())
-		queryRAMUserPct := fmt.Sprintf(fmtQueryRAMUserPct, windowStr, minsPerResolution, fmtOffset, env.GetPromClusterLabel(), windowStr, minsPerResolution, fmtOffset, env.GetPromClusterLabel())
+		queryCPUModePct := fmt.Sprintf(fmtQueryCPUModePct, env.GetPromClusterFilter(), windowStr, fmtOffset, env.GetPromClusterLabel(), env.GetPromClusterFilter(), windowStr, fmtOffset, env.GetPromClusterLabel())
+		queryRAMSystemPct := fmt.Sprintf(fmtQueryRAMSystemPct, env.GetPromClusterFilter(), windowStr, minsPerResolution, fmtOffset, env.GetPromClusterLabel(), env.GetPromClusterFilter(), windowStr, minsPerResolution, fmtOffset, env.GetPromClusterLabel())
+		queryRAMUserPct := fmt.Sprintf(fmtQueryRAMUserPct, env.GetPromClusterFilter(), windowStr, minsPerResolution, fmtOffset, env.GetPromClusterLabel(), env.GetPromClusterFilter(), windowStr, minsPerResolution, fmtOffset, env.GetPromClusterLabel())
 
 		bdResChs := ctx.QueryAll(
 			queryCPUModePct,
@@ -1220,10 +1236,10 @@ func ClusterCostsOverTime(cli prometheus.Client, provider models.Provider, start
 
 	fmtOffset := timeutil.DurationToPromOffsetString(offset)
 
-	qCores := fmt.Sprintf(queryClusterCores, fmtWindow, fmtOffset, env.GetPromClusterLabel(), fmtWindow, fmtOffset, env.GetPromClusterLabel(), fmtWindow, fmtOffset, env.GetPromClusterLabel(), env.GetPromClusterLabel())
-	qRAM := fmt.Sprintf(queryClusterRAM, fmtWindow, fmtOffset, env.GetPromClusterLabel(), fmtWindow, fmtOffset, env.GetPromClusterLabel(), env.GetPromClusterLabel())
-	qStorage := fmt.Sprintf(queryStorage, fmtWindow, fmtOffset, env.GetPromClusterLabel(), fmtWindow, fmtOffset, env.GetPromClusterLabel(), env.GetPromClusterLabel(), localStorageQuery)
-	qTotal := fmt.Sprintf(queryTotal, env.GetPromClusterLabel(), env.GetPromClusterLabel(), env.GetPromClusterLabel(), env.GetPromClusterLabel(), localStorageQuery)
+	qCores := fmt.Sprintf(queryClusterCores, env.GetPromClusterFilter(), fmtWindow, fmtOffset, env.GetPromClusterLabel(), env.GetPromClusterFilter(), fmtWindow, fmtOffset, env.GetPromClusterLabel(), env.GetPromClusterFilter(), fmtWindow, fmtOffset, env.GetPromClusterLabel(), env.GetPromClusterLabel())
+	qRAM := fmt.Sprintf(queryClusterRAM, env.GetPromClusterFilter(), fmtWindow, fmtOffset, env.GetPromClusterLabel(), env.GetPromClusterFilter(), fmtWindow, fmtOffset, env.GetPromClusterLabel(), env.GetPromClusterLabel())
+	qStorage := fmt.Sprintf(queryStorage, env.GetPromClusterFilter(), fmtWindow, fmtOffset, env.GetPromClusterLabel(), env.GetPromClusterFilter(), fmtWindow, fmtOffset, env.GetPromClusterLabel(), env.GetPromClusterLabel(), localStorageQuery)
+	qTotal := fmt.Sprintf(queryTotal, env.GetPromClusterFilter(), env.GetPromClusterLabel(), env.GetPromClusterFilter(), env.GetPromClusterLabel(), env.GetPromClusterFilter(), env.GetPromClusterLabel(), env.GetPromClusterLabel(), localStorageQuery)
 
 	ctx := prom.NewNamedContext(cli, prom.ClusterContextName)
 	resChClusterCores := ctx.QueryRange(qCores, start, end, window)
@@ -1273,7 +1289,7 @@ func ClusterCostsOverTime(cli prometheus.Client, provider models.Provider, start
 		// If clusterTotal query failed, it's likely because there are no PVs, which
 		// causes the qTotal query to return no data. Instead, query only node costs.
 		// If that fails, return an error because something is actually wrong.
-		qNodes := fmt.Sprintf(queryNodes, env.GetPromClusterLabel(), localStorageQuery)
+		qNodes := fmt.Sprintf(queryNodes, env.GetPromClusterFilter(), env.GetPromClusterLabel(), localStorageQuery)
 
 		resultNodes, warnings, err := ctx.QueryRangeSync(qNodes, start, end, window)
 		for _, warning := range warnings {
@@ -1360,7 +1376,7 @@ func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActi
 		diskMap[key].Bytes = bytes
 	}
 
-	customPricingEnabled := cloud.CustomPricesEnabled(cp)
+	customPricingEnabled := provider.CustomPricesEnabled(cp)
 	customPricingConfig, err := cp.GetConfig()
 	if err != nil {
 		log.Warnf("ClusterDisks: failed to load custom pricing: %s", err)
@@ -1405,7 +1421,7 @@ func pvCosts(diskMap map[DiskIdentifier]*Disk, resolution time.Duration, resActi
 		diskMap[key].Cost = cost * (diskMap[key].Bytes / 1024 / 1024 / 1024) * (diskMap[key].Minutes / 60)
 		providerID, _ := result.GetString("provider_id") // just put the providerID set up here, it's the simplest query.
 		if providerID != "" {
-			diskMap[key].ProviderID = cloud.ParsePVID(providerID)
+			diskMap[key].ProviderID = provider.ParsePVID(providerID)
 		}
 	}
 

pkg/costmodel/cluster_helpers.go

@@ -4,8 +4,8 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/opencost/opencost/pkg/cloud"
 	"github.com/opencost/opencost/pkg/cloud/models"
+	"github.com/opencost/opencost/pkg/cloud/provider"
 
 	"github.com/opencost/opencost/pkg/env"
 	"github.com/opencost/opencost/pkg/log"
@@ -41,7 +41,7 @@ func buildCPUCostMap(
 	cpuCostMap := make(map[NodeIdentifier]float64)
 	clusterAndNameToType := make(map[nodeIdentifierNoProviderID]string)
 
-	customPricingEnabled := cloud.CustomPricesEnabled(cp)
+	customPricingEnabled := provider.CustomPricesEnabled(cp)
 	customPricingConfig, err := cp.GetConfig()
 	if err != nil {
 		log.Warnf("ClusterNodes: failed to load custom pricing: %s", err)
@@ -65,7 +65,7 @@ func buildCPUCostMap(
 		key := NodeIdentifier{
 			Cluster:    cluster,
 			Name:       name,
-			ProviderID: cloud.ParseID(providerID),
+			ProviderID: provider.ParseID(providerID),
 		}
 		keyNon := nodeIdentifierNoProviderID{
 			Cluster: cluster,
@@ -115,7 +115,7 @@ func buildRAMCostMap(
 	ramCostMap := make(map[NodeIdentifier]float64)
 	clusterAndNameToType := make(map[nodeIdentifierNoProviderID]string)
 
-	customPricingEnabled := cloud.CustomPricesEnabled(cp)
+	customPricingEnabled := provider.CustomPricesEnabled(cp)
 	customPricingConfig, err := cp.GetConfig()
 	if err != nil {
 		log.Warnf("ClusterNodes: failed to load custom pricing: %s", err)
@@ -139,7 +139,7 @@ func buildRAMCostMap(
 		key := NodeIdentifier{
 			Cluster:    cluster,
 			Name:       name,
-			ProviderID: cloud.ParseID(providerID),
+			ProviderID: provider.ParseID(providerID),
 		}
 		keyNon := nodeIdentifierNoProviderID{
 			Cluster: cluster,
@@ -190,7 +190,7 @@ func buildGPUCostMap(
 	gpuCostMap := make(map[NodeIdentifier]float64)
 	clusterAndNameToType := make(map[nodeIdentifierNoProviderID]string)
 
-	customPricingEnabled := cloud.CustomPricesEnabled(cp)
+	customPricingEnabled := provider.CustomPricesEnabled(cp)
 	customPricingConfig, err := cp.GetConfig()
 	if err != nil {
 		log.Warnf("ClusterNodes: failed to load custom pricing: %s", err)
@@ -214,7 +214,7 @@ func buildGPUCostMap(
 		key := NodeIdentifier{
 			Cluster:    cluster,
 			Name:       name,
-			ProviderID: cloud.ParseID(providerID),
+			ProviderID: provider.ParseID(providerID),
 		}
 		keyNon := nodeIdentifierNoProviderID{
 			Cluster: cluster,
@@ -282,7 +282,7 @@ func buildGPUCountMap(
 		key := NodeIdentifier{
 			Cluster:    cluster,
 			Name:       name,
-			ProviderID: cloud.ParseID(providerID),
+			ProviderID: provider.ParseID(providerID),
 		}
 		gpuCountMap[key] = gpuCount
 	}
@@ -426,6 +426,43 @@ func buildCPUBreakdownMap(resNodeCPUModeTotal []*prom.QueryResult) map[nodeIdent
 	return cpuBreakdownMap
 }
 
+func buildOverheadMap(capRam, allocRam, capCPU, allocCPU map[nodeIdentifierNoProviderID]float64) map[nodeIdentifierNoProviderID]*NodeOverhead {
+	m := make(map[nodeIdentifierNoProviderID]*NodeOverhead, len(capRam))
+
+	for identifier, ramCapacity := range capRam {
+		allocatableRam, ok := allocRam[identifier]
+		if !ok {
+			log.Warnf("Could not find allocatable ram for node %s", identifier.Name)
+			continue
+		}
+		overheadBytes := ramCapacity - allocatableRam
+		m[identifier] = &NodeOverhead{
+			RamOverheadFraction: overheadBytes / ramCapacity,
+		}
+	}
+
+	for identifier, cpuCapacity := range capCPU {
+		allocatableCPU, ok := allocCPU[identifier]
+		if !ok {
+			log.Warnf("Could not find allocatable cpu for node %s", identifier.Name)
+			continue
+		}
+
+		overhead := cpuCapacity - allocatableCPU
+
+		if _, found := m[identifier]; found {
+			m[identifier].CpuOverheadFraction = overhead / cpuCapacity
+		} else {
+			m[identifier] = &NodeOverhead{
+				CpuOverheadFraction: overhead / cpuCapacity,
+			}
+		}
+
+	}
+
+	return m
+}
+
 func buildRAMUserPctMap(resNodeRAMUserPct []*prom.QueryResult) map[nodeIdentifierNoProviderID]float64 {
 
 	m := make(map[nodeIdentifierNoProviderID]float64)
@@ -511,7 +548,7 @@ func buildActiveDataMap(resActiveMins []*prom.QueryResult, resolution time.Durat
 		key := NodeIdentifier{
 			Cluster:    cluster,
 			Name:       name,
-			ProviderID: cloud.ParseID(providerID),
+			ProviderID: provider.ParseID(providerID),
 		}
 
 		if len(result.Values) == 0 {
@@ -560,7 +597,7 @@ func buildPreemptibleMap(
 		key := NodeIdentifier{
 			Cluster:    cluster,
 			Name:       nodeName,
-			ProviderID: cloud.ParseID(providerID),
+			ProviderID: provider.ParseID(providerID),
 		}
 
 		// TODO(michaelmdresser): check this condition at merge time?
@@ -707,6 +744,7 @@ func buildNodeMap(
 	labelsMap map[nodeIdentifierNoProviderID]map[string]string,
 	clusterAndNameToType map[nodeIdentifierNoProviderID]string,
 	res time.Duration,
+	overheadMap map[nodeIdentifierNoProviderID]*NodeOverhead,
 ) map[NodeIdentifier]*Node {
 
 	nodeMap := make(map[NodeIdentifier]*Node)
@@ -784,6 +822,16 @@ func buildNodeMap(
 		if labels, ok := labelsMap[clusterAndNameID]; ok {
 			nodePtr.Labels = labels
 		}
+
+		if overhead, ok := overheadMap[clusterAndNameID]; ok {
+			nodePtr.Overhead = overhead
+		} else {
+			// we were unable to compute overhead for this node
+			// assume default case of no overhead
+			nodePtr.Overhead = &NodeOverhead{}
+			log.Warnf("unable to compute overhead for node %s - defaulting to no overhead", clusterAndNameID.Name)
+		}
+
 	}
 
 	return nodeMap

pkg/costmodel/cluster_helpers_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/opencost/opencost/pkg/cloud"
+	"github.com/opencost/opencost/pkg/cloud/provider"
 	"github.com/opencost/opencost/pkg/config"
 	"github.com/opencost/opencost/pkg/prom"
 	"github.com/opencost/opencost/pkg/util"
@@ -150,6 +150,7 @@ func TestBuildNodeMap(t *testing.T) {
 		labelsMap            map[nodeIdentifierNoProviderID]map[string]string
 		clusterAndNameToType map[nodeIdentifierNoProviderID]string
 		expected             map[NodeIdentifier]*Node
+		overheadMap          map[nodeIdentifierNoProviderID]*NodeOverhead
 	}{
 		{
 			name:     "empty",
@@ -183,6 +184,7 @@ func TestBuildNodeMap(t *testing.T) {
 					CPUCost:      0.048,
 					CPUBreakdown: &ClusterCostsBreakdown{},
 					RAMBreakdown: &ClusterCostsBreakdown{},
+					Overhead:     &NodeOverhead{},
 				},
 			},
 		},
@@ -211,6 +213,7 @@ func TestBuildNodeMap(t *testing.T) {
 					CPUCost:      0.048,
 					CPUBreakdown: &ClusterCostsBreakdown{},
 					RAMBreakdown: &ClusterCostsBreakdown{},
+					Overhead:     &NodeOverhead{},
 				},
 			},
 		},
@@ -247,6 +250,7 @@ func TestBuildNodeMap(t *testing.T) {
 					CPUCost:      0.048,
 					CPUBreakdown: &ClusterCostsBreakdown{},
 					RAMBreakdown: &ClusterCostsBreakdown{},
+					Overhead:     &NodeOverhead{},
 				},
 				{
 					Cluster:    "cluster1",
@@ -260,6 +264,7 @@ func TestBuildNodeMap(t *testing.T) {
 					CPUCost:      0.087,
 					CPUBreakdown: &ClusterCostsBreakdown{},
 					RAMBreakdown: &ClusterCostsBreakdown{},
+					Overhead:     &NodeOverhead{},
 				},
 			},
 		},
@@ -489,6 +494,7 @@ func TestBuildNodeMap(t *testing.T) {
 					End:         time.Date(2020, 6, 16, 9, 20, 39, 0, time.UTC),
 					Minutes:     5*60 + 35 + (11.0 / 60.0),
 					Preemptible: true,
+					Overhead:    &NodeOverhead{},
 					Labels: map[string]string{
 						"labelname1_A": "labelvalue1_A",
 						"labelname1_B": "labelvalue1_B",
@@ -525,6 +531,7 @@ func TestBuildNodeMap(t *testing.T) {
 						"labelname1_A": "labelvalue1_A",
 						"labelname1_B": "labelvalue1_B",
 					},
+					Overhead: &NodeOverhead{},
 				},
 				{
 					Cluster:    "cluster1",
@@ -557,6 +564,7 @@ func TestBuildNodeMap(t *testing.T) {
 						"labelname2_A": "labelvalue2_A",
 						"labelname2_B": "labelvalue2_B",
 					},
+					Overhead: &NodeOverhead{},
 				},
 			},
 		},
@@ -595,6 +603,7 @@ func TestBuildNodeMap(t *testing.T) {
 					CPUCores:     partialCPUMap["e2-micro"],
 					CPUBreakdown: &ClusterCostsBreakdown{},
 					RAMBreakdown: &ClusterCostsBreakdown{},
+					Overhead:     &NodeOverhead{},
 				},
 			},
 		},
@@ -633,6 +642,7 @@ func TestBuildNodeMap(t *testing.T) {
 					CPUCores:     partialCPUMap["e2-small"],
 					CPUBreakdown: &ClusterCostsBreakdown{},
 					RAMBreakdown: &ClusterCostsBreakdown{},
+					Overhead:     &NodeOverhead{},
 				},
 			},
 		},
@@ -657,6 +667,15 @@ func TestBuildNodeMap(t *testing.T) {
 					Name:    "node1",
 				}: "e2-medium", // for this node type
 			},
+			overheadMap: map[nodeIdentifierNoProviderID]*NodeOverhead{
+				{
+					Cluster: "cluster1",
+					Name:    "node1",
+				}: {
+					CpuOverheadFraction: 0.5,
+					RamOverheadFraction: 0.25,
+				}, // for this node type
+			},
 			expected: map[NodeIdentifier]*Node{
 				{
 					Cluster:    "cluster1",
@@ -671,6 +690,10 @@ func TestBuildNodeMap(t *testing.T) {
 					CPUCores:     partialCPUMap["e2-medium"],
 					CPUBreakdown: &ClusterCostsBreakdown{},
 					RAMBreakdown: &ClusterCostsBreakdown{},
+					Overhead: &NodeOverhead{
+						CpuOverheadFraction: 0.5,
+						RamOverheadFraction: 0.25,
+					},
 				},
 			},
 		},
@@ -688,6 +711,7 @@ func TestBuildNodeMap(t *testing.T) {
 				testCase.labelsMap,
 				testCase.clusterAndNameToType,
 				time.Minute,
+				testCase.overheadMap,
 			)
 
 			if !reflect.DeepEqual(result, testCase.expected) {
@@ -853,8 +877,8 @@ func TestBuildGPUCostMap(t *testing.T) {
 
 	for _, testCase := range cases {
 		t.Run(testCase.name, func(t *testing.T) {
-			testProvider := &cloud.CustomProvider{
-				Config: cloud.NewProviderConfig(config.NewConfigFileManager(nil), "fakeFile"),
+			testProvider := &provider.CustomProvider{
+				Config: provider.NewProviderConfig(config.NewConfigFileManager(nil), "fakeFile"),
 			}
 			testPreemptible := make(map[NodeIdentifier]bool)
 			result, _ := buildGPUCostMap(testCase.promResult, testCase.countMap, testProvider, testPreemptible)
@@ -1042,8 +1066,8 @@ func TestAssetCustompricing(t *testing.T) {
 
 	for _, testCase := range cases {
 		t.Run(testCase.name, func(t *testing.T) {
-			testProvider := &cloud.CustomProvider{
-				Config: cloud.NewProviderConfig(config.NewConfigFileManager(nil), ""),
+			testProvider := &provider.CustomProvider{
+				Config: provider.NewProviderConfig(config.NewConfigFileManager(nil), ""),
 			}
 			testProvider.UpdateConfigFromConfigMap(testCase.customPricingMap)
 

pkg/costmodel/clusters/clustermap.go

@@ -143,7 +143,7 @@ func NewClusterMap(client prometheus.Client, cip ClusterInfoProvider, refresh ti
 
 // clusterInfoQuery returns the query string to load cluster info
 func clusterInfoQuery(offset string) string {
-	return fmt.Sprintf("kubecost_cluster_info%s", offset)
+	return fmt.Sprintf("kubecost_cluster_info{%s}%s", env.GetPromClusterFilter(), offset)
 }
 
 // loadClusters loads all the cluster info to map

pkg/costmodel/costmodel.go

@@ -144,9 +144,9 @@ const (
 		label_replace(
 			label_replace(
 				avg(
-					count_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte", container!="",container!="POD", node!=""}[%s] %s)
+					count_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte", container!="",container!="POD", node!="", %s}[%s] %s)
 					*
-					avg_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte", container!="",container!="POD", node!=""}[%s] %s)
+					avg_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte", container!="",container!="POD", node!="", %s}[%s] %s)
 				) by (namespace,container,pod,node,%s) , "container_name","$1","container","(.+)"
 			), "pod_name","$1","pod","(.+)"
 		)
@@ -156,7 +156,7 @@ const (
 			label_replace(
 				label_replace(
 					label_replace(
-						count_over_time(container_memory_working_set_bytes{container!="", container!="POD", instance!=""}[%s] %s), "node", "$1", "instance", "(.+)"
+						count_over_time(container_memory_working_set_bytes{container!="", container!="POD", instance!="", %s}[%s] %s), "node", "$1", "instance", "(.+)"
 					), "container_name", "$1", "container", "(.+)"
 				), "pod_name", "$1", "pod", "(.+)"
 			)
@@ -164,7 +164,7 @@ const (
 			label_replace(
 				label_replace(
 					label_replace(
-						avg_over_time(container_memory_working_set_bytes{container!="", container!="POD", instance!=""}[%s] %s), "node", "$1", "instance", "(.+)"
+						avg_over_time(container_memory_working_set_bytes{container!="", container!="POD", instance!="", %s}[%s] %s), "node", "$1", "instance", "(.+)"
 					), "container_name", "$1", "container", "(.+)"
 				), "pod_name", "$1", "pod", "(.+)"
 			)
@@ -174,9 +174,9 @@ const (
 		label_replace(
 			label_replace(
 				avg(
-					count_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="",container!="POD", node!=""}[%s] %s)
+					count_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="",container!="POD", node!="", %s}[%s] %s)
 					*
-					avg_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="",container!="POD", node!=""}[%s] %s)
+					avg_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="",container!="POD", node!="", %s}[%s] %s)
 				) by (namespace,container,pod,node,%s) , "container_name","$1","container","(.+)"
 			), "pod_name","$1","pod","(.+)"
 		)
@@ -186,7 +186,7 @@ const (
 			label_replace(
 				label_replace(
 					rate(
-						container_cpu_usage_seconds_total{container!="", container!="POD", instance!=""}[%s] %s
+						container_cpu_usage_seconds_total{container!="", container!="POD", instance!="", %s}[%s] %s
 					), "node", "$1", "instance", "(.+)"
 				), "container_name", "$1", "container", "(.+)"
 			), "pod_name", "$1", "pod", "(.+)"
@@ -196,19 +196,19 @@ const (
 		label_replace(
 			label_replace(
 				avg(
-					count_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!=""}[%s] %s)
+					count_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!="", %s}[%s] %s)
 					*
-					avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!=""}[%s] %s)
+					avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!="", %s}[%s] %s)
 					* %f
 				) by (namespace,container,pod,node,%s) , "container_name","$1","container","(.+)"
 			), "pod_name","$1","pod","(.+)"
 		)
 	) by (namespace,container_name,pod_name,node,%s)
-	* on (pod_name, namespace, %s) group_left(container) label_replace(avg(avg_over_time(kube_pod_status_phase{phase="Running"}[%s] %s)) by (pod,namespace,%s), "pod_name","$1","pod","(.+)")`
-	queryPVRequestsStr = `avg(avg(kube_persistentvolumeclaim_info{volumename != ""}) by (persistentvolumeclaim, storageclass, namespace, volumename, %s, kubernetes_node)
+	* on (pod_name, namespace, %s) group_left(container) label_replace(avg(avg_over_time(kube_pod_status_phase{phase="Running", %s}[%s] %s)) by (pod,namespace,%s), "pod_name","$1","pod","(.+)")`
+	queryPVRequestsStr = `avg(avg(kube_persistentvolumeclaim_info{volumename != "", %s}) by (persistentvolumeclaim, storageclass, namespace, volumename, %s, kubernetes_node)
 	*
 	on (persistentvolumeclaim, namespace, %s, kubernetes_node) group_right(storageclass, volumename)
-	sum(kube_persistentvolumeclaim_resource_requests_storage_bytes{}) by (persistentvolumeclaim, namespace, %s, kubernetes_node, kubernetes_name)) by (persistentvolumeclaim, storageclass, namespace, %s, volumename, kubernetes_node)`
+	sum(kube_persistentvolumeclaim_resource_requests_storage_bytes{%s}) by (persistentvolumeclaim, namespace, %s, kubernetes_node, kubernetes_name)) by (persistentvolumeclaim, storageclass, namespace, %s, volumename, kubernetes_node)`
 	// queryRAMAllocationByteHours yields the total byte-hour RAM allocation over the given
 	// window, aggregated by container.
 	//  [line 3]  sum_over_time(each byte) = [byte*scrape] by metric
@@ -218,7 +218,7 @@ const (
 	queryRAMAllocationByteHours = `
 		label_replace(label_replace(
 			sum(
-				sum_over_time(container_memory_allocation_bytes{container!="",container!="POD", node!=""}[%s])
+				sum_over_time(container_memory_allocation_bytes{container!="",container!="POD", node!="", %s}[%s])
 			) by (namespace,container,pod,node,%s) * %f / 60 / 60
 		, "container_name","$1","container","(.+)"), "pod_name","$1","pod","(.+)")`
 	// queryCPUAllocationVCPUHours yields the total VCPU-hour CPU allocation over the given
@@ -230,35 +230,35 @@ const (
 	queryCPUAllocationVCPUHours = `
 		label_replace(label_replace(
 			sum(
-				sum_over_time(container_cpu_allocation{container!="",container!="POD", node!=""}[%s])
+				sum_over_time(container_cpu_allocation{container!="",container!="POD", node!="", %s}[%s])
 			) by (namespace,container,pod,node,%s) * %f / 60 / 60
 		, "container_name","$1","container","(.+)"), "pod_name","$1","pod","(.+)")`
 	// queryPVCAllocationFmt yields the total byte-hour PVC allocation over the given window.
 	// sum_over_time(each byte) = [byte*scrape] by metric *(scalar(avg(prometheus_target_interval_length_seconds)) = [seconds/scrape] / 60 / 60 =  [hours/scrape] by pod
-	queryPVCAllocationFmt     = `sum(sum_over_time(pod_pvc_allocation[%s])) by (%s, namespace, pod, persistentvolume, persistentvolumeclaim) * %f/60/60`
-	queryPVHourlyCostFmt      = `avg_over_time(pv_hourly_cost[%s])`
-	queryNSLabels             = `avg_over_time(kube_namespace_labels[%s])`
-	queryPodLabels            = `avg_over_time(kube_pod_labels[%s])`
-	queryNSAnnotations        = `avg_over_time(kube_namespace_annotations[%s])`
-	queryPodAnnotations       = `avg_over_time(kube_pod_annotations[%s])`
-	queryDeploymentLabels     = `avg_over_time(deployment_match_labels[%s])`
-	queryStatefulsetLabels    = `avg_over_time(statefulSet_match_labels[%s])`
-	queryPodDaemonsets        = `sum(kube_pod_owner{owner_kind="DaemonSet"}) by (namespace,pod,owner_name,%s)`
-	queryPodJobs              = `sum(kube_pod_owner{owner_kind="Job"}) by (namespace,pod,owner_name,%s)`
-	queryServiceLabels        = `avg_over_time(service_selector_labels[%s])`
-	queryZoneNetworkUsage     = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="true"}[%s] %s)) by (namespace,pod_name,%s) / 1024 / 1024 / 1024`
-	queryRegionNetworkUsage   = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="false"}[%s] %s)) by (namespace,pod_name,%s) / 1024 / 1024 / 1024`
-	queryInternetNetworkUsage = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="true"}[%s] %s)) by (namespace,pod_name,%s) / 1024 / 1024 / 1024`
-	normalizationStr          = `max(count_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte"}[%s] %s))`
+	queryPVCAllocationFmt     = `sum(sum_over_time(pod_pvc_allocation{%s}[%s])) by (%s, namespace, pod, persistentvolume, persistentvolumeclaim) * %f/60/60`
+	queryPVHourlyCostFmt      = `avg_over_time(pv_hourly_cost{%s}[%s])`
+	queryNSLabels             = `avg_over_time(kube_namespace_labels{%s}[%s])`
+	queryPodLabels            = `avg_over_time(kube_pod_labels{%s}[%s])`
+	queryNSAnnotations        = `avg_over_time(kube_namespace_annotations{%s}[%s])`
+	queryPodAnnotations       = `avg_over_time(kube_pod_annotations{%s}[%s])`
+	queryDeploymentLabels     = `avg_over_time(deployment_match_labels{%s}[%s])`
+	queryStatefulsetLabels    = `avg_over_time(statefulSet_match_labels{%s}[%s])`
+	queryPodDaemonsets        = `sum(kube_pod_owner{owner_kind="DaemonSet", %s}) by (namespace,pod,owner_name,%s)`
+	queryPodJobs              = `sum(kube_pod_owner{owner_kind="Job", %s}) by (namespace,pod,owner_name,%s)`
+	queryServiceLabels        = `avg_over_time(service_selector_labels{%s}[%s])`
+	queryZoneNetworkUsage     = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="true", %s}[%s] %s)) by (namespace,pod_name,%s) / 1024 / 1024 / 1024`
+	queryRegionNetworkUsage   = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="false", %s}[%s] %s)) by (namespace,pod_name,%s) / 1024 / 1024 / 1024`
+	queryInternetNetworkUsage = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="true", %s}[%s] %s)) by (namespace,pod_name,%s) / 1024 / 1024 / 1024`
+	normalizationStr          = `max(count_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte", %s}[%s] %s))`
 )
 
 func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, cp costAnalyzerCloud.Provider, window string, offset string, filterNamespace string) (map[string]*CostData, error) {
-	queryRAMUsage := fmt.Sprintf(queryRAMUsageStr, window, offset, window, offset, env.GetPromClusterLabel())
-	queryCPUUsage := fmt.Sprintf(queryCPUUsageStr, window, offset, env.GetPromClusterLabel())
-	queryNetZoneRequests := fmt.Sprintf(queryZoneNetworkUsage, window, "", env.GetPromClusterLabel())
-	queryNetRegionRequests := fmt.Sprintf(queryRegionNetworkUsage, window, "", env.GetPromClusterLabel())
-	queryNetInternetRequests := fmt.Sprintf(queryInternetNetworkUsage, window, "", env.GetPromClusterLabel())
-	queryNormalization := fmt.Sprintf(normalizationStr, window, offset)
+	queryRAMUsage := fmt.Sprintf(queryRAMUsageStr, env.GetPromClusterFilter(), window, offset, env.GetPromClusterFilter(), window, offset, env.GetPromClusterLabel())
+	queryCPUUsage := fmt.Sprintf(queryCPUUsageStr, env.GetPromClusterFilter(), window, offset, env.GetPromClusterLabel())
+	queryNetZoneRequests := fmt.Sprintf(queryZoneNetworkUsage, env.GetPromClusterFilter(), window, "", env.GetPromClusterLabel())
+	queryNetRegionRequests := fmt.Sprintf(queryRegionNetworkUsage, env.GetPromClusterFilter(), window, "", env.GetPromClusterLabel())
+	queryNetInternetRequests := fmt.Sprintf(queryInternetNetworkUsage, env.GetPromClusterFilter(), window, "", env.GetPromClusterLabel())
+	queryNormalization := fmt.Sprintf(normalizationStr, env.GetPromClusterFilter(), window, offset)
 
 	// Cluster ID is specific to the source cluster
 	clusterID := env.GetClusterID()
@@ -734,7 +734,7 @@ func findUnmountedPVCostData(clusterMap clusters.ClusterMap, unmountedPVs map[st
 
 func findDeletedPodInfo(cli prometheusClient.Client, missingContainers map[string]*CostData, window string) error {
 	if len(missingContainers) > 0 {
-		queryHistoricalPodLabels := fmt.Sprintf(`kube_pod_labels{}[%s]`, window)
+		queryHistoricalPodLabels := fmt.Sprintf(`kube_pod_labels{%s}[%s]`, env.GetPromClusterFilter(), window)
 
 		podLabelsResult, _, err := prom.NewNamedContext(cli, prom.ComputeCostDataContextName).QuerySync(queryHistoricalPodLabels)
 		if err != nil {
@@ -773,9 +773,9 @@ func findDeletedNodeInfo(cli prometheusClient.Client, missingNodes map[string]*c
 			offsetStr = fmt.Sprintf("offset %s", offset)
 		}
 
-		queryHistoricalCPUCost := fmt.Sprintf(`avg(avg_over_time(node_cpu_hourly_cost[%s] %s)) by (node, instance, %s)`, window, offsetStr, env.GetPromClusterLabel())
-		queryHistoricalRAMCost := fmt.Sprintf(`avg(avg_over_time(node_ram_hourly_cost[%s] %s)) by (node, instance, %s)`, window, offsetStr, env.GetPromClusterLabel())
-		queryHistoricalGPUCost := fmt.Sprintf(`avg(avg_over_time(node_gpu_hourly_cost[%s] %s)) by (node, instance, %s)`, window, offsetStr, env.GetPromClusterLabel())
+		queryHistoricalCPUCost := fmt.Sprintf(`avg(avg_over_time(node_cpu_hourly_cost{%s}[%s] %s)) by (node, instance, %s)`, env.GetPromClusterFilter(), window, offsetStr, env.GetPromClusterLabel())
+		queryHistoricalRAMCost := fmt.Sprintf(`avg(avg_over_time(node_ram_hourly_cost{%s}[%s] %s)) by (node, instance, %s)`, env.GetPromClusterFilter(), window, offsetStr, env.GetPromClusterLabel())
+		queryHistoricalGPUCost := fmt.Sprintf(`avg(avg_over_time(node_gpu_hourly_cost{%s}[%s] %s)) by (node, instance, %s)`, env.GetPromClusterFilter(), window, offsetStr, env.GetPromClusterLabel())
 
 		ctx := prom.NewNamedContext(cli, prom.ComputeCostDataContextName)
 		cpuCostResCh := ctx.Query(queryHistoricalCPUCost)
@@ -1002,6 +1002,10 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 			region, _ := util.GetRegion(n.Labels)
 			newCnode.Region = region
 		}
+		if newCnode.ArchType == "" {
+			arch, _ := util.GetArchType(n.Labels)
+			newCnode.ArchType = arch
+		}
 		newCnode.ProviderID = n.Spec.ProviderID
 
 		var cpu float64
@@ -1625,20 +1629,20 @@ func (cm *CostModel) costDataRange(cli prometheusClient.Client, cp costAnalyzerC
 
 	ctx := prom.NewNamedContext(cli, prom.ComputeCostDataRangeContextName)
 
-	queryRAMAlloc := fmt.Sprintf(queryRAMAllocationByteHours, resStr, env.GetPromClusterLabel(), scrapeIntervalSeconds)
-	queryCPUAlloc := fmt.Sprintf(queryCPUAllocationVCPUHours, resStr, env.GetPromClusterLabel(), scrapeIntervalSeconds)
-	queryRAMRequests := fmt.Sprintf(queryRAMRequestsStr, resStr, "", resStr, "", env.GetPromClusterLabel(), env.GetPromClusterLabel())
-	queryRAMUsage := fmt.Sprintf(queryRAMUsageStr, resStr, "", resStr, "", env.GetPromClusterLabel())
-	queryCPURequests := fmt.Sprintf(queryCPURequestsStr, resStr, "", resStr, "", env.GetPromClusterLabel(), env.GetPromClusterLabel())
-	queryCPUUsage := fmt.Sprintf(queryCPUUsageStr, resStr, "", env.GetPromClusterLabel())
-	queryGPURequests := fmt.Sprintf(queryGPURequestsStr, resStr, "", resStr, "", resolution.Hours(), env.GetPromClusterLabel(), env.GetPromClusterLabel(), env.GetPromClusterLabel(), resStr, "", env.GetPromClusterLabel())
-	queryPVRequests := fmt.Sprintf(queryPVRequestsStr, env.GetPromClusterLabel(), env.GetPromClusterLabel(), env.GetPromClusterLabel(), env.GetPromClusterLabel())
-	queryPVCAllocation := fmt.Sprintf(queryPVCAllocationFmt, resStr, env.GetPromClusterLabel(), scrapeIntervalSeconds)
-	queryPVHourlyCost := fmt.Sprintf(queryPVHourlyCostFmt, resStr)
-	queryNetZoneRequests := fmt.Sprintf(queryZoneNetworkUsage, resStr, "", env.GetPromClusterLabel())
-	queryNetRegionRequests := fmt.Sprintf(queryRegionNetworkUsage, resStr, "", env.GetPromClusterLabel())
-	queryNetInternetRequests := fmt.Sprintf(queryInternetNetworkUsage, resStr, "", env.GetPromClusterLabel())
-	queryNormalization := fmt.Sprintf(normalizationStr, resStr, "")
+	queryRAMAlloc := fmt.Sprintf(queryRAMAllocationByteHours, env.GetPromClusterFilter(), resStr, env.GetPromClusterLabel(), scrapeIntervalSeconds)
+	queryCPUAlloc := fmt.Sprintf(queryCPUAllocationVCPUHours, env.GetPromClusterFilter(), resStr, env.GetPromClusterLabel(), scrapeIntervalSeconds)
+	queryRAMRequests := fmt.Sprintf(queryRAMRequestsStr, env.GetPromClusterFilter(), resStr, "", env.GetPromClusterFilter(), resStr, "", env.GetPromClusterLabel(), env.GetPromClusterLabel())
+	queryRAMUsage := fmt.Sprintf(queryRAMUsageStr, env.GetPromClusterFilter(), resStr, "", env.GetPromClusterFilter(), resStr, "", env.GetPromClusterLabel())
+	queryCPURequests := fmt.Sprintf(queryCPURequestsStr, env.GetPromClusterFilter(), resStr, "", env.GetPromClusterFilter(), resStr, "", env.GetPromClusterLabel(), env.GetPromClusterLabel())
+	queryCPUUsage := fmt.Sprintf(queryCPUUsageStr, env.GetPromClusterFilter(), resStr, "", env.GetPromClusterLabel())
+	queryGPURequests := fmt.Sprintf(queryGPURequestsStr, env.GetPromClusterFilter(), resStr, "", env.GetPromClusterFilter(), resStr, "", resolution.Hours(), env.GetPromClusterLabel(), env.GetPromClusterLabel(), env.GetPromClusterLabel(), env.GetPromClusterFilter(), resStr, "", env.GetPromClusterLabel())
+	queryPVRequests := fmt.Sprintf(queryPVRequestsStr, env.GetPromClusterFilter(), env.GetPromClusterLabel(), env.GetPromClusterLabel(), env.GetPromClusterFilter(), env.GetPromClusterLabel(), env.GetPromClusterLabel())
+	queryPVCAllocation := fmt.Sprintf(queryPVCAllocationFmt, env.GetPromClusterFilter(), resStr, env.GetPromClusterLabel(), scrapeIntervalSeconds)
+	queryPVHourlyCost := fmt.Sprintf(queryPVHourlyCostFmt, env.GetPromClusterFilter(), resStr)
+	queryNetZoneRequests := fmt.Sprintf(queryZoneNetworkUsage, env.GetPromClusterFilter(), resStr, "", env.GetPromClusterLabel())
+	queryNetRegionRequests := fmt.Sprintf(queryRegionNetworkUsage, env.GetPromClusterFilter(), resStr, "", env.GetPromClusterLabel())
+	queryNetInternetRequests := fmt.Sprintf(queryInternetNetworkUsage, env.GetPromClusterFilter(), resStr, "", env.GetPromClusterLabel())
+	queryNormalization := fmt.Sprintf(normalizationStr, env.GetPromClusterFilter(), resStr, "")
 
 	// Submit all queries for concurrent evaluation
 	resChRAMRequests := ctx.QueryRange(queryRAMRequests, start, end, resolution)
@@ -1654,15 +1658,15 @@ func (cm *CostModel) costDataRange(cli prometheusClient.Client, cp costAnalyzerC
 	resChNetZoneRequests := ctx.QueryRange(queryNetZoneRequests, start, end, resolution)
 	resChNetRegionRequests := ctx.QueryRange(queryNetRegionRequests, start, end, resolution)
 	resChNetInternetRequests := ctx.QueryRange(queryNetInternetRequests, start, end, resolution)
-	resChNSLabels := ctx.QueryRange(fmt.Sprintf(queryNSLabels, resStr), start, end, resolution)
-	resChPodLabels := ctx.QueryRange(fmt.Sprintf(queryPodLabels, resStr), start, end, resolution)
-	resChNSAnnotations := ctx.QueryRange(fmt.Sprintf(queryNSAnnotations, resStr), start, end, resolution)
-	resChPodAnnotations := ctx.QueryRange(fmt.Sprintf(queryPodAnnotations, resStr), start, end, resolution)
-	resChServiceLabels := ctx.QueryRange(fmt.Sprintf(queryServiceLabels, resStr), start, end, resolution)
-	resChDeploymentLabels := ctx.QueryRange(fmt.Sprintf(queryDeploymentLabels, resStr), start, end, resolution)
-	resChStatefulsetLabels := ctx.QueryRange(fmt.Sprintf(queryStatefulsetLabels, resStr), start, end, resolution)
-	resChJobs := ctx.QueryRange(fmt.Sprintf(queryPodJobs, env.GetPromClusterLabel()), start, end, resolution)
-	resChDaemonsets := ctx.QueryRange(fmt.Sprintf(queryPodDaemonsets, env.GetPromClusterLabel()), start, end, resolution)
+	resChNSLabels := ctx.QueryRange(fmt.Sprintf(queryNSLabels, env.GetPromClusterFilter(), resStr), start, end, resolution)
+	resChPodLabels := ctx.QueryRange(fmt.Sprintf(queryPodLabels, env.GetPromClusterFilter(), resStr), start, end, resolution)
+	resChNSAnnotations := ctx.QueryRange(fmt.Sprintf(queryNSAnnotations, env.GetPromClusterFilter(), resStr), start, end, resolution)
+	resChPodAnnotations := ctx.QueryRange(fmt.Sprintf(queryPodAnnotations, env.GetPromClusterFilter(), resStr), start, end, resolution)
+	resChServiceLabels := ctx.QueryRange(fmt.Sprintf(queryServiceLabels, env.GetPromClusterFilter(), resStr), start, end, resolution)
+	resChDeploymentLabels := ctx.QueryRange(fmt.Sprintf(queryDeploymentLabels, env.GetPromClusterFilter(), resStr), start, end, resolution)
+	resChStatefulsetLabels := ctx.QueryRange(fmt.Sprintf(queryStatefulsetLabels, env.GetPromClusterFilter(), resStr), start, end, resolution)
+	resChJobs := ctx.QueryRange(fmt.Sprintf(queryPodJobs, env.GetPromClusterFilter(), env.GetPromClusterLabel()), start, end, resolution)
+	resChDaemonsets := ctx.QueryRange(fmt.Sprintf(queryPodDaemonsets, env.GetPromClusterFilter(), env.GetPromClusterLabel()), start, end, resolution)
 	resChNormalization := ctx.QueryRange(queryNormalization, start, end, resolution)
 
 	// Pull k8s pod, controller, service, and namespace details

pkg/costmodel/metrics.go

@@ -147,7 +147,7 @@ func initCostModelMetrics(clusterCache clustercache.ClusterCache, provider model
 		cpuGv = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Name: "node_cpu_hourly_cost",
 			Help: "node_cpu_hourly_cost hourly cost for each cpu on this node",
-		}, []string{"instance", "node", "instance_type", "region", "provider_id"})
+		}, []string{"instance", "node", "instance_type", "region", "provider_id", "arch"})
 		if _, disabled := disabledMetrics["node_cpu_hourly_cost"]; !disabled {
 			toRegisterGV = append(toRegisterGV, cpuGv)
 		}
@@ -155,7 +155,7 @@ func initCostModelMetrics(clusterCache clustercache.ClusterCache, provider model
 		ramGv = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Name: "node_ram_hourly_cost",
 			Help: "node_ram_hourly_cost hourly cost for each gb of ram on this node",
-		}, []string{"instance", "node", "instance_type", "region", "provider_id"})
+		}, []string{"instance", "node", "instance_type", "region", "provider_id", "arch"})
 		if _, disabled := disabledMetrics["node_ram_hourly_cost"]; !disabled {
 			toRegisterGV = append(toRegisterGV, ramGv)
 		}
@@ -163,7 +163,7 @@ func initCostModelMetrics(clusterCache clustercache.ClusterCache, provider model
 		gpuGv = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Name: "node_gpu_hourly_cost",
 			Help: "node_gpu_hourly_cost hourly cost for each gpu on this node",
-		}, []string{"instance", "node", "instance_type", "region", "provider_id"})
+		}, []string{"instance", "node", "instance_type", "region", "provider_id", "arch"})
 		if _, disabled := disabledMetrics["node_gpu_hourly_cost"]; !disabled {
 			toRegisterGV = append(toRegisterGV, gpuGv)
 		}
@@ -171,7 +171,7 @@ func initCostModelMetrics(clusterCache clustercache.ClusterCache, provider model
 		gpuCountGv = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Name: "node_gpu_count",
 			Help: "node_gpu_count count of gpu on this node",
-		}, []string{"instance", "node", "instance_type", "region", "provider_id"})
+		}, []string{"instance", "node", "instance_type", "region", "provider_id", "arch"})
 		if _, disabled := disabledMetrics["node_gpu_count"]; !disabled {
 			toRegisterGV = append(toRegisterGV, gpuCountGv)
 		}
@@ -195,7 +195,7 @@ func initCostModelMetrics(clusterCache clustercache.ClusterCache, provider model
 		totalGv = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Name: "node_total_hourly_cost",
 			Help: "node_total_hourly_cost Total node cost per hour",
-		}, []string{"instance", "node", "instance_type", "region", "provider_id"})
+		}, []string{"instance", "node", "instance_type", "region", "provider_id", "arch"})
 		if _, disabled := disabledMetrics["node_total_hourly_cost"]; !disabled {
 			toRegisterGV = append(toRegisterGV, totalGv)
 		}
@@ -528,22 +528,22 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 					nodeCostAverages[labelKey] = avgCosts
 				}
 
-				cmme.GPUCountRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(gpu)
-				cmme.GPUPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(gpuCost)
+				cmme.GPUCountRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID, node.ArchType).Set(gpu)
+				cmme.GPUPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID, node.ArchType).Set(gpuCost)
 
 				const outlierFactor float64 = 30
 				// don't record cpuCost, ramCost, or gpuCost in the case of wild outliers
 				// k8s api sometimes causes cost spikes as described here:
 				// https://github.com/opencost/opencost/issues/927
 				if cpuCost < outlierFactor*avgCosts.CpuCostAverage {
-					cmme.CPUPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(cpuCost)
+					cmme.CPUPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID, node.ArchType).Set(cpuCost)
 					avgCosts.CpuCostAverage = (avgCosts.CpuCostAverage*avgCosts.NumCpuDataPoints + cpuCost) / (avgCosts.NumCpuDataPoints + 1)
 					avgCosts.NumCpuDataPoints += 1
 				} else {
 					log.Warnf("CPU cost outlier detected; skipping data point.")
 				}
 				if ramCost < outlierFactor*avgCosts.RamCostAverage {
-					cmme.RAMPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(ramCost)
+					cmme.RAMPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID, node.ArchType).Set(ramCost)
 					avgCosts.RamCostAverage = (avgCosts.RamCostAverage*avgCosts.NumRamDataPoints + ramCost) / (avgCosts.NumRamDataPoints + 1)
 					avgCosts.NumRamDataPoints += 1
 				} else {
@@ -552,7 +552,7 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 				// skip redording totalCost if any constituent costs were outliers
 				if cpuCost < outlierFactor*avgCosts.CpuCostAverage &&
 					ramCost < outlierFactor*avgCosts.RamCostAverage {
-					cmme.NodeTotalPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID).Set(totalCost)
+					cmme.NodeTotalPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, node.ProviderID, node.ArchType).Set(totalCost)
 				}
 
 				nodeCostAverages[labelKey] = avgCosts

pkg/costmodel/router.go

@@ -18,6 +18,7 @@ import (
 	"github.com/microcosm-cc/bluemonday"
 	"github.com/opencost/opencost/pkg/cloud/aws"
 	"github.com/opencost/opencost/pkg/cloud/gcp"
+	"github.com/opencost/opencost/pkg/cloud/provider"
 	"github.com/opencost/opencost/pkg/config"
 	"github.com/opencost/opencost/pkg/kubeconfig"
 	"github.com/opencost/opencost/pkg/metrics"
@@ -34,7 +35,6 @@ import (
 
 	"github.com/getsentry/sentry-go"
 
-	"github.com/opencost/opencost/pkg/cloud"
 	"github.com/opencost/opencost/pkg/cloud/azure"
 	"github.com/opencost/opencost/pkg/cloud/models"
 	"github.com/opencost/opencost/pkg/cloud/utils"
@@ -1592,13 +1592,13 @@ func Initialize(additionalConfigWatchers ...*watcher.ConfigMapWatcher) *Accesses
 	k8sCache.Run()
 
 	cloudProviderKey := env.GetCloudProviderAPIKey()
-	cloudProvider, err := cloud.NewProvider(k8sCache, cloudProviderKey, confManager)
+	cloudProvider, err := provider.NewProvider(k8sCache, cloudProviderKey, confManager)
 	if err != nil {
 		panic(err.Error())
 	}
 
 	// Append the pricing config watcher
-	configWatchers.AddWatcher(cloud.ConfigWatcherFor(cloudProvider))
+	configWatchers.AddWatcher(provider.ConfigWatcherFor(cloudProvider))
 	configWatchers.AddWatcher(metrics.GetMetricsConfigWatcher())
 
 	watchConfigFunc := configWatchers.ToWatchFunc()

pkg/env/costmodelenv.go

@@ -1,6 +1,7 @@
 package env
 
 import (
+	"fmt"
 	"regexp"
 	"strconv"
 	"time"
@@ -67,7 +68,8 @@ const (
 
 	KubeConfigPathEnvVar = "KUBECONFIG_PATH"
 
-	UTCOffsetEnvVar = "UTC_OFFSET"
+	UTCOffsetEnvVar                  = "UTC_OFFSET"
+	CurrentClusterIdFilterEnabledVar = "CURRENT_CLUSTER_ID_FILTER_ENABLED"
 
 	CacheWarmingEnabledEnvVar            = "CACHE_WARMING_ENABLED"
 	ETLEnabledEnvVar                     = "ETL_ENABLED"
@@ -213,7 +215,13 @@ func IsEmitKsmV1MetricsOnly() bool {
 // GetAWSAccessKeyID returns the environment variable value for AWSAccessKeyIDEnvVar which represents
 // the AWS access key for authentication
 func GetAWSAccessKeyID() string {
-	return Get(AWSAccessKeyIDEnvVar, "")
+	awsAccessKeyID := Get(AWSAccessKeyIDEnvVar, "")
+	// If the sample nil service key name is set, zero it out so that it is not
+	// misinterpreted as a real service key.
+	if awsAccessKeyID == "AKIXXX" {
+		awsAccessKeyID = ""
+	}
+	return awsAccessKeyID
 }
 
 // GetAWSAccessKeySecret returns the environment variable value for AWSAccessKeySecretEnvVar which represents
@@ -284,6 +292,15 @@ func GetClusterID() string {
 	return Get(ClusterIDEnvVar, "")
 }
 
+// GetPromClusterFilter returns environment variable value CurrentClusterIdFilterEnabledVar which
+// represents additional prometheus filter for all metrics for current cluster id
+func GetPromClusterFilter() string {
+	if GetBool(CurrentClusterIdFilterEnabledVar, false) {
+		return fmt.Sprintf("%s=\"%s\"", GetPromClusterLabel(), GetClusterID())
+	}
+	return ""
+}
+
 // GetPrometheusServerEndpoint returns the environment variable value for PrometheusServerEndpointEnvVar which
 // represents the prometheus server endpoint used to execute prometheus queries.
 func GetPrometheusServerEndpoint() string {

pkg/filemanager/filemanager_test.go

@@ -14,25 +14,33 @@ import (
 
 func Test_NewFileManager(t *testing.T) {
 	t.Run("file system", func(t *testing.T) {
+		// Create File Manager
 		tmpPath := filepath.Join(os.TempDir(), fmt.Sprintf("opencost-test-file-manager-%d", rand.Int31()))
+		defer os.Remove(tmpPath) // remove managed file on completion to ensure it does not exist for next run
 		fm, err := NewFileManager(tmpPath)
 		require.NoError(t, err)
 
+		// Attempt to download Managed file into a new file, this should fail because managed file has not been initalized
 		downloadFile, err := os.CreateTemp("", "opencost-test-file-manager-*")
 		require.NoError(t, err)
 		err = fm.Download(context.TODO(), downloadFile)
 		require.ErrorIs(t, err, ErrNotFound)
 
+		// Create a file and add content to it
 		uploadFile, err := os.CreateTemp("", "opencost-test-file-manager-*")
 		require.NoError(t, err)
 		_, err = uploadFile.WriteString("test-content")
 		require.NoError(t, err)
-		require.NoError(t, err)
+
+		// Upload file to managed file
 		err = fm.Upload(context.TODO(), uploadFile)
 		require.NoError(t, err)
 
+		// Download managed file to original file
 		err = fm.Download(context.TODO(), downloadFile)
 		require.NoError(t, err)
+
+		// Check that content matches
 		_, err = downloadFile.Seek(0, io.SeekStart)
 		require.NoError(t, err)
 		data, err := io.ReadAll(downloadFile)

pkg/filter/cloudcost/cloudcost.go

@@ -0,0 +1,155 @@
+package cloudcost
+
+import (
+	"reflect"
+	"strings"
+
+	"github.com/opencost/opencost/pkg/filter"
+	"github.com/opencost/opencost/pkg/kubecost"
+	"github.com/opencost/opencost/pkg/log"
+	"github.com/opencost/opencost/pkg/util/mapper"
+)
+
+type CloudCostFilter struct {
+	AccountIDs       []string `json:"accountIDs,omitempty"`
+	Categories       []string `json:"categories,omitempty"`
+	InvoiceEntityIDs []string `json:"invoiceEntityIDs,omitempty"`
+	Labels           []string `json:"labels,omitempty"`
+	Providers        []string `json:"providers,omitempty"`
+	ProviderIDs      []string `json:"providerIDs,omitempty"`
+	Services         []string `json:"services,omitempty"`
+}
+
+func (g *CloudCostFilter) Equals(that CloudCostFilter) bool {
+	return reflect.DeepEqual(g.AccountIDs, that.AccountIDs) &&
+		reflect.DeepEqual(g.Categories, that.Categories) &&
+		reflect.DeepEqual(g.InvoiceEntityIDs, that.InvoiceEntityIDs) &&
+		reflect.DeepEqual(g.Labels, that.Labels) &&
+		reflect.DeepEqual(g.Providers, that.Providers) &&
+		reflect.DeepEqual(g.ProviderIDs, that.ProviderIDs) &&
+		reflect.DeepEqual(g.Services, that.Services)
+}
+func parseWildcardEnd(rawFilterValue string) (string, bool) {
+	return strings.TrimSuffix(rawFilterValue, "*"), strings.HasSuffix(rawFilterValue, "*")
+}
+
+func CloudCostFilterFromParams(pmr mapper.PrimitiveMapReader) filter.Filter[*kubecost.CloudCost] {
+	ccFilter := convertFilterQueryParams(pmr)
+	return ParseCloudCostFilter(ccFilter)
+}
+
+func convertFilterQueryParams(pmr mapper.PrimitiveMapReader) CloudCostFilter {
+	return CloudCostFilter{
+		AccountIDs:       pmr.GetList("filterAccountIDs", ","),
+		Categories:       pmr.GetList("filterCategories", ","),
+		InvoiceEntityIDs: pmr.GetList("filterInvoiceEntityIDs", ","),
+		Labels:           pmr.GetList("filterLabels", ","),
+		Providers:        pmr.GetList("filterProviders", ","),
+		ProviderIDs:      pmr.GetList("filterProviderIDs", ","),
+		Services:         pmr.GetList("filterServices", ","),
+	}
+}
+func ParseCloudCostFilter(filters CloudCostFilter) filter.Filter[*kubecost.CloudCost] {
+	result := filter.And[*kubecost.CloudCost]{
+		Filters: []filter.Filter[*kubecost.CloudCost]{},
+	}
+
+	if len(filters.InvoiceEntityIDs) > 0 {
+		result.Filters = append(result.Filters, filterV1SingleValueFromList(filters.InvoiceEntityIDs, kubecost.CloudCostInvoiceEntityIDProp))
+	}
+
+	if len(filters.AccountIDs) > 0 {
+		result.Filters = append(result.Filters, filterV1SingleValueFromList(filters.AccountIDs, kubecost.CloudCostAccountIDProp))
+	}
+
+	if len(filters.Providers) > 0 {
+		result.Filters = append(result.Filters, filterV1SingleValueFromList(filters.Providers, kubecost.CloudCostProviderProp))
+	}
+
+	if len(filters.ProviderIDs) > 0 {
+		result.Filters = append(result.Filters, filterV1SingleValueFromList(filters.ProviderIDs, kubecost.CloudCostProviderIDProp))
+	}
+
+	if len(filters.Services) > 0 {
+		result.Filters = append(result.Filters, filterV1SingleValueFromList(filters.Services, kubecost.CloudCostServiceProp))
+	}
+
+	if len(filters.Categories) > 0 {
+		result.Filters = append(result.Filters, filterV1SingleValueFromList(filters.Categories, kubecost.CloudCostCategoryProp))
+	}
+
+	if len(filters.Labels) > 0 {
+		result.Filters = append(result.Filters, filterV1DoubleValueFromList(filters.Labels, kubecost.CloudCostLabelProp))
+	}
+
+	if len(result.Filters) == 0 {
+		return nil
+	}
+
+	return result
+}
+
+func filterV1SingleValueFromList(rawFilterValues []string, field string) filter.Filter[*kubecost.CloudCost] {
+	result := filter.Or[*kubecost.CloudCost]{
+		Filters: []filter.Filter[*kubecost.CloudCost]{},
+	}
+
+	for _, filterValue := range rawFilterValues {
+		filterValue = strings.TrimSpace(filterValue)
+		filterValue, wildcard := parseWildcardEnd(filterValue)
+
+		subFilter := filter.StringProperty[*kubecost.CloudCost]{
+			Field: field,
+			Op:    filter.StringEquals,
+			Value: filterValue,
+		}
+
+		if wildcard {
+			subFilter.Op = filter.StringStartsWith
+		}
+
+		result.Filters = append(result.Filters, subFilter)
+	}
+
+	return result
+}
+
+// filterV1DoubleValueFromList creates an OR of key:value equality filters for
+// colon-split filter values.
+//
+// The v1 query language (e.g. "filterLabels=app:foo,l2:bar") uses OR within
+// a field (e.g. label[app] = foo OR label[l2] = bar)
+func filterV1DoubleValueFromList(rawFilterValuesUnsplit []string, filterField string) filter.Filter[*kubecost.CloudCost] {
+	result := filter.Or[*kubecost.CloudCost]{
+		Filters: []filter.Filter[*kubecost.CloudCost]{},
+	}
+
+	for _, unsplit := range rawFilterValuesUnsplit {
+		if unsplit != "" {
+			split := strings.Split(unsplit, ":")
+			if len(split) != 2 {
+				log.Warnf("illegal key/value filter (ignoring): %s", unsplit)
+				continue
+			}
+			labelName := strings.TrimSpace(split[0])
+			val := strings.TrimSpace(split[1])
+			val, wildcard := parseWildcardEnd(val)
+
+			subFilter := filter.StringMapProperty[*kubecost.CloudCost]{
+				Field: filterField,
+				// All v1 filters are equality comparisons
+				Op:    filter.StringMapEquals,
+				Key:   labelName,
+				Value: val,
+			}
+
+			if wildcard {
+				subFilter.Op = filter.StringMapStartsWith
+			}
+
+			result.Filters = append(result.Filters, subFilter)
+		}
+	}
+
+	return result
+}

pkg/filter/cloudcost/cloudcost_test.go

@@ -0,0 +1,134 @@
+package cloudcost
+
+import (
+	"testing"
+)
+
+type CloudCostFilterEqualsTestcase struct {
+	name     string
+	this     CloudCostFilter
+	that     CloudCostFilter
+	expected bool
+}
+
+func TestCloudCostFilter_Equals(t *testing.T) {
+	testCases := []CloudCostFilterEqualsTestcase{
+		{
+			name: "both filters nil",
+			this: CloudCostFilter{
+				AccountIDs:       nil,
+				Categories:       nil,
+				InvoiceEntityIDs: nil,
+				Labels:           nil,
+				Providers:        nil,
+				ProviderIDs:      nil,
+				Services:         nil,
+			},
+			that: CloudCostFilter{
+				AccountIDs:       nil,
+				Categories:       nil,
+				InvoiceEntityIDs: nil,
+				Labels:           nil,
+				Providers:        nil,
+				ProviderIDs:      nil,
+				Services:         nil,
+			},
+			expected: true,
+		},
+		{
+			name: "both filters not nil and matching",
+			this: CloudCostFilter{
+				AccountIDs:       []string{"account1", "account2", "account3"},
+				Categories:       []string{"category1", "category2"},
+				InvoiceEntityIDs: []string{"invoice1", "invoice2"},
+				Labels:           []string{"label1", "label2"},
+				Providers:        []string{"provider1", "provider2"},
+				ProviderIDs:      []string{"provider1", "provider2"},
+				Services:         []string{"s1"},
+			},
+			that: CloudCostFilter{
+				AccountIDs:       []string{"account1", "account2", "account3"},
+				Categories:       []string{"category1", "category2"},
+				InvoiceEntityIDs: []string{"invoice1", "invoice2"},
+				Labels:           []string{"label1", "label2"},
+				Providers:        []string{"provider1", "provider2"},
+				ProviderIDs:      []string{"provider1", "provider2"},
+				Services:         []string{"s1"},
+			},
+			expected: true,
+		},
+		{
+			name: "both filters diff count",
+			this: CloudCostFilter{
+				AccountIDs:       []string{"account1", "account2", "account3"},
+				Categories:       []string{"category1", "category2"},
+				InvoiceEntityIDs: []string{"invoice1", "invoice2"},
+				Labels:           []string{"label1", "label2"},
+				Providers:        []string{"provider1", "provider2"},
+				ProviderIDs:      []string{"provider1", "provider2"},
+				Services:         []string{"s1"},
+			},
+			that: CloudCostFilter{
+				AccountIDs:       []string{"account1", "account2"},
+				Categories:       []string{"category1", "category2"},
+				InvoiceEntityIDs: []string{"invoice1", "invoice2"},
+				Labels:           []string{"label1", "label2"},
+				Providers:        []string{"provider1", "provider2"},
+				ProviderIDs:      []string{"provider1", "provider2"},
+				Services:         []string{"s1"},
+			},
+			expected: false,
+		},
+		{
+			name: "slight mismatch",
+			this: CloudCostFilter{
+				AccountIDs:       []string{"account1", "account2", "account3"},
+				Categories:       []string{"category1", "category2"},
+				InvoiceEntityIDs: []string{"invoice1", "invoice2"},
+				Labels:           []string{"label1", "label2"},
+				Providers:        []string{"provider1", "provider2"},
+				ProviderIDs:      []string{"provider1", "provider2"},
+				Services:         []string{"s1"},
+			},
+			that: CloudCostFilter{
+				AccountIDs:       []string{"account10", "account2", "account3"},
+				Categories:       []string{"category1", "category2"},
+				InvoiceEntityIDs: []string{"invoice1", "invoice2"},
+				Labels:           []string{"label1", "label2"},
+				Providers:        []string{"provider1", "provider2"},
+				ProviderIDs:      []string{"provider1", "provider2"},
+				Services:         []string{"s1"},
+			},
+			expected: false,
+		},
+		{
+			name: "one nil, one not",
+			this: CloudCostFilter{
+				AccountIDs:       []string{"account1", "account2", "account3"},
+				Categories:       []string{"category1", "category2"},
+				InvoiceEntityIDs: []string{"invoice1", "invoice2"},
+				Labels:           []string{"label1", "label2"},
+				Providers:        []string{"provider1", "provider2"},
+				ProviderIDs:      []string{"provider1", "provider2"},
+				Services:         []string{"s1"},
+			},
+			that: CloudCostFilter{
+				AccountIDs:       nil,
+				Categories:       nil,
+				InvoiceEntityIDs: nil,
+				Labels:           nil,
+				Providers:        nil,
+				ProviderIDs:      nil,
+				Services:         nil,
+			},
+			expected: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		got := tc.this.Equals(tc.that)
+		if got != tc.expected {
+			t.Fatalf("expected %t, got: %t for test case: %s", tc.expected, got, tc.name)
+		}
+	}
+}

pkg/filter/util/cloudcost.go

@@ -1,119 +0,0 @@
-package util
-
-import (
-	"strings"
-
-	"github.com/opencost/opencost/pkg/filter"
-	"github.com/opencost/opencost/pkg/kubecost"
-	"github.com/opencost/opencost/pkg/log"
-	"github.com/opencost/opencost/pkg/util/mapper"
-)
-
-func parseWildcardEnd(rawFilterValue string) (string, bool) {
-	return strings.TrimSuffix(rawFilterValue, "*"), strings.HasSuffix(rawFilterValue, "*")
-}
-
-func CloudCostFilterFromParams(pmr mapper.PrimitiveMapReader) filter.Filter[*kubecost.CloudCost] {
-	filter := filter.And[*kubecost.CloudCost]{
-		Filters: []filter.Filter[*kubecost.CloudCost]{},
-	}
-
-	if raw := pmr.GetList("filterInvoiceEntityIDs", ","); len(raw) > 0 {
-		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostInvoiceEntityIDProp))
-	}
-
-	if raw := pmr.GetList("filterAccountIDs", ","); len(raw) > 0 {
-		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostAccountIDProp))
-	}
-
-	if raw := pmr.GetList("filterProviders", ","); len(raw) > 0 {
-		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostProviderProp))
-	}
-
-	if raw := pmr.GetList("filterProviderIDs", ","); len(raw) > 0 {
-		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostProviderIDProp))
-	}
-
-	if raw := pmr.GetList("filterServices", ","); len(raw) > 0 {
-		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostServiceProp))
-	}
-
-	if raw := pmr.GetList("filterCategories", ","); len(raw) > 0 {
-		filter.Filters = append(filter.Filters, filterV1SingleValueFromList(raw, kubecost.CloudCostCategoryProp))
-	}
-
-	if raw := pmr.GetList("filterLabels", ","); len(raw) > 0 {
-		filter.Filters = append(filter.Filters, filterV1DoubleValueFromList(raw, kubecost.CloudCostLabelProp))
-	}
-
-	if len(filter.Filters) == 0 {
-		return nil
-	}
-
-	return filter
-}
-
-func filterV1SingleValueFromList(rawFilterValues []string, field string) filter.Filter[*kubecost.CloudCost] {
-	result := filter.Or[*kubecost.CloudCost]{
-		Filters: []filter.Filter[*kubecost.CloudCost]{},
-	}
-
-	for _, filterValue := range rawFilterValues {
-		filterValue = strings.TrimSpace(filterValue)
-		filterValue, wildcard := parseWildcardEnd(filterValue)
-
-		subFilter := filter.StringProperty[*kubecost.CloudCost]{
-			Field: field,
-			Op:    filter.StringEquals,
-			Value: filterValue,
-		}
-
-		if wildcard {
-			subFilter.Op = kubecost.FilterStartsWith
-		}
-
-		result.Filters = append(result.Filters, subFilter)
-	}
-
-	return result
-}
-
-// filterV1DoubleValueFromList creates an OR of key:value equality filters for
-// colon-split filter values.
-//
-// The v1 query language (e.g. "filterLabels=app:foo,l2:bar") uses OR within
-// a field (e.g. label[app] = foo OR label[l2] = bar)
-func filterV1DoubleValueFromList(rawFilterValuesUnsplit []string, filterField string) filter.Filter[*kubecost.CloudCost] {
-	result := filter.Or[*kubecost.CloudCost]{
-		Filters: []filter.Filter[*kubecost.CloudCost]{},
-	}
-
-	for _, unsplit := range rawFilterValuesUnsplit {
-		if unsplit != "" {
-			split := strings.Split(unsplit, ":")
-			if len(split) != 2 {
-				log.Warnf("illegal key/value filter (ignoring): %s", unsplit)
-				continue
-			}
-			labelName := strings.TrimSpace(split[0])
-			val := strings.TrimSpace(split[1])
-			val, wildcard := parseWildcardEnd(val)
-
-			subFilter := filter.StringMapProperty[*kubecost.CloudCost]{
-				Field: filterField,
-				// All v1 filters are equality comparisons
-				Op:    filter.StringMapEquals,
-				Key:   labelName,
-				Value: val,
-			}
-
-			if wildcard {
-				subFilter.Op = filter.StringMapStartsWith
-			}
-
-			result.Filters = append(result.Filters, subFilter)
-		}
-	}
-
-	return result
-}

pkg/filter21/allocation/fields.go

@@ -0,0 +1,37 @@
+package allocation
+
+// AllocationField is an enum that represents Allocation-specific fields that can be
+// filtered on (namespace, label, etc.)
+type AllocationField string
+
+// If you add a AllocationFilterField, make sure to update field maps to return the correct
+// Allocation value
+// does not enforce exhaustive pattern matching on "enum" types.
+const (
+	FieldClusterID      AllocationField = "cluster"
+	FieldNode           AllocationField = "node"
+	FieldNamespace      AllocationField = "namespace"
+	FieldControllerKind AllocationField = "controllerKind"
+	FieldControllerName AllocationField = "controllerName"
+	FieldPod            AllocationField = "pod"
+	FieldContainer      AllocationField = "container"
+	FieldProvider       AllocationField = "provider"
+	FieldServices       AllocationField = "services"
+	FieldLabel          AllocationField = "label"
+	FieldAnnotation     AllocationField = "annotation"
+)
+
+// AllocationAlias represents an alias field type for allocations.
+// Filtering based on label aliases (team, department, etc.) should be a
+// responsibility of the query handler. By the time it reaches this
+// structured representation, we shouldn't have to be aware of what is
+// aliased to what.
+type AllocationAlias string
+
+const (
+	AliasDepartment  AllocationAlias = "department"
+	AliasEnvironment AllocationAlias = "environment"
+	AliasOwner       AllocationAlias = "owner"
+	AliasProduct     AllocationAlias = "product"
+	AliasTeam        AllocationAlias = "team"
+)

pkg/filter21/allocation/parser.go

@@ -0,0 +1,51 @@
+package allocation
+
+import "github.com/opencost/opencost/pkg/filter21/ast"
+
+// a slice of all the allocation field instances the lexer should recognize as
+// valid left-hand comparators
+var allocationFilterFields []*ast.Field = []*ast.Field{
+	ast.NewField(FieldClusterID),
+	ast.NewField(FieldNode),
+	ast.NewField(FieldNamespace),
+	ast.NewField(FieldControllerName),
+	ast.NewField(FieldControllerKind),
+	ast.NewField(FieldContainer),
+	ast.NewField(FieldPod),
+	ast.NewField(FieldProvider),
+	ast.NewAliasField(AliasDepartment),
+	ast.NewAliasField(AliasEnvironment),
+	ast.NewAliasField(AliasOwner),
+	ast.NewAliasField(AliasProduct),
+	ast.NewAliasField(AliasTeam),
+	ast.NewSliceField(FieldServices),
+	ast.NewMapField(FieldLabel),
+	ast.NewMapField(FieldAnnotation),
+}
+
+// fieldMap is a lazily loaded mapping from AllocationField to ast.Field
+var fieldMap map[AllocationField]*ast.Field
+
+// DefaultFieldByName returns only default allocation filter fields by name.
+func DefaultFieldByName(field AllocationField) *ast.Field {
+	if fieldMap == nil {
+		fieldMap = make(map[AllocationField]*ast.Field, len(allocationFilterFields))
+		for _, f := range allocationFilterFields {
+			ff := *f
+			fieldMap[AllocationField(ff.Name)] = &ff
+		}
+	}
+
+	if af, ok := fieldMap[field]; ok {
+		afcopy := *af
+		return &afcopy
+	}
+
+	return nil
+}
+
+// NewAllocationFilterParser creates a new `ast.FilterParser` implementation
+// which uses allocation specific fields
+func NewAllocationFilterParser() ast.FilterParser {
+	return ast.NewFilterParser(allocationFilterFields)
+}

pkg/filter21/allocation/parser_test.go

@@ -0,0 +1,289 @@
+package allocation
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/opencost/opencost/pkg/filter21/ast"
+)
+
+var parser ast.FilterParser = NewAllocationFilterParser()
+
+func TestParse(t *testing.T) {
+	cases := []struct {
+		name  string
+		input string
+	}{
+		{
+			name: "Empty",
+			input: `              
+			
+			`,
+		},
+		{
+			name:  "Single",
+			input: `namespace: "kubecost"`,
+		},
+		{
+			name:  "Single Group",
+			input: `(namespace: "kubecost")`,
+		},
+		{
+			name:  "Single Double Group",
+			input: `((namespace: "kubecost"))`,
+		},
+		{
+			name:  "And 2x Expression",
+			input: `(namespace: "kubecost" + services~:"foo")`,
+		},
+		{
+			name:  "And 4x Expression",
+			input: `(namespace: "kubecost" + services~:"foo" + cluster:"cluster-one" + controllerKind:"deployment")`,
+		},
+		{
+			name:  "Nested And Groups",
+			input: `namespace: "kubecost" + services~:"foo" + (cluster:"cluster-one" + controllerKind:"deployment")`,
+		},
+		{
+			name:  "Nested Or Groups",
+			input: `namespace: "kubecost" | services~:"foo" | (cluster:"cluster-one" | controllerKind:"deployment")`,
+		},
+		{
+			name:  "Nested AndOr Groups",
+			input: `namespace: "kubecost" + services~:"foo" + (cluster:"cluster-one" | controllerKind:"deployment")`,
+		},
+		{
+			name:  "Nested OrAnd Groups",
+			input: `namespace: "kubecost" | services~:"foo" | (cluster:"cluster-one" + controllerKind:"deployment")`,
+		},
+		{
+			name:  "Nested OrAndOr Groups",
+			input: `namespace: "kubecost" | services~:"foo" | (cluster:"cluster-one" + controllerKind:"deployment") | namespace:"bar","test"`,
+		},
+		{
+			name:  "Non-uniform Whitespace",
+			input: `node:"node a b c" , "node 12 3"` + string('\n') + "+" + string('\n') + string('\r') + `namespace : "kubecost"`,
+		},
+		{
+			name:  "Group Or Comparison",
+			input: `(namespace:"kubecost" | cluster<~:"cluster-") + services~:"foo"`,
+		},
+		{
+			name:  "Group Or Group",
+			input: `(label~:"foo" + label[foo]:"bar") | (label!~:"foo" + annotation~:"foo" + annotation[foo]:"bar")`,
+		},
+		{
+			name:  "MultiDepth Groups",
+			input: `namespace: "kubecost" | ((services~:"foo" | (cluster:"cluster-one" + controllerKind:"deployment") | namespace:"bar","test") + cluster~:"cluster-")`,
+		},
+		{
+			name: "Long Query",
+			input: `
+				namespace:"kubecost" +
+				(label[app]:"cost_analyzer" +
+				annotation[a1]:"b2" +
+				cluster:"cluster-one") +
+				node!:
+				"node-123",
+				"node-456" +
+				controllerName:
+				"kubecost-cost-analyzer",
+				"kubecost-prometheus-server" +
+				controllerKind!:
+				"daemonset",
+				"statefulset",
+				"job" +
+				container!:"123-abc_foo" +
+				pod!:"aaaaaaaaaaaaaaaaaaaaaaaaa" +
+				services~:"abc123" + 
+				owner!:"kubecost"
+			`,
+		},
+	}
+
+	for i, c := range cases {
+		t.Run(fmt.Sprintf("%d:%s", i, c.name), func(t *testing.T) {
+			t.Logf("Query: %s", c.input)
+			tree, err := parser.Parse(c.input)
+			if err != nil {
+				t.Fatalf("Unexpected parse error: %s", err)
+			}
+			t.Logf("%s", ast.ToPreOrderString(tree))
+		})
+	}
+}
+
+func TestFailingParses(t *testing.T) {
+	cases := []struct {
+		name   string
+		input  string
+		errors int
+	}{
+		{
+			name:   "Empty Parens",
+			input:  `()`,
+			errors: 1,
+		},
+		{
+			name:   "Invalid Op",
+			input:  `namespace.:"kubecost"`,
+			errors: 1,
+		},
+		{
+			name:   "Extra Closing Paren",
+			input:  `(namespace:"kubecost"))`,
+			errors: 1,
+		},
+		{
+			name:   "Extra Opening Paren",
+			input:  `((namespace:"kubecost")`,
+			errors: 1,
+		},
+		{
+			name:   "Or And Mixing",
+			input:  `namespace:"kubecost" | services~:"foo" + cluster:"bar"`,
+			errors: 1,
+		},
+		{
+			name:   "And Or Mixing",
+			input:  `namespace:"kubecost" + services~:"foo" | cluster:"bar"`,
+			errors: 1,
+		},
+		{
+			name:   "And Or Mixing With Extra Closing Paren",
+			input:  `(namespace:"kubecost" + (services~:"foo" | cluster:"bar") | controllerKind<~:"dep"))`,
+			errors: 2,
+		},
+		// NOTE: This test includes coverage for an extra closing paren _early_, which basically enforces an
+		// NOTE: early return. Scoping errors don't allow the parser to continue collecting errors.
+		{
+			name:   "And Or Mixing With Extra Early Closing Paren",
+			input:  `(namespace:"kubecost" + (services~:"foo" | cluster:"bar")) | controllerKind<~:"dep")`,
+			errors: 1,
+		},
+	}
+
+	for i, c := range cases {
+		t.Run(fmt.Sprintf("%d:%s", i, c.name), func(t *testing.T) {
+			t.Logf("Query: %s", c.input)
+			tree, err := parser.Parse(c.input)
+			if err == nil {
+				t.Fatalf("Expected parsing failure. Instead, got a valid tree: \n%s\n", ast.ToPreOrderString(tree))
+			}
+
+			t.Logf("Errors: %s\n", err)
+
+			mErr := errors.Unwrap(err)
+			totalErrors := len(mErr.(*multierror.Error).Errors)
+			if totalErrors != c.errors {
+				t.Fatalf("Expected %d errors from parsing. Got %d", c.errors, totalErrors)
+			}
+		})
+	}
+}
+
+func TestShortPrint(t *testing.T) {
+	cases := []struct {
+		name  string
+		input string
+	}{
+		{
+			name: "Empty",
+			input: `              
+			
+			`,
+		},
+		{
+			name:  "Single",
+			input: `namespace: "kubecost"`,
+		},
+		{
+			name:  "Single Group",
+			input: `(namespace: "kubecost")`,
+		},
+		{
+			name:  "Single Double Group",
+			input: `((namespace: "kubecost"))`,
+		},
+		{
+			name:  "And 2x Expression",
+			input: `(namespace: "kubecost" + services~:"foo")`,
+		},
+		{
+			name:  "And 4x Expression",
+			input: `(namespace: "kubecost" + services~:"foo" + cluster:"cluster-one" + controllerKind:"deployment")`,
+		},
+		{
+			name:  "Nested And Groups",
+			input: `namespace: "kubecost" + services~:"foo" + (cluster:"cluster-one" + controllerKind:"deployment")`,
+		},
+		{
+			name:  "Nested Or Groups",
+			input: `namespace: "kubecost" | services~:"foo" | (cluster:"cluster-one" | controllerKind:"deployment")`,
+		},
+		{
+			name:  "Nested AndOr Groups",
+			input: `namespace: "kubecost" + services~:"foo" + (cluster:"cluster-one" | controllerKind:"deployment")`,
+		},
+		{
+			name:  "Nested OrAnd Groups",
+			input: `namespace: "kubecost" | services~:"foo" | (cluster:"cluster-one" + controllerKind:"deployment")`,
+		},
+		{
+			name:  "Nested OrAndOr Groups",
+			input: `namespace: "kubecost" | services~:"foo" | (cluster:"cluster-one" + controllerKind:"deployment") | namespace:"bar","test"`,
+		},
+		{
+			name:  "Non-uniform Whitespace",
+			input: `node:"node a b c" , "node 12 3"` + string('\n') + "+" + string('\n') + string('\r') + `namespace : "kubecost"`,
+		},
+		{
+			name:  "Group Or Comparison",
+			input: `(namespace:"kubecost" | cluster<~:"cluster-") + services~:"foo"`,
+		},
+		{
+			name:  "Group Or Group",
+			input: `(label~:"foo" + label[foo]:"bar") | (label!~:"foo" + annotation~:"foo" + annotation[foo]:"bar")`,
+		},
+		{
+			name:  "MultiDepth Groups",
+			input: `namespace: "kubecost" | ((services~:"foo" | (cluster:"cluster-one" + controllerKind:"deployment") | namespace:"bar","test") + cluster~:"cluster-")`,
+		},
+		{
+			name: "Long Query",
+			input: `
+				namespace:"kubecost" +
+				(label[app]:"cost_analyzer" +
+				annotation[a1]:"b2" +
+				cluster:"cluster-one") +
+				node!:
+				"node-123",
+				"node-456" +
+				controllerName:
+				"kubecost-cost-analyzer",
+				"kubecost-prometheus-server" +
+				controllerKind!:
+				"daemonset",
+				"statefulset",
+				"job" +
+				container!:"123-abc_foo" +
+				pod!:"aaaaaaaaaaaaaaaaaaaaaaaaa" +
+				services~:"abc123" + 
+				owner!:"kubecost"
+			`,
+		},
+	}
+
+	for i, c := range cases {
+		t.Run(fmt.Sprintf("%d:%s", i, c.name), func(t *testing.T) {
+			t.Logf("Query: %s", c.input)
+			tree, err := parser.Parse(c.input)
+			if err != nil {
+				t.Fatalf("Unexpected parse error: %s", err)
+			}
+			t.Logf("%s", ast.ToPreOrderShortString(tree))
+		})
+	}
+}

pkg/filter21/ast/fields.go

@@ -0,0 +1,79 @@
+package ast
+
+// FieldType is an enumeration of specific types relevant to lexing and
+// parsing a filter.
+type FieldType int
+
+const (
+	FieldTypeDefault FieldType = iota
+	FieldTypeSlice
+	FieldTypeMap
+	FieldTypeAlias
+)
+
+// Field is a Lexer input which acts as a mapping of identifiers used to lex/parse filters.
+type Field struct {
+	// Name contains the name of the specific field as it appears in language.
+	Name string
+
+	fieldType FieldType
+}
+
+// Field equivalence is determined by name and type.
+func (f *Field) Equal(other *Field) bool {
+	if f == nil || other == nil {
+		return false
+	}
+
+	return f.Name == other.Name && f.fieldType == other.fieldType
+}
+
+// IsSlice returns true if the field is a slice. This instructs the lexer that the field
+// should allow contains operations.
+func (f *Field) IsSlice() bool {
+	return f.fieldType == FieldTypeSlice
+}
+
+// IsMap returns true if the field is a map. This instructs the lexer that the field should
+// allow keyed-access operations.
+func (f *Field) IsMap() bool {
+	return f.fieldType == FieldTypeMap
+}
+
+// IsAlias returns true if the field is an alias type. This instructs the lexer that the field
+// is an alias for custom logical resolution by an external compiler.
+func (f *Field) IsAlias() bool {
+	return f.fieldType == FieldTypeAlias
+}
+
+// NewField creates a default string field using the provided name.
+func NewField[T ~string](name T) *Field {
+	return &Field{
+		Name:      string(name),
+		fieldType: FieldTypeDefault,
+	}
+}
+
+// NewSliceField creates a slice field using the provided name.
+func NewSliceField[T ~string](name T) *Field {
+	return &Field{
+		Name:      string(name),
+		fieldType: FieldTypeSlice,
+	}
+}
+
+// NewMapField creates a new map field using the provided name.
+func NewMapField[T ~string](name T) *Field {
+	return &Field{
+		Name:      string(name),
+		fieldType: FieldTypeMap,
+	}
+}
+
+// NewAliasField creates a new alias field using the provided name.
+func NewAliasField[T ~string](name T) *Field {
+	return &Field{
+		Name:      string(name),
+		fieldType: FieldTypeAlias,
+	}
+}

pkg/util/allocationfilterutil/v2/lexer.go → pkg/filter21/ast/lexer.go

@@ -1,11 +1,9 @@
-package allocationfilterutil
+package ast
 
 import (
 	"fmt"
 
 	multierror "github.com/hashicorp/go-multierror"
-
-	"github.com/opencost/opencost/pkg/kubecost"
 )
 
 // ============================================================================
@@ -21,37 +19,29 @@ const (
 	colon tokenKind = iota // ':'
 	comma                  // ','
 	plus                   // '+'
+	or                     // '|'
+
+	bangColon           // '!:'
+	tildeColon          // '~:'
+	bangTildeColon      // '!~:'
+	startTildeColon     // '<~:'
+	bangStartTildeColon // '!<~:'
+	tildeEndColon       // '~>:'
+	bangTildeEndColon   // '!~>:'
 
-	bangColon // '!:'
+	parenOpen  // '('
+	parenClose // ')'
 
 	str // '"foo"'
 
-	filterField1 // 'namespace', 'cluster'
-	filterField2 // 'label', 'annotation'
-	keyedAccess  // '[app]', '[foo]', etc.
-	identifier   // K8s valid name + sanitized Prom: 'app', 'abc_label'
+	filterField // 'namespace', 'cluster'
+	mapField    // 'label', 'annotation'
+	keyedAccess // '[app]', '[foo]', etc.
+	identifier  // K8s valid name + sanitized Prom: 'app', 'abc_label'
 
 	eof
 )
 
-// These maps serve a dual purpose. (1) to help the lexer identify special
-// strings that should become filterField1/2 instead of identifiers and (2) to
-// help the parser convert tokens into AllocationFilterConditions.
-var ff1ToKCFilterField = map[string]kubecost.FilterField{
-	"cluster":        kubecost.FilterClusterID,
-	"node":           kubecost.FilterNode,
-	"namespace":      kubecost.FilterNamespace,
-	"controllerName": kubecost.FilterControllerName,
-	"controllerKind": kubecost.FilterControllerKind,
-	"container":      kubecost.FilterContainer,
-	"pod":            kubecost.FilterPod,
-	"services":       kubecost.FilterServices,
-}
-var ff2ToKCFilterField = map[string]kubecost.FilterField{
-	"label":      kubecost.FilterLabel,
-	"annotation": kubecost.FilterAnnotation,
-}
-
 func (tk tokenKind) String() string {
 	switch tk {
 	case colon:
@@ -60,13 +50,31 @@ func (tk tokenKind) String() string {
 		return "comma"
 	case plus:
 		return "plus"
+	case or:
+		return "or"
 	case bangColon:
 		return "bangColon"
+	case tildeColon:
+		return "tildeColon"
+	case bangTildeColon:
+		return "bangTildeColon"
+	case startTildeColon:
+		return "startTildeColon"
+	case bangStartTildeColon:
+		return "bangStartTildeColon"
+	case tildeEndColon:
+		return "tildeEndColon"
+	case bangTildeEndColon:
+		return "bangTildeEndColon"
+	case parenOpen:
+		return "parenOpen"
+	case parenClose:
+		return "parenClose"
 	case str:
 		return "str"
-	case filterField1:
+	case filterField:
 		return "filterField1"
-	case filterField2:
+	case mapField:
 		return "filterField2"
 	case keyedAccess:
 		return "keyedAccess"
@@ -100,6 +108,9 @@ type scanner struct {
 	tokens []token
 	errors []error
 
+	fields    map[string]*Field
+	mapFields map[string]*Field
+
 	lexemeStartByte int
 	nextByte        int
 }
@@ -169,12 +180,62 @@ func (s *scanner) scanToken() {
 		s.addToken(comma)
 	case '+':
 		s.addToken(plus)
+	case '|':
+		s.addToken(or)
 	case '!':
 		if s.match(':') {
 			s.addToken(bangColon)
+		} else if s.match('~') {
+			if s.match(':') {
+				s.addToken(bangTildeColon)
+			} else if s.match('>') {
+				if s.match(':') {
+					s.addToken(bangTildeEndColon)
+				} else {
+					s.errors = append(s.errors, fmt.Errorf("Position %d: Unexpected '>'", s.nextByte-1))
+				}
+			} else {
+				s.errors = append(s.errors, fmt.Errorf("Position %d: Unexpected '~'", s.nextByte-1))
+			}
+		} else if s.match('<') {
+			if s.match('~') {
+				if s.match(':') {
+					s.addToken(bangStartTildeColon)
+				} else {
+					s.errors = append(s.errors, fmt.Errorf("Position %d: Unexpected '~'", s.nextByte-1))
+				}
+			} else {
+				s.errors = append(s.errors, fmt.Errorf("Position %d: Unexpected '<'", s.nextByte-1))
+			}
 		} else {
 			s.errors = append(s.errors, fmt.Errorf("Position %d: Unexpected '!'", s.nextByte-1))
 		}
+	case '(':
+		s.addToken(parenOpen)
+	case ')':
+		s.addToken(parenClose)
+	case '<':
+		if s.match('~') {
+			if s.match(':') {
+				s.addToken(startTildeColon)
+			} else {
+				s.errors = append(s.errors, fmt.Errorf("Position %d: Unexpected '~'", s.nextByte-1))
+			}
+		} else {
+			s.errors = append(s.errors, fmt.Errorf("Position %d: Unexpected '<'", s.nextByte-1))
+		}
+	case '~':
+		if s.match(':') {
+			s.addToken(tildeColon)
+		} else if s.match('>') {
+			if s.match(':') {
+				s.addToken(tildeEndColon)
+			} else {
+				s.errors = append(s.errors, fmt.Errorf("Position %d: Unexpected '>'", s.nextByte-1))
+			}
+		} else {
+			s.errors = append(s.errors, fmt.Errorf("Position %d: Unexpected '~'", s.nextByte-1))
+		}
 	// strings
 	case '"':
 		s.string()
@@ -255,17 +316,22 @@ func (s *scanner) identifier() {
 	}
 
 	tokenText := s.source[s.lexemeStartByte:s.nextByte]
-	if _, ok := ff1ToKCFilterField[tokenText]; ok {
-		s.addToken(filterField1)
-	} else if _, ok := ff2ToKCFilterField[tokenText]; ok {
-		s.addToken(filterField2)
+	if _, ok := s.fields[tokenText]; ok {
+		s.addToken(filterField)
+	} else if _, ok := s.mapFields[tokenText]; ok {
+		s.addToken(mapField)
 	} else {
 		s.addToken(identifier)
 	}
 }
 
-func lexAllocationFilterV2(raw string) ([]token, error) {
-	s := scanner{source: raw}
+// lex will generate a slice of tokens provided a raw string and the filter field definitions
+func lex(raw string, fields map[string]*Field, mapFields map[string]*Field) ([]token, error) {
+	s := scanner{
+		source:    raw,
+		fields:    fields,
+		mapFields: mapFields,
+	}
 	s.scanTokens()
 
 	if len(s.errors) > 0 {

pkg/util/allocationfilterutil/v2/lexer_test.go → pkg/filter21/ast/lexer_test.go

@@ -1,9 +1,40 @@
-package allocationfilterutil
+package ast
 
 import (
 	"testing"
 )
 
+var allocFields map[string]*Field = map[string]*Field{
+	"cluster":        NewField("cluster"),
+	"node":           NewField("node"),
+	"namespace":      NewField("namespace"),
+	"controllerName": NewField("controllerName"),
+	"controllerKind": NewField("controllerKind"),
+	"container":      NewField("container"),
+	"pod":            NewField("pod"),
+	"services":       NewSliceField("services"),
+}
+
+var allocMapFields map[string]*Field = map[string]*Field{
+	"label":      NewMapField("label"),
+	"annotation": NewMapField("annotation"),
+}
+
+func TestLexerGroup(t *testing.T) {
+	tokens, err := lex(
+		`cluster:"cluster-one"+namespace:"kubecost"+(controllerKind!:"daemonset","deployment")+controllerName:"kubecost-network-costs"+container:"kubecost-network-costs"`,
+		allocFields,
+		allocMapFields)
+
+	if err != nil {
+		t.Errorf("Error: %s", err)
+	}
+
+	for _, token := range tokens {
+		t.Logf("%s", token)
+	}
+}
+
 func TestLexer(t *testing.T) {
 	cases := []struct {
 		name string
@@ -32,14 +63,49 @@ func TestLexer(t *testing.T) {
 			input:    "+",
 			expected: []token{{kind: plus, s: "+"}, {kind: eof}},
 		},
+		{
+			name:     "or",
+			input:    "|",
+			expected: []token{{kind: or, s: "|"}, {kind: eof}},
+		},
 		{
 			name:     "bangColon",
 			input:    "!:",
 			expected: []token{{kind: bangColon, s: "!:"}, {kind: eof}},
 		},
+		{
+			name:     "tildeColon",
+			input:    "~:",
+			expected: []token{{kind: tildeColon, s: "~:"}, {kind: eof}},
+		},
+		{
+			name:     "bangTildeColon",
+			input:    "!~:",
+			expected: []token{{kind: bangTildeColon, s: "!~:"}, {kind: eof}},
+		},
+		{
+			name:     "startTildeColon",
+			input:    "<~:",
+			expected: []token{{kind: startTildeColon, s: "<~:"}, {kind: eof}},
+		},
+		{
+			name:     "bangStartTildeColon",
+			input:    "!<~:",
+			expected: []token{{kind: bangStartTildeColon, s: "!<~:"}, {kind: eof}},
+		},
+		{
+			name:     "tildeEndColon",
+			input:    "~>:",
+			expected: []token{{kind: tildeEndColon, s: "~>:"}, {kind: eof}},
+		},
+		{
+			name:     "bangTildeEndColon",
+			input:    "!~>:",
+			expected: []token{{kind: bangTildeEndColon, s: "!~>:"}, {kind: eof}},
+		},
 		{
 			name: "multiple symbols",
-			// This is a valid string to lex but not to parse.
+			// This is a valid string to parse but not to lex
 			input:    "!::,+",
 			expected: []token{{kind: bangColon, s: "!:"}, {kind: colon, s: ":"}, {kind: comma, s: ","}, {kind: plus, s: "+"}, {kind: eof}},
 		},
@@ -79,12 +145,12 @@ func TestLexer(t *testing.T) {
 			name:  "whitespace separated accesses",
 			input: `node : "abc" , "def" ` + string('\r') + string('\n') + string('\t') + `namespace : "123"`,
 			expected: []token{
-				{kind: filterField1, s: "node"},
+				{kind: filterField, s: "node"},
 				{kind: colon, s: ":"},
 				{kind: str, s: "abc"},
 				{kind: comma, s: ","},
 				{kind: str, s: "def"},
-				{kind: filterField1, s: "namespace"},
+				{kind: filterField, s: "namespace"},
 				{kind: colon, s: ":"},
 				{kind: str, s: "123"},
 				{kind: eof},
@@ -95,7 +161,7 @@ func TestLexer(t *testing.T) {
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
 			t.Logf("Input: '%s'", c.input)
-			result, err := lexAllocationFilterV2(c.input)
+			result, err := lex(c.input, allocFields, allocMapFields)
 			if c.expectError && err == nil {
 				t.Errorf("expected error but got nil")
 			} else if !c.expectError && err != nil {

pkg/filter21/ast/ops.go

@@ -0,0 +1,193 @@
+package ast
+
+// FilterOp is an enum that represents operations that can be performed
+// when filtering (equality, inequality, etc.)
+type FilterOp string
+
+// If you add a FilterOp, MAKE SURE TO UPDATE ALL FILTER IMPLEMENTATIONS! Go
+// does not enforce exhaustive pattern matching on "enum" types.
+const (
+	// FilterOpEquals is the equality operator
+	//
+	// "kube-system" FilterOpEquals "kube-system" = true
+	// "kube-syste" FilterOpEquals "kube-system" = false
+	FilterOpEquals FilterOp = "equals"
+
+	// FilterOpNotEquals is the inverse of equals.
+	FilterOpNotEquals = "notequals"
+
+	// FilterOpContains supports string fields, slice fields, and map fields.
+	// For maps, this is equivalent to map.HasKey(x)
+	//
+	// "kube-system" FilterOpContains "e-s" = true
+	// ["a", "b", "c"] FilterOpContains "a" = true
+	// { "namespace": "kubecost", "cluster": "cluster-one" } FilterOpContains "namespace" = true
+	FilterOpContains = "contains"
+
+	// FilterOpNotContains is the inverse of contains.
+	FilterOpNotContains = "notcontains"
+
+	// FilterOpContainsPrefix is like FilterOpContains, but checks against the start of a string.
+	// For maps, this checks to see if any of the keys start with the prefix
+	//
+	// "kube-system" ContainsPrefix "kube" = true
+	// ["kube-system", "abc123"] ContainsPrefix "kube" = true
+	// { "kube-label": "test", "abc": "123" } ContainsPrefix "ab" = true
+	FilterOpContainsPrefix = "containsprefix"
+
+	// FilterOpNotContainsPrefix is the inverse of FilterOpContainsPrefix
+	FilterOpNotContainsPrefix = "notcontainsprefix"
+
+	// FilterOpContainsSuffix is like FilterOpContains, but checks against the end of a string.
+	// For maps, this checks to see if any of the keys end with the suffix
+	//
+	// "kube-system" ContainsSuffix "system" = true
+	// ["kube-system", "abc123"] ContainsSuffix "system" = true
+	// { "kube-label": "test", "abc": "123" } ContainsSuffix "-label" = true
+	FilterOpContainsSuffix = "containssuffix"
+
+	// FilterOpNotContainsSuffix is the inverse of FilterOpContainsSuffix
+	FilterOpNotContainsSuffix = "notcontainssuffix"
+
+	// FilterOpVoid is base-depth operator that is used for an empty filter
+	FilterOpVoid = "void"
+
+	// FilterOpContradiction is a base-depth operator that filters all data.
+	FilterOpContradiction = "contradiction"
+
+	// FilterOpAnd is an operator that succeeds if all parameters succeed.
+	FilterOpAnd = "and"
+
+	// FilterOpOr is an operator that succeeds if any parameter succeeds
+	FilterOpOr = "or"
+
+	// FilterOpNot is an operator that contains a single operand
+	FilterOpNot = "not"
+)
+
+// VoidOp is base-depth operator that is used for an empty filter
+type VoidOp struct{}
+
+// Op returns the FilterOp enumeration value for the operator.
+func (_ *VoidOp) Op() FilterOp {
+	return FilterOpVoid
+}
+
+// ContradictionOp is a base-depth operator that filters all data.
+type ContradictionOp struct{}
+
+// Op returns the FilterOp enumeration value for the operator.
+func (_ *ContradictionOp) Op() FilterOp {
+	return FilterOpContradiction
+}
+
+// AndOp is a filter operation that contains a flat list of nodes which should all resolve
+// to true in order for the result to be true.
+type AndOp struct {
+	Operands []FilterNode
+}
+
+// Op returns the FilterOp enumeration value for the operator.
+func (_ *AndOp) Op() FilterOp {
+	return FilterOpAnd
+}
+
+// Add appends a filter node to the flat list of operands within the AND operator
+func (ao *AndOp) Add(node FilterNode) {
+	ao.Operands = append(ao.Operands, node)
+}
+
+// OrOp is a filter operation that contains a flat list of nodes which at least one node
+// should resolve to true in order for the result to be true.
+type OrOp struct {
+	Operands []FilterNode
+}
+
+// Op returns the FilterOp enumeration value for the operator.
+func (_ *OrOp) Op() FilterOp {
+	return FilterOpOr
+}
+
+// Add appends a filter node to the flat list of operands within the OR operator
+func (oo *OrOp) Add(node FilterNode) {
+	oo.Operands = append(oo.Operands, node)
+}
+
+// NotOp is a filter operation that logically inverts result of the child operand.
+type NotOp struct {
+	Operand FilterNode
+}
+
+// Op returns the FilterOp enumeration value for the operator.
+func (_ *NotOp) Op() FilterOp {
+	return FilterOpNot
+}
+
+// Add sets the not operand to the parameter
+func (no *NotOp) Add(node FilterNode) {
+	no.Operand = node
+}
+
+// EqualOp is a filter operation that compares a resolvable identifier (Left) to a
+// string value (Right)
+type EqualOp struct {
+	// Left contains a resolvable Identifier (property of an input type) which can be
+	// used to compare against the Right value.
+	Left Identifier
+
+	// Right contains the value which we wish to compare the resolved identifier to.
+	Right string
+}
+
+// Op returns the FilterOp enumeration value for the operator.
+func (_ *EqualOp) Op() FilterOp {
+	return FilterOpEquals
+}
+
+// ContainsOp is a filter operation that checks to see if a resolvable identifier (Left) contains a
+// string value (Right)
+type ContainsOp struct {
+	// Left contains a resolvable Identifier (property of an input type) which can be
+	// used to query against using the Right value.
+	Left Identifier
+
+	// Right contains the value which we use to search the resolved Left identifier with.
+	Right string
+}
+
+// Op returns the FilterOp enumeration value for the operator.
+func (_ *ContainsOp) Op() FilterOp {
+	return FilterOpContains
+}
+
+// ContainsPrefixOp is a filter operation that checks to see if a resolvable identifier (Left) starts with a
+// string value (Right)
+type ContainsPrefixOp struct {
+	// Left contains a resolvable Identifier (property of an input type) which can be
+	// used to query against using the Right value.
+	Left Identifier
+
+	// Right contains the value which we use to search the resolved Left identifier with.
+	Right string
+}
+
+// Op returns the FilterOp enumeration value for the operator.
+func (_ *ContainsPrefixOp) Op() FilterOp {
+	return FilterOpContainsPrefix
+}
+
+// ContainsSuffixOp is a filter operation that checks to see if a resolvable identifier (Left) ends with a
+// string value (Right)
+type ContainsSuffixOp struct {
+	// Left contains a resolvable Identifier (property of an input type) which can be
+	// used to query against using the Right value.
+	Left Identifier
+
+	// Right contains the value which we use to search the resolved Left identifier with.
+	Right string
+}
+
+// Op returns the FilterOp enumeration value for the operator.
+func (_ *ContainsSuffixOp) Op() FilterOp {
+	return FilterOpContainsSuffix
+}

pkg/filter21/ast/parser.go

@@ -0,0 +1,589 @@
+// allocationfilterutil provides functionality for parsing V2 of the Kubecost
+// filter language for Allocation types.
+//
+// e.g. "filter=namespace:kubecost+controllerkind:deployment"
+package ast
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/go-multierror"
+)
+
+// The grammar is approximately as follows:
+//
+// <filter>         ::= <filter-element> (<group-op> <filter-element>)*
+// <filter-element> ::= <comparison> | <group-filter>
+// <group-filter>   ::= '(' <filter> ')'
+// <group-op>       ::= '+' | '|'
+// <comparison>     ::= <filter-key> <filter-op> <filter-value>
+// <filter-key>     ::= <map-field> <keyed-access> | <filter-field>
+// <filter-op>      ::= ':' | '!:' | '~:' | '!~:' | '<~:' | '!<~:' | '~>:' | '!~>:'
+// <filter-value>   ::= '"' <identifier> '"' (',' <filter-value>)*
+// <keyed-access>   ::= '[' <identifier> ']'
+// <map-field>      ::= --- (fields passed into lexer)
+// <filter-field>   ::= --- (fields passed into lexer)
+// <identifier>     ::= --- valid K8s name or Prom-sanitized K8s name
+
+// ============================================================================
+// Parser
+//
+// Based on the Parser class in Chapter 6: Parsing Expressions of Crafting
+// Interpreters by Robert Nystrom
+// ============================================================================
+
+// parseError produces error messages tailored to the needs of the parser
+func parseError(t token, message string) error {
+	if t.kind == eof {
+		return fmt.Errorf("at end: %s", message)
+	}
+
+	return fmt.Errorf("at '%s': %s", t.s, message)
+}
+
+type parser struct {
+	tokens  []token
+	current int
+
+	fields    map[string]*Field
+	mapFields map[string]*Field
+}
+
+// ----------------------------------------------------------------------------
+// Parser helper methods for token handling
+// ----------------------------------------------------------------------------
+
+func (p *parser) atEnd() bool {
+	return p.peek().kind == eof
+}
+
+func (p *parser) advance() token {
+	if !p.atEnd() {
+		p.current += 1
+	}
+
+	return p.previous()
+}
+
+func (p *parser) previous() token {
+	return p.tokens[p.current-1]
+}
+
+// match return true and advances the parser by one token if the next token has
+// a kind that matches one of the arguments. Otherwise, it returns false and
+// DOES NOT advance the parser.
+func (p *parser) match(tokenKinds ...tokenKind) bool {
+	for _, kind := range tokenKinds {
+		if p.check(kind) {
+			p.advance()
+			return true
+		}
+	}
+	return false
+}
+
+// check returns true iff the next token matches the provided kind.
+func (p *parser) check(tk tokenKind) bool {
+	if p.atEnd() {
+		return false
+	}
+	return p.peek().kind == tk
+}
+
+func (p *parser) peek() token {
+	return p.tokens[p.current]
+}
+
+// consume is a "next token must be this kind" method. If the next token is of
+// the correct kind, the parser is advanced and that token is returned. If it
+// is not of the correct kind, a parse error is returned and the parser is NOT
+// advanced.
+func (p *parser) consume(tk tokenKind, message string) (token, error) {
+	if p.check(tk) {
+		return p.advance(), nil
+	}
+
+	return token{}, parseError(p.peek(), message)
+}
+
+// synchronize attempts to skip forward until the next tokenKind, indicating the
+// start of a new (plus, or, or parenClose).
+func (p *parser) synchronize(tokens ...tokenKind) {
+	if len(tokens) == 0 {
+		return
+	}
+
+	for !p.atEnd() {
+		kind := p.peek().kind
+		for _, token := range tokens {
+			if kind == token {
+				return
+			}
+		}
+
+		p.advance()
+	}
+}
+
+// ----------------------------------------------------------------------------
+// Parser grammar rules as recursive descent methods
+// ----------------------------------------------------------------------------
+
+// filter is the main method of the parser. It turns the token stream into an
+// FilterNode tree, reporting parse errors that occurred along the way. The depth
+// parameter is the number of edges from the node to the tree's root node, which
+// is initially 0. As we recurse into the tree, the depth will increase.
+func (p *parser) filter(depth int) (FilterNode, error) {
+	var errs *multierror.Error
+
+	// ----------------------------------------------------------------------------
+	//  Capture Starting Op
+	// ----------------------------------------------------------------------------
+	// Since every valid filter starts with an operand, this is always our first
+	// step. Depending on the _next_ token, we can either stop here or use a grouping
+	// operator (+ or |).
+	var top FilterNode
+
+	// If we determine after parsing the first op that we have a group op, we'll create
+	// the group based on the operator and push the top into the group.
+	var f FilterGroup = nil
+
+	// Special Case: Empty Filter on depth = 0 and first token is eof
+	if depth == 0 && p.peek().kind == eof {
+		return &VoidOp{}, errs.ErrorOrNil()
+	}
+
+	// Open Paren indicates a new filter depth, so we recursively call filter with depth+1.
+	if p.match(parenOpen) {
+		node, err := p.filter(depth + 1)
+		if err != nil {
+			errs = multierror.Append(errs, err)
+		} else {
+			top = node
+		}
+	} else {
+		comparison, err := p.comparison()
+		if err != nil {
+			errs = multierror.Append(errs, err)
+			p.synchronize(plus, or, parenClose)
+		} else {
+			top = comparison
+		}
+	}
+
+	// Handles case `( <comparison> )` with no grouping ops.
+	if p.match(parenClose) {
+		if depth <= 0 {
+			errs = multierror.Append(errs, fmt.Errorf("Found ')' without matching '('"))
+		}
+
+		return top, errs.ErrorOrNil()
+	}
+
+	// ----------------------------------------------------------------------------
+	//  Determine Group Operator
+	// ----------------------------------------------------------------------------
+	// Once we land here, we expect an operator as the next token. This operator will
+	// determine the group for this scope and be used to continue parsing as long as
+	// the operators following the initial are _the same_.
+	//
+	// For instance:
+	// ( <comparison> | <comparison> | <comparison> ) is allowed
+	// ( <comparison> + <comparison> + (<comparison> | <comparison>)) is allowed
+	// ( <comparison> | <comparison> + <comparison> ) is _NOT_ allowed
+
+	// Create the proper grouping operator based on the current token kind,
+	// then use a while to capture each repition of the _same_ operator.
+	selectedOp := p.peek().kind
+	if selectedOp == plus || selectedOp == or {
+		if selectedOp == plus {
+			f = &AndOp{}
+		} else if selectedOp == or {
+			f = &OrOp{}
+		}
+
+		// Once we determine we are using a group operator, it's safe to push
+		// the current top level operand into the group
+		f.Add(top)
+
+		// Capture each repetition
+		for p.match(selectedOp) {
+			if p.match(parenOpen) {
+				node, err := p.filter(depth + 1)
+				if err != nil {
+					errs = multierror.Append(errs, err)
+				} else {
+					f.Add(node)
+				}
+			} else {
+				right, err := p.comparison()
+				if err != nil {
+					errs = multierror.Append(errs, err)
+					p.synchronize(plus, or, parenClose)
+				} else {
+					f.Add(right)
+				}
+			}
+
+			if p.match(parenClose) {
+				if depth <= 0 {
+					errs = multierror.Append(errs, fmt.Errorf("Found ')' without matching '('"))
+				}
+
+				return f, errs.ErrorOrNil()
+			}
+
+			// The following code enforces continued use of a single operator within a scope.
+			// ie: (a | b + c) is disallowed
+			//
+			// In order to continue parsing (to continue to collect parse errors), we need to fast-
+			// forward to the next instance of an operator or scope close.
+			nextOp := p.peek().kind
+			if nextOp != eof && nextOp != selectedOp {
+				errs = multierror.Append(errs, fmt.Errorf("Found \"%s\", Expected \"%s\"", nextOp.String(), selectedOp.String()))
+				// since we were peeking for this check, to correctly synchronize, we must advance at least once
+				p.advance()
+				p.synchronize(plus, or, parenClose)
+
+				// since it's possible to synchronize to a paren close, we need to ensure we correctly pop the
+				// current scope if that's the case.
+				if p.match(parenClose) {
+					return f, errs.ErrorOrNil()
+				}
+			}
+		}
+	}
+
+	// It should not be possible to reach this point on a non-zero depth, so we
+	// must have a () mismatch
+	if depth > 0 {
+		errs = multierror.Append(errs, fmt.Errorf("Found '(' without matching ')'"))
+	}
+
+	// If we didn't have a grouping operator, we simply return the single op
+	if f == nil {
+		return top, errs.ErrorOrNil()
+	}
+
+	return f, errs.ErrorOrNil()
+}
+
+func (p *parser) comparison() (FilterNode, error) {
+	field, key, err := p.filterKey()
+	if err != nil {
+		return nil, err
+	}
+
+	opToken, err := p.filterOp()
+	if err != nil {
+		return nil, err
+	}
+
+	var op FilterOp
+
+	switch opToken.kind {
+	case colon:
+		// for ':' using a slice or key-less map, treat as '~:'
+		if field.IsSlice() || (field.IsMap() && key == "") {
+			op = FilterOpContains
+		} else {
+			op = FilterOpEquals
+		}
+	case bangColon:
+		// for '!:' using a slice or key-less map, treat as '!~:'
+		if field.IsSlice() || (field.IsMap() && key == "") {
+			op = FilterOpNotContains
+		} else {
+			op = FilterOpNotEquals
+		}
+	case tildeColon:
+		op = FilterOpContains
+	case bangTildeColon:
+		op = FilterOpNotContains
+	case startTildeColon:
+		op = FilterOpContainsPrefix
+	case bangStartTildeColon:
+		op = FilterOpNotContainsPrefix
+	case tildeEndColon:
+		op = FilterOpContainsSuffix
+	case bangTildeEndColon:
+		op = FilterOpNotContainsSuffix
+	default:
+		return nil, parseError(opToken, "implementation problem: unhandled op token")
+	}
+
+	values, err := p.filterValues()
+	if err != nil {
+		return nil, err
+	}
+
+	switch opToken.kind {
+	// In the != case, a sequence of filter values is ANDed
+	// Example:
+	// namespace!:"foo","bar" -> (and (notequals namespace foo)
+	//                                (notequals namespace bar))
+	case bangColon, bangTildeColon, bangStartTildeColon, bangTildeEndColon:
+		// Only a single filter value, don't need to wrap in AND
+		if len(values) == 1 {
+			node, err := toFilterNode(field, key, op, values[0])
+			if err != nil {
+				return nil, fmt.Errorf("Parse Error: %s", err)
+			}
+
+			return node, nil
+		}
+
+		// Multiple filter values, wrap in AND
+		baseFilter := &AndOp{}
+		for _, v := range values {
+			node, err := toFilterNode(field, key, op, v)
+			if err != nil {
+				return nil, fmt.Errorf("Parse Error: %s", err)
+			}
+
+			baseFilter.Operands = append(baseFilter.Operands, node)
+		}
+
+		return baseFilter, nil
+
+	default:
+		// Only a single filter value, don't need to wrap in OR
+		if len(values) == 1 {
+			node, err := toFilterNode(field, key, op, values[0])
+			if err != nil {
+				return nil, fmt.Errorf("Parse Error: %s", err)
+			}
+
+			return node, nil
+		}
+
+		// Multiple filter values, wrap in OR
+		baseFilter := &OrOp{}
+		for _, v := range values {
+			node, err := toFilterNode(field, key, op, v)
+			if err != nil {
+				return nil, fmt.Errorf("Parse Error: %s", err)
+			}
+
+			baseFilter.Operands = append(baseFilter.Operands, node)
+		}
+
+		return baseFilter, nil
+	}
+
+}
+
+// filterKey parses a series of tokens that represent a "filter key", returning
+// an error if a filter key cannot be constructed.
+//
+// Examples:
+// tokens = [filterField2:label keyedAccess:app] -> FilterLabel, app, nil
+// tokens = [filterField1:namespace] -> FilterNamespace, "", nil
+func (p *parser) filterKey() (field *Field, key string, err error) {
+	if p.match(mapField) {
+		rawField := p.previous().s
+		mappedField, ok := p.mapFields[rawField]
+		if !ok {
+			return nil, "", parseError(p.previous(), "expect key-mapped filter field, like 'label' or 'annotation'")
+		}
+
+		// keyed-access is optional after a map field
+		if p.match(keyedAccess) {
+			key = p.previous().s
+		} else {
+			key = ""
+		}
+
+		return mappedField, key, nil
+	}
+
+	_, err = p.consume(filterField, "expect filter field")
+	if err != nil {
+		return nil, "", err
+	}
+
+	rawField := p.previous().s
+	mappedField, ok := p.fields[rawField]
+	if !ok {
+		return nil, "", parseError(p.previous(), "expect known filter field, like 'cluster' or 'namespace'")
+	}
+
+	return mappedField, "", nil
+}
+
+func (p *parser) filterOp() (token, error) {
+	if p.match(colon, bangColon, tildeColon, bangTildeColon, startTildeColon, bangStartTildeColon, tildeEndColon, bangTildeEndColon) {
+		return p.previous(), nil
+	}
+
+	return token{}, parseError(p.peek(), "expect filter op like ':', '!:', '~:', or '!~:'")
+}
+
+func (p *parser) filterValues() ([]string, error) {
+	vals := []string{}
+
+	_, err := p.consume(str, "expect string as filter value")
+	if err != nil {
+		return nil, err
+	}
+	vals = append(vals, p.previous().s)
+
+	for p.match(comma) {
+		_, err := p.consume(str, "expect string as filter value")
+		if err != nil {
+			return nil, err
+		}
+
+		vals = append(vals, p.previous().s)
+	}
+
+	return vals, nil
+}
+
+func toFilterNode(field *Field, key string, op FilterOp, value string) (FilterNode, error) {
+	switch op {
+	case FilterOpEquals:
+		return &EqualOp{
+			Left: Identifier{
+				Field: field,
+				Key:   key,
+			},
+			Right: value,
+		}, nil
+
+	case FilterOpNotEquals:
+		return &NotOp{
+			Operand: &EqualOp{
+				Left: Identifier{
+					Field: field,
+					Key:   key,
+				},
+				Right: value,
+			},
+		}, nil
+
+	case FilterOpContains:
+		return &ContainsOp{
+			Left: Identifier{
+				Field: field,
+				Key:   key,
+			},
+			Right: value,
+		}, nil
+
+	case FilterOpNotContains:
+		return &NotOp{
+			Operand: &ContainsOp{
+				Left: Identifier{
+					Field: field,
+					Key:   key,
+				},
+				Right: value,
+			},
+		}, nil
+
+	case FilterOpContainsPrefix:
+		return &ContainsPrefixOp{
+			Left: Identifier{
+				Field: field,
+				Key:   key,
+			},
+			Right: value,
+		}, nil
+
+	case FilterOpNotContainsPrefix:
+		return &NotOp{
+			Operand: &ContainsPrefixOp{
+				Left: Identifier{
+					Field: field,
+					Key:   key,
+				},
+				Right: value,
+			},
+		}, nil
+
+	case FilterOpContainsSuffix:
+		return &ContainsSuffixOp{
+			Left: Identifier{
+				Field: field,
+				Key:   key,
+			},
+			Right: value,
+		}, nil
+
+	case FilterOpNotContainsSuffix:
+		return &NotOp{
+			Operand: &ContainsSuffixOp{
+				Left: Identifier{
+					Field: field,
+					Key:   key,
+				},
+				Right: value,
+			},
+		}, nil
+
+	default:
+		return nil, fmt.Errorf("Failed to parse op: %s", op)
+	}
+}
+
+// FilterParser is an object capable of parsing a filter string into a `FilterNode`
+// AST
+type FilterParser interface {
+	// Parse parses a filter string into a FilterNode AST.
+	Parse(filter string) (FilterNode, error)
+}
+
+// default implementation of FilterParser
+type defaultFilterParser struct {
+	fields    map[string]*Field
+	mapFields map[string]*Field
+}
+
+// Parse parses a filter string into a FilterNode AST.
+func (dfp *defaultFilterParser) Parse(filter string) (FilterNode, error) {
+	tokens, err := lex(filter, dfp.fields, dfp.mapFields)
+	if err != nil {
+		return nil, fmt.Errorf("lexing filter: %w", err)
+	}
+
+	p := parser{
+		tokens:    tokens,
+		fields:    dfp.fields,
+		mapFields: dfp.mapFields,
+	}
+
+	parsedFilter, err := p.filter(0)
+	if err != nil {
+		return nil, fmt.Errorf("parsing filter: %w", err)
+	}
+
+	return parsedFilter, nil
+}
+
+// splits a slice of Field instances into a map of fields (key'd by name) that have no key-based
+// access and those that have key-based access.
+func fieldsToMaps(fs []*Field) (fields map[string]*Field, mapFields map[string]*Field) {
+	fields = make(map[string]*Field)
+	mapFields = make(map[string]*Field)
+
+	for _, f := range fs {
+		if f.IsMap() {
+			mapFields[f.Name] = f
+		} else {
+			fields[f.Name] = f
+		}
+	}
+	return
+}
+
+// NewFilterParser creates a new `FilterParser` instance with the provided `Field` definitions to
+// use when lexing and parsing.
+func NewFilterParser(fields []*Field) FilterParser {
+	f, m := fieldsToMaps(fields)
+
+	return &defaultFilterParser{
+		fields:    f,
+		mapFields: m,
+	}
+}