Merge branch 'develop' into kaelan-asset-unmarshal

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,25 @@
+## What does this PR change?
+
+
+
+## Does this PR rely on any other PRs?
+
+- 
+- 
+
+
+## How does this PR impact users? (This is the kind of thing that goes in release notes!)
+
+
+
+## Links to Issues or ZD tickets this PR addresses or fixes
+
+- 
+- 
+
+
+## How was this PR tested?
+
+
+## Have you made an update to documentation?
+

Dockerfile.metrics

@@ -0,0 +1,27 @@
+FROM golang:latest as build-env
+
+RUN mkdir /app
+WORKDIR /app
+COPY go.mod .
+COPY go.sum .
+
+# Get dependencies - will also be cached if we won't change mod/sum
+RUN go mod download
+# COPY the source code as the last step
+COPY . .
+# Build the binary
+RUN set -e ;\
+    cd cmd/kubemetrics;\
+    CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
+    go build -a -installsuffix cgo -o /go/bin/app
+
+FROM alpine:latest
+RUN apk add --update --no-cache ca-certificates
+COPY --from=build-env /go/bin/app /go/bin/app
+ADD ./configs/default.json /models/default.json
+ADD ./configs/azure.json /models/azure.json
+ADD ./configs/aws.json /models/aws.json
+ADD ./configs/gcp.json /models/gcp.json
+
+USER 1001
+ENTRYPOINT ["/go/bin/app"]

README.md

@@ -72,9 +72,9 @@ As an example, let's imagine a node with 1 CPU and 1 Gb of RAM that costs $20/mo
 
 Resources are allocated based on the time-weighted maximum of resource Requests and Usage over the measured period. For example, a pod with no usage and 1 CPU requested for 12 hours out of a 24 hour window would be allocated 12 CPU hours. For pods with BestEffort quality of service (i.e. no requests) allocation is done solely on resource usage. 
 
-#### How do I set my AWS Spot bids for accurate allocation?
+#### How do I set my AWS Spot estimates for cost allocation?
 
-Modify [spotCPU](https://github.com/kubecost/cost-model/blob/master/configs/default.json#L5) and  [spotRAM](https://github.com/kubecost/cost-model/blob/master/configs/default.json#L7) in default.json to the price of your bid. Allocation will use these bid prices, but it does not take into account what you are actually charged by AWS. Alternatively, you can provide an AWS key to allow access to the Spot data feed. This will provide accurate Spot prices. 
+Modify [spotCPU](https://github.com/kubecost/cost-model/blob/master/configs/default.json#L5) and  [spotRAM](https://github.com/kubecost/cost-model/blob/master/configs/default.json#L7) in default.json to the level of recent market prices. Allocation will use these prices, but it does not take into account what you are actually charged by AWS. Alternatively, you can provide an AWS key to allow access to the Spot data feed. This will provide accurate Spot price reconciliation. 
 
 #### Do I need a GCP billing API key?
 

cmd/kubemetrics/main.go

@@ -0,0 +1,184 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/kubecost/cost-model/pkg/cloud"
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/costmodel"
+	"github.com/kubecost/cost-model/pkg/costmodel/clusters"
+	"github.com/kubecost/cost-model/pkg/env"
+	"github.com/kubecost/cost-model/pkg/prom"
+	"github.com/kubecost/cost-model/pkg/util/watcher"
+
+	prometheus "github.com/prometheus/client_golang/api"
+	prometheusAPI "github.com/prometheus/client_golang/api/prometheus/v1"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/rs/cors"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog"
+)
+
+func Healthz(w http.ResponseWriter, _ *http.Request) {
+	w.WriteHeader(200)
+	w.Header().Set("Content-Length", "0")
+	w.Header().Set("Content-Type", "text/plain")
+}
+
+// initializes the kubernetes client cache
+func newKubernetesClusterCache() (clustercache.ClusterCache, error) {
+	var err error
+
+	// Kubernetes API setup
+	var kc *rest.Config
+	if kubeconfig := env.GetKubeConfigPath(); kubeconfig != "" {
+		kc, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
+	} else {
+		kc, err = rest.InClusterConfig()
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	kubeClientset, err := kubernetes.NewForConfig(kc)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create Kubernetes Cluster Cache + Watchers
+	k8sCache := clustercache.NewKubernetesClusterCache(kubeClientset)
+	k8sCache.Run()
+
+	return k8sCache, nil
+}
+
+func newPrometheusClient() (prometheus.Client, error) {
+	address := env.GetPrometheusServerEndpoint()
+	if address == "" {
+		return nil, fmt.Errorf("No address for prometheus set in $%s. Aborting.", env.PrometheusServerEndpointEnvVar)
+	}
+
+	queryConcurrency := env.GetMaxQueryConcurrency()
+	klog.Infof("Prometheus Client Max Concurrency set to %d", queryConcurrency)
+
+	timeout := 120 * time.Second
+	keepAlive := 120 * time.Second
+
+	promCli, err := prom.NewPrometheusClient(address, timeout, keepAlive, queryConcurrency, "")
+	if err != nil {
+		return nil, fmt.Errorf("Failed to create prometheus client, Error: %v", err)
+	}
+
+	m, err := prom.Validate(promCli)
+	if err != nil || !m.Running {
+		if err != nil {
+			klog.Errorf("Failed to query prometheus at %s. Error: %s . Troubleshooting help available at: %s", address, err.Error(), prom.PrometheusTroubleshootingURL)
+		} else if !m.Running {
+			klog.Errorf("Prometheus at %s is not running. Troubleshooting help available at: %s", address, prom.PrometheusTroubleshootingURL)
+		}
+	} else {
+		klog.V(1).Info("Success: retrieved the 'up' query against prometheus at: " + address)
+	}
+
+	api := prometheusAPI.NewAPI(promCli)
+	_, err = api.Config(context.Background())
+	if err != nil {
+		klog.Infof("No valid prometheus config file at %s. Error: %s . Troubleshooting help available at: %s. Ignore if using cortex/thanos here.", address, err.Error(), prom.PrometheusTroubleshootingURL)
+	} else {
+		klog.Infof("Retrieved a prometheus config file from: %s", address)
+	}
+
+	return promCli, nil
+}
+
+func main() {
+	klog.InitFlags(nil)
+	flag.Set("v", "3")
+	flag.Parse()
+
+	klog.V(1).Infof("Starting kubecost-metrics...")
+
+	configWatchers := watcher.NewConfigMapWatchers()
+
+	scrapeInterval := time.Minute
+	promCli, err := newPrometheusClient()
+	if err != nil {
+		panic(err.Error())
+	}
+
+	// Lookup scrape interval for kubecost job, update if found
+	si, err := prom.ScrapeIntervalFor(promCli, env.GetKubecostJobName())
+	if err == nil {
+		scrapeInterval = si
+	}
+
+	klog.Infof("Using scrape interval of %f", scrapeInterval.Seconds())
+
+	// initialize kubernetes client and cluster cache
+	clusterCache, err := newKubernetesClusterCache()
+	if err != nil {
+		panic(err.Error())
+	}
+
+	cloudProviderKey := env.GetCloudProviderAPIKey()
+	cloudProvider, err := cloud.NewProvider(clusterCache, cloudProviderKey)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	// Append the pricing config watcher
+	configWatchers.AddWatcher(cloud.ConfigWatcherFor(cloudProvider))
+	watchConfigFunc := configWatchers.ToWatchFunc()
+	watchedConfigs := configWatchers.GetWatchedConfigs()
+
+	k8sClient := clusterCache.GetClient()
+	kubecostNamespace := env.GetKubecostNamespace()
+
+	// We need an initial invocation because the init of the cache has happened before we had access to the provider.
+	for _, cw := range watchedConfigs {
+		configs, err := k8sClient.CoreV1().ConfigMaps(kubecostNamespace).Get(context.Background(), cw, metav1.GetOptions{})
+		if err != nil {
+			klog.Infof("No %s configmap found at install time, using existing configs: %s", cw, err.Error())
+		} else {
+			watchConfigFunc(configs)
+		}
+	}
+
+	clusterCache.SetConfigMapUpdateFunc(watchConfigFunc)
+
+	// Initialize ClusterMap for maintaining ClusterInfo by ClusterID
+	clusterMap := clusters.NewClusterMap(
+		promCli,
+		costmodel.NewLocalClusterInfoProvider(k8sClient, cloudProvider),
+		5*time.Minute)
+
+	costModel := costmodel.NewCostModel(promCli, cloudProvider, clusterCache, clusterMap, scrapeInterval)
+
+	// initialize Kubernetes Metrics Emitter
+	metricsEmitter := costmodel.NewCostModelMetricsEmitter(promCli, clusterCache, cloudProvider, costModel)
+
+	// download pricing data
+	err = cloudProvider.DownloadPricingData()
+	if err != nil {
+		klog.Errorf("Error downloading pricing data: %s", err)
+	}
+
+	// start emitting metrics
+	metricsEmitter.Start()
+
+	rootMux := http.NewServeMux()
+	rootMux.HandleFunc("/healthz", Healthz)
+	rootMux.Handle("/metrics", promhttp.Handler())
+	handler := cors.AllowAll().Handler(rootMux)
+
+	klog.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", env.GetKubecostMetricsPort()), handler))
+}

configs/azure.json

@@ -9,6 +9,8 @@
     "zoneNetworkEgress": "0.01",
     "regionNetworkEgress": "0.01",
     "internetNetworkEgress": "0.0725",
+    "spotLabel": "kubernetes.azure.com/scalesetpriority",
+    "spotLabelValue": "spot",
     "azureSubscriptionID": "",
     "azureClientID": "" ,
     "azureClientSecret": "" ,

go.mod

@@ -12,34 +12,31 @@ require (
 	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
 	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
 	github.com/aws/aws-sdk-go v1.28.9
+	github.com/aws/aws-sdk-go-v2 v1.9.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/getsentry/sentry-go v0.6.1
-	github.com/google/martian v2.1.0+incompatible // indirect
 	github.com/google/uuid v1.1.2
-	github.com/googleapis/gax-go v2.0.2+incompatible // indirect
-	github.com/gophercloud/gophercloud v0.2.0 // indirect
 	github.com/json-iterator/go v1.1.10
 	github.com/jszwec/csvutil v1.2.1
-	github.com/julienschmidt/httprouter v1.2.0
+	github.com/julienschmidt/httprouter v1.3.0
 	github.com/lib/pq v1.2.0
-	github.com/microcosm-cc/bluemonday v1.0.2
+	github.com/microcosm-cc/bluemonday v1.0.5
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/prometheus/client_golang v1.0.0
 	github.com/prometheus/client_model v0.2.0
-	github.com/rs/cors v1.7.0 // indirect
+	github.com/rs/cors v1.7.0
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 // indirect
 	go.etcd.io/bbolt v1.3.5
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	google.golang.org/api v0.20.0
-	gopkg.in/yaml.v2 v2.2.8
+	gopkg.in/yaml.v2 v2.3.0
 	k8s.io/api v0.20.4
 	k8s.io/apimachinery v0.20.4
 	k8s.io/client-go v0.20.4
 	k8s.io/klog v0.4.0
-	sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e // indirect
 	sigs.k8s.io/yaml v1.2.0
 )
 
-go 1.13
+go 1.16

go.sum

@@ -1,5 +1,4 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
@@ -16,41 +15,36 @@ cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNF
 cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U=
 github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k=
-github.com/Azure/azure-sdk-for-go v24.1.0+incompatible h1:P7GocB7bhkyGbRL1tCy0m9FDqb1V/dqssch3jZieUHk=
-github.com/Azure/azure-sdk-for-go v24.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-sdk-for-go v51.1.0+incompatible h1:7uk6GWtUqKg6weLv2dbKnzwb0ml1Qn70AdtRccZ543w=
 github.com/Azure/azure-sdk-for-go v51.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-storage-blob-go v0.13.0 h1:lgWHvFh+UYBNVQLFHXkvul2f6yOPA9PIH82RTG2cSwc=
 github.com/Azure/azure-storage-blob-go v0.13.0/go.mod h1:pA9kNqtjUeQF2zOSu4s//nUdBD+e64lEuc4sVnuOfNs=
-github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
 github.com/Azure/go-autorest/autorest v0.11.17 h1:2zCdHwNgRH+St1J+ZMf66xI8aLr/5KMy+wWLH97zwYM=
 github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
 github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
-github.com/Azure/go-autorest/autorest/adal v0.9.2 h1:Aze/GQeAN1RRbGmnUJvUj+tFGBzFdIg3293/A9rbxC4=
 github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE=
 github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
 github.com/Azure/go-autorest/autorest/adal v0.9.10 h1:r6fZHMaHD8B6LDCn0o5vyBFHIHrM6Ywwx7mb49lPItI=
 github.com/Azure/go-autorest/autorest/adal v0.9.10/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
-github.com/Azure/go-autorest/autorest/adal v0.9.11 h1:L4/pmq7poLdsy41Bj1FayKvBhayuWRYkx9HU5i4Ybl0=
-github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk=
 github.com/Azure/go-autorest/autorest/azure/auth v0.5.6 h1:cgiBtUxatlt/e3qY6fQJioqbocWHr5osz259MomF5M0=
 github.com/Azure/go-autorest/autorest/azure/auth v0.5.6/go.mod h1:nYlP+G+n8MhD5CjIi6W8nFTIJn/PnTHes5nUbK6BxD0=
-github.com/Azure/go-autorest/autorest/azure/auth v0.5.7 h1:8DQB8yl7aLQuP+nuR5e2RO6454OvFlSTXXaNHshc16s=
-github.com/Azure/go-autorest/autorest/azure/auth v0.5.7/go.mod h1:AkzUsqkrdmNhfP2i54HqINVQopw0CLDnvHpJ88Zz1eI=
 github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY=
 github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM=
 github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
@@ -66,6 +60,7 @@ github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8
 github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw=
@@ -73,9 +68,7 @@ github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mo
 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
 github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
-github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
@@ -85,12 +78,19 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.28.9 h1:grIuBQc+p3dTRXerh5+2OxSuWFi0iXuxbFdTSg0jaW0=
 github.com/aws/aws-sdk-go v1.28.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go-v2 v1.9.0 h1:+S+dSqQCN3MSU5vJRu1HqHrq00cJn6heIMU7X9hcsoo=
+github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
+github.com/aws/smithy-go v1.8.0 h1:AEwwwXQZtUwP5Mz506FeXXrKBe0jA8gVM+1gEcSRooc=
+github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
+github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
+github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/chris-ramon/douceur v0.2.0 h1:IDMEdxlEUUBYBKE4z/mJnFyVXox+MjuEVDJNN27glkU=
+github.com/chris-ramon/douceur v0.2.0/go.mod h1:wDW5xjJdeoMm1mRt4sD4c/LbF/mWdEpRXQKjTR8nIBE=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -100,17 +100,12 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
-github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
-github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda h1:NyywMz59neOoVRFDz+ccfKWxn784fiHMDnZSy6T+JXY=
-github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
-github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4=
 github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
@@ -118,22 +113,17 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
-github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
-github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550 h1:mV9jbLoSW/8m4VK16ZkHTozJa8sesK5u5kTMFysTYac=
-github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
@@ -154,31 +144,22 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7
 github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
-github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
-github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
-github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
 github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
-github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
-github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
@@ -188,10 +169,7 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
@@ -205,24 +183,18 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
-github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
-github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
-github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck=
-github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
-github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -234,58 +206,44 @@ github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww=
-github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
-github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
-github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8 h1:L9JPKrtsHMQ4VCRQfHvbbHBfB2Urn8xf6QZeXZ+OrN4=
-github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4=
-github.com/gophercloud/gophercloud v0.2.0 h1:lD2Bce2xBAMNNcFZ0dObTpXkGLlVIb33RPVUNVpw6ic=
-github.com/gophercloud/gophercloud v0.2.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
+github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=
 github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0=
 github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI=
 github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
-github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
-github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be h1:AHimNtVIpiBjPUhEF5KNCkrUyqTSA5zWUl8sQ2bfGBE=
-github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
-github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jszwec/csvutil v1.2.1 h1:9+vmGqMdYxIbeDmVbTrVryibx2izwHAfKdPwl4GPNHM=
 github.com/jszwec/csvutil v1.2.1/go.mod h1:8YHz6C3KVdIeCxLMvwbbIVDCTA/Wi2df93AZlQNaE2U=
@@ -293,8 +251,9 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
 github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
 github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
 github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
-github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
 github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk=
 github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U=
@@ -307,7 +266,6 @@ github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
 github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -319,7 +277,6 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL
 github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
@@ -333,16 +290,15 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg=
 github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ=
-github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
 github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
+github.com/microcosm-cc/bluemonday v1.0.5 h1:cF59UCKMmmUgqN1baLvqU/B1ZsMori+duLVTLpgiG3w=
+github.com/microcosm-cc/bluemonday v1.0.5/go.mod h1:8iwZnFn2CDDNZ0r6UXhF4xawGvzaqzCRa1n3/lO3W2w=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -357,37 +313,28 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
-github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -410,19 +357,12 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
-github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4=
-github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@@ -430,11 +370,8 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
-github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -457,25 +394,19 @@ github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDf
 github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
 go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
-go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
@@ -501,6 +432,7 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
@@ -508,17 +440,15 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -528,11 +458,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc h1:gkKoSkUmnU6bpS/VhkuO27bzQeSA51uaEfbOW5dNb68=
-golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191112182307-2180aed22343 h1:00ohfJ4K98s3m6BGUoBd8nyfp4Yl0GoIKvw5abItTjI=
 golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -545,9 +471,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2l
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
-golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
@@ -555,22 +478,16 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -578,12 +495,10 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -593,47 +508,36 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200828194041-157a740278f4 h1:kCCpuwSAoYJPkNc6x0xT9yTtV4oKtARo4RGBQWOfg9E=
 golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY=
 golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
-golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU=
-golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.0.0-20161028155119-f51c12702a4d h1:TnM+PKb3ylGmZvyPXmo9m/wktg7Jn/a/fNmr33HSj8g=
-golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138 h1:H3uGjxCR/6Ds0Mjgyp7LMK81+LvmbvWWEnJhzk1Pi9E=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -661,14 +565,13 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb h1:iKlO7ROJc6SttHKlxzwGytRtBUqX4VARrNTgP2YLX5M=
 golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0 h1:KKgc1aqhV8wDPbDzlDtpvyjZFY3vjz85FP7p4wcQUyI=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -682,12 +585,11 @@ google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40=
 google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19 h1:Lj2SnHtxkRGJDqnGaSjo+CCdIieEnwVazbOXILwQemk=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -707,9 +609,7 @@ google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfG
 google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
@@ -729,7 +629,6 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
@@ -738,19 +637,16 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
 gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
-gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o=
-gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -758,48 +654,28 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.0.0-20190620084959-7cf5895f2711 h1:BblVYz/wE5WtBsD/Gvu54KyBUTJMflolzc5I2DTvh50=
-k8s.io/api v0.0.0-20190620084959-7cf5895f2711/go.mod h1:TBhBqb1AWbBQbW3XRusr7n7E4v2+5ZY8r8sAMnyFC5A=
-k8s.io/api v0.0.0-20190913080256-21721929cffa h1:5HxstS7zbT60CcA8qiFOeJtUxIwenu0dVIR5Ne0BUI8=
-k8s.io/api v0.0.0-20190913080256-21721929cffa/go.mod h1:jESdJL4e7Q+sDnEXOZ1ysc1WBxR4I34RbRh5QqGT9kQ=
 k8s.io/api v0.20.4 h1:xZjKidCirayzX6tHONRQyTNDVIR55TYVqgATqo6ZULY=
 k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ=
-k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719 h1:uV4S5IB5g4Nvi+TBVNf3e9L4wrirlwYJ6w88jUQxTUw=
-k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719/go.mod h1:I4A+glKBHiTgiEjQiCCQfCAIcIMFGt291SmsvcrFzJA=
-k8s.io/apimachinery v0.0.0-20190913075812-e119e5e154b6 h1:tGU1C/vMoUV2ZakSH6wQq2shk9KiFtjoH2vDDHlhpA4=
-k8s.io/apimachinery v0.0.0-20190913075812-e119e5e154b6/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4=
 k8s.io/apimachinery v0.20.4 h1:vhxQ0PPUUU2Ns1b9r4/UFp13UPs8cw2iOoTjnY9faa0=
 k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
-k8s.io/client-go v0.0.0-20190620085101-78d2af792bab h1:E8Fecph0qbNsAbijJJQryKu4Oi9QTp5cVpjTE+nqg6g=
-k8s.io/client-go v0.0.0-20190620085101-78d2af792bab/go.mod h1:E95RaSlHr79aHaX0aGSwcPNfygDiPKOVXdmivCIZT0k=
 k8s.io/client-go v0.20.4 h1:85crgh1IotNkLpKYKZHVNI1JT86nr/iDCvq2iWKsql4=
 k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k=
-k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
-k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.4.0 h1:lCJCxf/LIowc2IGS9TPjWDyXY4nOmdGdfcwwDQCOURQ=
 k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ=
 k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI=
-k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
-k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
 k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
-k8s.io/utils v0.0.0-20190221042446-c2654d5206da h1:ElyM7RPonbKnQqOcw7dG2IK5uvQQn3b/WPHqD5mBvP4=
-k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ=
-sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

pkg/cloud/awsprovider.go

@@ -3,6 +3,7 @@ package cloud
 import (
 	"bytes"
 	"compress/gzip"
+	"context"
 	"encoding/csv"
 	"fmt"
 	"io"
@@ -24,6 +25,8 @@ import (
 	"github.com/kubecost/cost-model/pkg/errors"
 	"github.com/kubecost/cost-model/pkg/log"
 	"github.com/kubecost/cost-model/pkg/util"
+	"github.com/kubecost/cost-model/pkg/util/cloudutil"
+	"github.com/kubecost/cost-model/pkg/util/fileutil"
 	"github.com/kubecost/cost-model/pkg/util/json"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -35,6 +38,9 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+
+	awsV2 "github.com/aws/aws-sdk-go-v2/aws"
+
 	"github.com/jszwec/csvutil"
 
 	v1 "k8s.io/api/core/v1"
@@ -47,7 +53,7 @@ const PreemptibleType = "preemptible"
 
 const APIPricingSource = "Public API"
 const SpotPricingSource = "Spot Data Feed"
-const ReservedInstancePricingSource = "Savings Plan, Reservied Instance, and Out-Of-Cluster"
+const ReservedInstancePricingSource = "Savings Plan, Reserved Instance, and Out-Of-Cluster"
 
 func (aws *AWS) PricingSourceStatus() map[string]*PricingSource {
 
@@ -56,7 +62,10 @@ func (aws *AWS) PricingSourceStatus() map[string]*PricingSource {
 	sps := &PricingSource{
 		Name: SpotPricingSource,
 	}
-	sps.Error = aws.SpotPricingStatus
+	sps.Error = ""
+	if aws.SpotPricingError != nil {
+		sps.Error = aws.SpotPricingError.Error()
+	}
 	if sps.Error != "" {
 		sps.Available = false
 	} else if len(aws.SpotPricingByInstanceID) > 0 {
@@ -69,7 +78,10 @@ func (aws *AWS) PricingSourceStatus() map[string]*PricingSource {
 	rps := &PricingSource{
 		Name: ReservedInstancePricingSource,
 	}
-	rps.Error = aws.RIPricingStatus
+	rps.Error = ""
+	if aws.RIPricingError != nil {
+		rps.Error = aws.RIPricingError.Error()
+	}
 	if rps.Error != "" {
 		rps.Available = false
 	} else {
@@ -118,9 +130,9 @@ type AWS struct {
 	SpotPricingUpdatedAt        *time.Time
 	SpotRefreshRunning          bool
 	SpotPricingLock             sync.RWMutex
-	SpotPricingStatus           string
+	SpotPricingError           error
 	RIPricingByInstanceID       map[string]*RIData
-	RIPricingStatus             string
+	RIPricingError             error
 	RIDataRunning               bool
 	RIDataLock                  sync.RWMutex
 	SavingsPlanDataByInstanceID map[string]*SavingsPlanData
@@ -153,6 +165,15 @@ type AWSAccessKey struct {
 	SecretAccessKey string `json:"aws_secret_access_key"`
 }
 
+// Retrieve returns a set of awsV2 credentials using the AWSAccessKey's key and secret.
+// This fullfils the awsV2.CredentialsProvider interface contract.
+func (accessKey AWSAccessKey) Retrieve(ctx context.Context) (awsV2.Credentials, error) {
+	return awsV2.Credentials{
+		AccessKeyID:     accessKey.AccessKeyID,
+		SecretAccessKey: accessKey.SecretAccessKey,
+	}, nil
+}
+
 // AWSPricing maps a k8s node to an AWS Pricing "product"
 type AWSPricing struct {
 	Products map[string]*AWSProduct `json:"products"`
@@ -314,7 +335,7 @@ var regionToBillingRegionCode = map[string]string{
 var loadedAWSSecret bool = false
 var awsSecret *AWSAccessKey = nil
 
-func (aws *AWS) GetLocalStorageQuery(window, offset string, rate bool, used bool) string {
+func (aws *AWS) GetLocalStorageQuery(window, offset time.Duration, rate bool, used bool) string {
 	return ""
 }
 
@@ -366,14 +387,17 @@ func (aws *AWS) GetManagementPlatform() (string, error) {
 
 func (aws *AWS) GetConfig() (*CustomPricing, error) {
 	c, err := aws.Config.GetCustomPricingData()
+	if err != nil {
+		return nil, err
+	}
 	if c.Discount == "" {
 		c.Discount = "0%"
 	}
 	if c.NegotiatedDiscount == "" {
 		c.NegotiatedDiscount = "0%"
 	}
-	if err != nil {
-		return nil, err
+	if c.ShareTenancyCosts == "" {
+		c.ShareTenancyCosts = defaultShareTenancyCost
 	}
 
 	return c, nil
@@ -435,12 +459,7 @@ func (aws *AWS) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, er
 						return err
 					}
 				} else {
-					sci := v.(map[string]interface{})
-					sc := make(map[string]string)
-					for k, val := range sci {
-						sc[k] = val.(string)
-					}
-					c.SharedCosts = sc //todo: support reflection/multiple map fields
+					return fmt.Errorf("type error while updating config for %s", kUpper)
 				}
 			}
 		}
@@ -546,10 +565,10 @@ func (key *awsPVKey) Features() string {
 	// Storage class names are generally EBS volume types (gp2)
 	// Keys in Pricing are based on UsageTypes (EBS:VolumeType.gp2)
 	// Converts between the 2
-	region, _ := util.GetRegion(key.Labels)
-	//if region == "" {
-	//	region = "us-east-1"
-	//}
+	region, ok := util.GetRegion(key.Labels)
+	if !ok {
+		region = key.DefaultRegion
+	}
 	class, ok := volTypes[storageClass]
 	if !ok {
 		klog.V(4).Infof("No voltype mapping for %s's storageClass: %s", key.Name, storageClass)
@@ -923,10 +942,10 @@ func (aws *AWS) refreshSpotPricing(force bool) {
 	sp, err := aws.parseSpotData(aws.SpotDataBucket, aws.SpotDataPrefix, aws.ProjectID, aws.SpotDataRegion)
 	if err != nil {
 		klog.V(1).Infof("Skipping AWS spot data download: %s", err.Error())
-		aws.SpotPricingStatus = err.Error()
+		aws.SpotPricingError = err
 		return
 	}
-	aws.SpotPricingStatus = ""
+	aws.SpotPricingError = nil
 
 	// update time last updated
 	aws.SpotPricingUpdatedAt = &now
@@ -1313,7 +1332,7 @@ func (aws *AWS) loadAWSAuthSecret(force bool) (*AWSAccessKey, error) {
 	}
 	loadedAWSSecret = true
 
-	exists, err := util.FileExists(authSecretPath)
+	exists, err := fileutil.FileExists(authSecretPath)
 	if !exists || err != nil {
 		return nil, fmt.Errorf("Failed to locate service account file: %s", authSecretPath)
 	}
@@ -1537,52 +1556,6 @@ func (a *AWS) GetDisks() ([]byte, error) {
 	})
 }
 
-// ConvertToGlueColumnFormat takes a string and runs through various regex
-// and string replacement statements to convert it to a format compatible
-// with AWS Glue and Athena column names.
-// Following guidance from AWS provided here ('Column Names' section):
-// https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/run-athena-sql.html
-// It returns a string containing the column name in proper column name format and length.
-func ConvertToGlueColumnFormat(column_name string) string {
-	klog.V(5).Infof("Converting string \"%s\" to proper AWS Glue column name.", column_name)
-
-	// An underscore is added in front of uppercase letters
-	capital_underscore := regexp.MustCompile(`[A-Z]`)
-	final := capital_underscore.ReplaceAllString(column_name, `_$0`)
-
-	// Any non-alphanumeric characters are replaced with an underscore
-	no_space_punc := regexp.MustCompile(`[\s]{1,}|[^A-Za-z0-9]`)
-	final = no_space_punc.ReplaceAllString(final, "_")
-
-	// Duplicate underscores are removed
-	no_dup_underscore := regexp.MustCompile(`_{2,}`)
-	final = no_dup_underscore.ReplaceAllString(final, "_")
-
-	// Any leading and trailing underscores are removed
-	no_front_end_underscore := regexp.MustCompile(`(^\_|\_$)`)
-	final = no_front_end_underscore.ReplaceAllString(final, "")
-
-	// Uppercase to lowercase
-	final = strings.ToLower(final)
-
-	// Longer column name than expected - remove _ left to right
-	allowed_col_len := 128
-	undersc_to_remove := len(final) - allowed_col_len
-	if undersc_to_remove > 0 {
-		final = strings.Replace(final, "_", "", undersc_to_remove)
-	}
-
-	// If removing all of the underscores still didn't
-	// make the column name < 128 characters, trim it!
-	if len(final) > allowed_col_len {
-		final = final[:allowed_col_len]
-	}
-
-	klog.V(5).Infof("Column name being returned: \"%s\". Length: \"%d\".", final, len(final))
-
-	return final
-}
-
 func generateAWSGroupBy(lastIdx int) string {
 	sequence := []string{}
 	for i := 1; i < lastIdx+1; i++ {
@@ -1754,14 +1727,14 @@ func (a *AWS) GetSavingsPlanDataFromAthena() error {
 	end := tNow.Format("2006-01-02")
 	// Use Savings Plan Effective Rate as an estimation for cost, assuming the 1h most recent period got a fully loaded savings plan.
 	//
-	q := `SELECT   
+	q := `SELECT
 		line_item_usage_start_date,
 		savings_plan_savings_plan_a_r_n,
 		line_item_resource_id,
-		savings_plan_savings_plan_rate 
+		savings_plan_savings_plan_rate
 	FROM %s as cost_data
 	WHERE line_item_usage_start_date BETWEEN date '%s' AND date '%s'
-	AND line_item_line_item_type = 'SavingsPlanCoveredUsage' ORDER BY 
+	AND line_item_line_item_type = 'SavingsPlanCoveredUsage' ORDER BY
 	line_item_usage_start_date DESC`
 
 	page := 0
@@ -1844,22 +1817,22 @@ func (a *AWS) GetReservationDataFromAthena() error {
 		tOneDayAgo := tNow.Add(time.Duration(-25) * time.Hour) // Also get files from one day ago to avoid boundary conditions
 		start := tOneDayAgo.Format("2006-01-02")
 		end := tNow.Format("2006-01-02")
-		q := `SELECT   
+		q := `SELECT
 		line_item_usage_start_date,
 		reservation_reservation_a_r_n,
 		line_item_resource_id,
 		reservation_effective_cost
 	FROM %s as cost_data
 	WHERE line_item_usage_start_date BETWEEN date '%s' AND date '%s'
-	AND reservation_reservation_a_r_n <> '' ORDER BY 
+	AND reservation_reservation_a_r_n <> '' ORDER BY
 	line_item_usage_start_date DESC`
 		query := fmt.Sprintf(q, cfg.AthenaTable, start, end)
 		op, err := a.QueryAthenaBillingData(query)
 		if err != nil {
-			a.RIPricingStatus = err.Error()
+			a.RIPricingError = err
 			return fmt.Errorf("Error fetching Reserved Instance Data: %s", err)
 		}
-		a.RIPricingStatus = ""
+		a.RIPricingError = nil
 		klog.Infof("Fetching RI data...")
 		if len(op.ResultSet.Rows) > 1 {
 			a.RIDataLock.Lock()
@@ -1893,7 +1866,7 @@ func (a *AWS) GetReservationDataFromAthena() error {
 		}
 	} else {
 		klog.Infof("No reserved data available in Athena")
-		a.RIPricingStatus = ""
+		a.RIPricingError = nil
 	}
 	return nil
 }
@@ -1952,7 +1925,7 @@ func (a *AWS) ExternalAllocations(start string, end string, aggregators []string
 	formattedAggregators := []string{}
 	for _, agg := range aggregators {
 		aggregator_column_name := "resource_tags_user_" + agg
-		aggregator_column_name = ConvertToGlueColumnFormat(aggregator_column_name)
+		aggregator_column_name = cloudutil.ConvertToGlueColumnFormat(aggregator_column_name)
 		formattedAggregators = append(formattedAggregators, aggregator_column_name)
 	}
 	aggregatorNames := strings.Join(formattedAggregators, ",")
@@ -1960,26 +1933,26 @@ func (a *AWS) ExternalAllocations(start string, end string, aggregators []string
 	aggregatorOr = aggregatorOr + " <> ''"
 
 	filter_column_name := "resource_tags_user_" + filterType
-	filter_column_name = ConvertToGlueColumnFormat(filter_column_name)
+	filter_column_name = cloudutil.ConvertToGlueColumnFormat(filter_column_name)
 
 	var query string
 	var lastIdx int
 	if filterType != "kubernetes_" { // This gets appended upstream and is equivalent to no filter.
 		lastIdx = len(formattedAggregators) + 3
 		groupby := generateAWSGroupBy(lastIdx)
-		query = fmt.Sprintf(`SELECT   
+		query = fmt.Sprintf(`SELECT
 			CAST(line_item_usage_start_date AS DATE) as start_date,
 			%s,
 			line_item_product_code,
 			%s,
 			SUM(line_item_blended_cost) as blended_cost
 		FROM %s as cost_data
-		WHERE (%s='%s') AND line_item_usage_start_date BETWEEN date '%s' AND date '%s' AND (%s) 
+		WHERE (%s='%s') AND line_item_usage_start_date BETWEEN date '%s' AND date '%s' AND (%s)
 		GROUP BY %s`, aggregatorNames, filter_column_name, customPricing.AthenaTable, filter_column_name, filterValue, start, end, aggregatorOr, groupby)
 	} else {
 		lastIdx = len(formattedAggregators) + 2
 		groupby := generateAWSGroupBy(lastIdx)
-		query = fmt.Sprintf(`SELECT   
+		query = fmt.Sprintf(`SELECT
 			CAST(line_item_usage_start_date AS DATE) as start_date,
 			%s,
 			line_item_product_code,
@@ -2360,41 +2333,6 @@ func (aws *AWS) CombinedDiscountForNode(instanceType string, isPreemptible bool,
 	return 1.0 - ((1.0 - defaultDiscount) * (1.0 - negotiatedDiscount))
 }
 
-func (aws *AWS) ParseID(id string) string {
-	// It's of the form aws:///us-east-2a/i-0fea4fd46592d050b and we want i-0fea4fd46592d050b, if it exists
-	rx := regexp.MustCompile("aws://[^/]*/[^/]*/([^/]+)")
-	match := rx.FindStringSubmatch(id)
-	if len(match) < 2 {
-		if id != "" {
-			log.Infof("awsprovider.ParseID: failed to parse %s", id)
-		}
-		return id
-	}
-
-	return match[1]
-}
-
-func (aws *AWS) ParsePVID(id string) string {
-	rx := regexp.MustCompile("aws:/[^/]*/[^/]*/([^/]+)") // Capture "vol-0fc54c5e83b8d2b76" from "aws://us-east-2a/vol-0fc54c5e83b8d2b76"
-	match := rx.FindStringSubmatch(id)
-	if len(match) < 2 {
-		if id != "" {
-			log.Infof("awsprovider.ParseID: failed to parse %s", id)
-		}
-		return id
-	}
-
-	return match[1]
-}
-
-func (aws *AWS) ParseLBID(id string) string {
-	rx := regexp.MustCompile("^([^-]+)-.+$") // Capture "ad9d88195b52a47c89b5055120f28c58" from "ad9d88195b52a47c89b5055120f28c58-1037804914.us-east-2.elb.amazonaws.com"
-	match := rx.FindStringSubmatch(id)
-	if len(match) < 2 {
-		if id != "" {
-			log.Infof("awsprovider.ParseLBID: failed to parse %s, %v", id, match)
-		}
-		return id
-	}
-	return match[1]
+func (aws *AWS) Regions() []string {
+	return awsRegions
 }

pkg/cloud/azureprovider.go

@@ -4,18 +4,23 @@ import (
 	"context"
 	"encoding/csv"
 	"fmt"
-	"github.com/kubecost/cost-model/pkg/kubecost"
 	"io"
 	"io/ioutil"
+	"net/http"
+	"net/url"
 	"regexp"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/kubecost/cost-model/pkg/log"
+
 	"github.com/kubecost/cost-model/pkg/clustercache"
 	"github.com/kubecost/cost-model/pkg/env"
+	"github.com/kubecost/cost-model/pkg/kubecost"
 	"github.com/kubecost/cost-model/pkg/util"
+	"github.com/kubecost/cost-model/pkg/util/fileutil"
 	"github.com/kubecost/cost-model/pkg/util/json"
 
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-09-01/skus"
@@ -36,6 +41,8 @@ const (
 	AzureDiskPremiumSSDStorageClass  = "premium_ssd"
 	AzureDiskStandardSSDStorageClass = "standard_ssd"
 	AzureDiskStandardStorageClass    = "standard_hdd"
+	defaultSpotLabel                 = "kubernetes.azure.com/scalesetpriority"
+	defaultSpotLabelValue            = "spot"
 )
 
 var (
@@ -66,6 +73,86 @@ var (
 	mtStandardN, _ = regexp.Compile(`^Standard_N[C|D|V]\d+r?[_v\d]*[_Promo]*$`)
 )
 
+// List obtained by installing the Azure CLI tool "az", described here:
+// https://docs.microsoft.com/en-us/cli/azure/install-azure-cli-linux?pivots=apt
+// logging into an Azure account, and running command `az account list-locations`
+var azureRegions = []string{
+	"eastus",
+	"eastus2",
+	"southcentralus",
+	"westus2",
+	"westus3",
+	"australiaeast",
+	"southeastasia",
+	"northeurope",
+	"swedencentral",
+	"uksouth",
+	"westeurope",
+	"centralus",
+	"northcentralus",
+	"westus",
+	"southafricanorth",
+	"centralindia",
+	"eastasia",
+	"japaneast",
+	"jioindiawest",
+	"koreacentral",
+	"canadacentral",
+	"francecentral",
+	"germanywestcentral",
+	"norwayeast",
+	"switzerlandnorth",
+	"uaenorth",
+	"brazilsouth",
+	"centralusstage",
+	"eastusstage",
+	"eastus2stage",
+	"northcentralusstage",
+	"southcentralusstage",
+	"westusstage",
+	"westus2stage",
+	"asia",
+	"asiapacific",
+	"australia",
+	"brazil",
+	"canada",
+	"europe",
+	"france",
+	"germany",
+	"global",
+	"india",
+	"japan",
+	"korea",
+	"norway",
+	"southafrica",
+	"switzerland",
+	"uae",
+	"uk",
+	"unitedstates",
+	"eastasiastage",
+	"southeastasiastage",
+	"centraluseuap",
+	"eastus2euap",
+	"westcentralus",
+	"southafricawest",
+	"australiacentral",
+	"australiacentral2",
+	"australiasoutheast",
+	"japanwest",
+	"jioindiacentral",
+	"koreasouth",
+	"southindia",
+	"westindia",
+	"canadaeast",
+	"francesouth",
+	"germanynorth",
+	"norwaywest",
+	"switzerlandwest",
+	"ukwest",
+	"uaecentral",
+	"brazilsoutheast",
+}
+
 const AzureLayout = "2006-01-02"
 
 var HeaderStrings = []string{"MeterCategory", "UsageDateTime", "InstanceId", "AdditionalInfo", "Tags", "PreTaxCost", "SubscriptionGuid", "ConsumedService", "ResourceGroup", "ResourceType"}
@@ -149,6 +236,72 @@ func getRegions(service string, subscriptionsClient subscriptions.Client, provid
 	}
 }
 
+func getRetailPrice(region string, skuName string, currencyCode string, spot bool) (string, error) {
+	pricingURL := "https://prices.azure.com/api/retail/prices?$skip=0"
+
+	if currencyCode != "" {
+		pricingURL += fmt.Sprintf("&currencyCode='%s'", currencyCode)
+	}
+
+	var filterParams []string
+
+	if region != "" {
+		regionParam := fmt.Sprintf("armRegionName eq '%s'", region)
+		filterParams = append(filterParams, regionParam)
+	}
+
+	if skuName != "" {
+		skuNameParam := fmt.Sprintf("armSkuName eq '%s'", skuName)
+		filterParams = append(filterParams, skuNameParam)
+	}
+
+	if len(filterParams) > 0 {
+		filterParamsEscaped := url.QueryEscape(strings.Join(filterParams[:], " and "))
+		pricingURL += fmt.Sprintf("&$filter=%s", filterParamsEscaped)
+	}
+
+	log.Infof("starting download retail price payload from \"%s\"", pricingURL)
+	resp, err := http.Get(pricingURL)
+
+	if err != nil {
+		return "", fmt.Errorf("bogus fetch of \"%s\": %v", pricingURL, err)
+	}
+
+	if resp.StatusCode < 200 && resp.StatusCode > 299 {
+		return "", fmt.Errorf("retail price responded with error status code %d", resp.StatusCode)
+	}
+
+	pricingPayload := AzureRetailPricing{}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("Error getting response: %v", err)
+	}
+
+	jsonErr := json.Unmarshal(body, &pricingPayload)
+	if jsonErr != nil {
+		return "", fmt.Errorf("Error unmarshalling data: %v", jsonErr)
+	}
+
+	retailPrice := ""
+	for _, item := range pricingPayload.Items {
+		if item.Type == "Consumption" && !strings.Contains(item.ProductName, "Windows") {
+			// if spot is true SkuName should contain "spot, if it is false it should not
+			if spot == strings.Contains(strings.ToLower(item.SkuName), " spot") {
+				retailPrice = fmt.Sprintf("%f", item.RetailPrice)
+			}
+		}
+	}
+
+	log.DedupedInfof(5, "done parsing retail price payload from \"%s\"\n", pricingURL)
+
+	if retailPrice == "" {
+		return retailPrice, fmt.Errorf("Couldn't find price for product \"%s\" in \"%s\" region", skuName, region)
+	}
+
+	return retailPrice, nil
+}
+
 func toRegionID(meterRegion string, regions map[string]string) (string, error) {
 	var rp regionParts = strings.Split(strings.ToLower(meterRegion), " ")
 	regionCode := regionCodeMappings[rp[0]]
@@ -182,6 +335,41 @@ func checkRegionID(regionID string, regions map[string]string) bool {
 	return false
 }
 
+// AzureRetailPricing struct for unmarshalling Azure Retail pricing api JSON response
+type AzureRetailPricing struct {
+	BillingCurrency    string                         `json:"BillingCurrency"`
+	CustomerEntityId   string                         `json:"CustomerEntityId"`
+	CustomerEntityType string                         `json:"CustomerEntityType"`
+	Items              []AzureRetailPricingAttributes `json:"Items"`
+	NextPageLink       string                         `json:"NextPageLink"`
+	Count              int                            `json:"Count"`
+}
+
+//AzureRetailPricingAttributes struct for unmarshalling Azure Retail pricing api JSON response
+type AzureRetailPricingAttributes struct {
+	CurrencyCode         string     `json:"currencyCode"`
+	TierMinimumUnits     float32    `json:"tierMinimumUnits"`
+	RetailPrice          float32    `json:"retailPrice"`
+	UnitPrice            float32    `json:"unitPrice"`
+	ArmRegionName        string     `json:"armRegionName"`
+	Location             string     `json:"location"`
+	EffectiveStartDate   *time.Time `json:"effectiveStartDate"`
+	EffectiveEndDate     *time.Time `json:"effectiveEndDate"`
+	MeterId              string     `json:"meterId"`
+	MeterName            string     `json:"meterName"`
+	ProductId            string     `json:"productId"`
+	SkuId                string     `json:"skuId"`
+	ProductName          string     `json:"productName"`
+	SkuName              string     `json:"skuName"`
+	ServiceName          string     `json:"serviceName"`
+	ServiceId            string     `json:"serviceId"`
+	ServiceFamily        string     `json:"serviceFamily"`
+	UnitOfMeasure        string     `json:"unitOfMeasure"`
+	Type                 string     `json:"type"`
+	IsPrimaryMeterRegion bool       `json:"isPrimaryMeterRegion"`
+	ArmSkuName           string     `json:"armSkuName"`
+}
+
 // AzurePricing either contains a Node or PV
 type AzurePricing struct {
 	Node *Node
@@ -194,6 +382,7 @@ type Azure struct {
 	Clientset               clustercache.ClusterCache
 	Config                  *ProviderConfig
 	ServiceAccountChecks    map[string]*ServiceAccountCheck
+	RateCardPricingError    error
 }
 
 type azureKey struct {
@@ -210,6 +399,7 @@ func (k *azureKey) Features() string {
 	return fmt.Sprintf("%s,%s,%s", region, instance, usageType)
 }
 
+// GPUType returns value of GPULabel if present
 func (k *azureKey) GPUType() string {
 	if t, ok := k.Labels[k.GPULabel]; ok {
 		return t
@@ -217,6 +407,10 @@ func (k *azureKey) GPUType() string {
 	return ""
 }
 
+func (k *azureKey) isValidGPUNode() bool {
+	return k.GPUType() == k.GPULabelValue && k.GetGPUCount() != "0"
+}
+
 func (k *azureKey) ID() string {
 	return ""
 }
@@ -260,9 +454,10 @@ func (k *azureKey) GetGPUCount() string {
 
 // Represents an azure storage config
 type AzureStorageConfig struct {
-	AccountName   string `json:"azureStorageAccount"`
-	AccessKey     string `json:"azureStorageAccessKey"`
-	ContainerName string `json:"azureStorageContainer"`
+	SubscriptionId string `json:"azureSubscriptionID"`
+	AccountName    string `json:"azureStorageAccount"`
+	AccessKey      string `json:"azureStorageAccessKey"`
+	ContainerName  string `json:"azureStorageContainer"`
 }
 
 // Represents an azure app key
@@ -376,7 +571,7 @@ func (az *Azure) loadAzureAuthSecret(force bool) (*AzureServiceKey, error) {
 	}
 	loadedAzureSecret = true
 
-	exists, err := util.FileExists(authSecretPath)
+	exists, err := fileutil.FileExists(authSecretPath)
 	if !exists || err != nil {
 		return nil, fmt.Errorf("Failed to locate service account file: %s", authSecretPath)
 	}
@@ -405,7 +600,7 @@ func (az *Azure) loadAzureStorageConfig(force bool) (*AzureStorageConfig, error)
 	}
 	loadedAzureStorageConfigSecret = true
 
-	exists, err := util.FileExists(storageConfigSecretPath)
+	exists, err := fileutil.FileExists(storageConfigSecretPath)
 	if !exists || err != nil {
 		return nil, fmt.Errorf("Failed to locate azure storage config file: %s", storageConfigSecretPath)
 	}
@@ -537,6 +732,7 @@ func (az *Azure) DownloadPricingData() error {
 
 	config, err := az.GetConfig()
 	if err != nil {
+		az.RateCardPricingError = err
 		return err
 	}
 
@@ -553,6 +749,7 @@ func (az *Azure) DownloadPricingData() error {
 		credentialsConfig := auth.NewClientCredentialsConfig(config.AzureClientID, config.AzureClientSecret, config.AzureTenantID)
 		a, err := credentialsConfig.Authorizer()
 		if err != nil {
+			az.RateCardPricingError = err
 			return err
 		}
 		authorizer = a
@@ -564,6 +761,7 @@ func (az *Azure) DownloadPricingData() error {
 		if err != nil { // Failed to create authorizer from environment, try from file
 			a, err := auth.NewAuthorizerFromFile(azure.PublicCloud.ResourceManagerEndpoint)
 			if err != nil {
+				az.RateCardPricingError = err
 				return err
 			}
 			authorizer = a
@@ -590,19 +788,17 @@ func (az *Azure) DownloadPricingData() error {
 	klog.Infof("Using ratecard query %s", rateCardFilter)
 	result, err := rcClient.Get(context.TODO(), rateCardFilter)
 	if err != nil {
+		az.RateCardPricingError = err
 		return err
 	}
 	allPrices := make(map[string]*AzurePricing)
 	regions, err := getRegions("compute", sClient, providersClient, config.AzureSubscriptionID)
 	if err != nil {
+		az.RateCardPricingError = err
 		return err
 	}
 
-	c, err := az.GetConfig()
-	if err != nil {
-		return err
-	}
-	baseCPUPrice := c.CPU
+	baseCPUPrice := config.CPU
 
 	for _, v := range *result.Meters {
 		meterName := *v.MeterName
@@ -697,6 +893,7 @@ func (az *Azure) DownloadPricingData() error {
 						Node: &Node{
 							Cost:         priceStr,
 							BaseCPUPrice: baseCPUPrice,
+							UsageType:    usageType,
 						},
 					}
 				}
@@ -720,9 +917,17 @@ func (az *Azure) DownloadPricingData() error {
 	}
 
 	az.Pricing = allPrices
+	az.RateCardPricingError = nil
 	return nil
 }
 
+func (az *Azure) addPricing(features string, azurePricing *AzurePricing) {
+	if az.Pricing == nil {
+		az.Pricing = map[string]*AzurePricing{}
+	}
+	az.Pricing[features] = azurePricing
+}
+
 // AllNodePricing returns the Azure pricing objects stored
 func (az *Azure) AllNodePricing() (interface{}, error) {
 	az.DownloadPricingDataLock.RLock()
@@ -734,24 +939,67 @@ func (az *Azure) AllNodePricing() (interface{}, error) {
 func (az *Azure) NodePricing(key Key) (*Node, error) {
 	az.DownloadPricingDataLock.RLock()
 	defer az.DownloadPricingDataLock.RUnlock()
-	if n, ok := az.Pricing[key.Features()]; ok {
-		klog.V(4).Infof("Returning pricing for node %s: %+v from key %s", key, n, key.Features())
-		if key.GPUType() != "" {
-			n.Node.GPU = key.(*azureKey).GetGPUCount()
+
+	azKey, ok := key.(*azureKey)
+	if !ok {
+		return nil, fmt.Errorf("azure: NodePricing: key is of type %T", key)
+	}
+	config, _ := az.GetConfig()
+	if slv, ok := azKey.Labels[config.SpotLabel]; ok && slv == config.SpotLabelValue && config.SpotLabel != "" && config.SpotLabelValue != "" {
+		features := strings.Split(azKey.Features(), ",")
+		region := features[0]
+		instance := features[1]
+		spotFeatures := fmt.Sprintf("%s,%s,%s", region, instance, "spot")
+		if n, ok := az.Pricing[spotFeatures]; ok {
+			log.DedupedInfof(5, "Returning pricing for node %s: %+v from key %s", azKey, n, spotFeatures)
+			if azKey.isValidGPUNode() {
+				n.Node.GPU = "1" // TODO: support multiple GPUs
+			}
+			return n.Node, nil
+		}
+		log.Infof("[Info] found spot instance, trying to get retail price for %s: %s, ", spotFeatures, azKey)
+
+		spotCost, err := getRetailPrice(region, instance, config.CurrencyCode, true)
+		if err != nil {
+			log.DedupedWarningf(5, "failed to retrieve spot retail pricing")
+		} else {
+			gpu := ""
+			if azKey.isValidGPUNode() {
+				gpu = "1"
+			}
+			spotNode := &Node{
+				Cost:      spotCost,
+				UsageType: "spot",
+				GPU:       gpu,
+			}
+
+			az.addPricing(spotFeatures, &AzurePricing{
+				Node: spotNode,
+			})
+
+			return spotNode, nil
+		}
+	}
+
+	if n, ok := az.Pricing[azKey.Features()]; ok {
+		klog.V(4).Infof("Returning pricing for node %s: %+v from key %s", azKey, n, azKey.Features())
+		if azKey.isValidGPUNode() {
+			n.Node.GPU = azKey.GetGPUCount()
 		}
 		return n.Node, nil
 	}
-	klog.V(1).Infof("[Warning] no pricing data found for %s: %s", key.Features(), key)
+	klog.V(1).Infof("[Warning] no pricing data found for %s: %s", azKey.Features(), azKey)
 	c, err := az.GetConfig()
 	if err != nil {
 		return nil, fmt.Errorf("No default pricing data available")
 	}
-	if key.GPUType() != "" {
+	if azKey.isValidGPUNode() {
 		return &Node{
-			VCPUCost: c.CPU,
-			RAMCost:  c.RAM,
-			GPUCost:  c.GPU,
-			GPU:      key.(*azureKey).GetGPUCount(),
+			VCPUCost:         c.CPU,
+			RAMCost:          c.RAM,
+			UsesBaseCPUPrice: true,
+			GPUCost:          c.GPU,
+			GPU:              azKey.GetGPUCount(),
 		}, nil
 	}
 	return &Node{
@@ -910,12 +1158,7 @@ func (az *Azure) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, e
 					return err
 				}
 			} else {
-				sci := v.(map[string]interface{})
-				sc := make(map[string]string)
-				for k, val := range sci {
-					sc[k] = val.(string)
-				}
-				c.SharedCosts = sc //todo: support reflection/multiple map fields
+				return fmt.Errorf("type error while updating config for %s", kUpper)
 			}
 		}
 
@@ -931,6 +1174,9 @@ func (az *Azure) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, e
 }
 func (az *Azure) GetConfig() (*CustomPricing, error) {
 	c, err := az.Config.GetCustomPricingData()
+	if err != nil {
+		return nil, err
+	}
 	if c.Discount == "" {
 		c.Discount = "0%"
 	}
@@ -943,8 +1189,14 @@ func (az *Azure) GetConfig() (*CustomPricing, error) {
 	if c.AzureBillingRegion == "" {
 		c.AzureBillingRegion = "US"
 	}
-	if err != nil {
-		return nil, err
+	if c.ShareTenancyCosts == "" {
+		c.ShareTenancyCosts = defaultShareTenancyCost
+	}
+	if c.SpotLabel == "" {
+		c.SpotLabel = defaultSpotLabel
+	}
+	if c.SpotLabelValue == "" {
+		c.SpotLabelValue = defaultSpotLabelValue
 	}
 	return c, nil
 }
@@ -1116,7 +1368,7 @@ func (az *Azure) PVPricing(pvk PVKey) (*PV, error) {
 	return pricing.PV, nil
 }
 
-func (az *Azure) GetLocalStorageQuery(window, offset string, rate bool, used bool) string {
+func (az *Azure) GetLocalStorageQuery(window, offset time.Duration, rate bool, used bool) string {
 	return ""
 }
 
@@ -1130,8 +1382,29 @@ func (az *Azure) ServiceAccountStatus() *ServiceAccountStatus {
 	}
 }
 
+const rateCardPricingSource = "Rate Card API"
+
+// PricingSourceStatus returns the status of the rate card api
 func (az *Azure) PricingSourceStatus() map[string]*PricingSource {
-	return make(map[string]*PricingSource)
+	sources := make(map[string]*PricingSource)
+	errMsg := ""
+	if az.RateCardPricingError != nil {
+		errMsg = az.RateCardPricingError.Error()
+	}
+	rcps := &PricingSource {
+		Name: rateCardPricingSource,
+		Error: errMsg,
+	}
+	if rcps.Error != "" {
+		rcps.Available = false
+	} else if len(az.Pricing) == 0 {
+		rcps.Error = "No Pricing Data Available"
+		rcps.Available = false
+	}else {
+		rcps.Available = true
+	}
+	sources[rateCardPricingSource] = rcps
+	return sources
 }
 
 func (*Azure) ClusterManagementPricing() (string, float64, error) {
@@ -1142,14 +1415,6 @@ func (az *Azure) CombinedDiscountForNode(instanceType string, isPreemptible bool
 	return 1.0 - ((1.0 - defaultDiscount) * (1.0 - negotiatedDiscount))
 }
 
-func (az *Azure) ParseID(id string) string {
-	return id
-}
-
-func (az *Azure) ParsePVID(id string) string {
-	return id
-}
-
-func (az *Azure) ParseLBID(id string) string {
-	return id
+func (az *Azure) Regions() []string {
+	return azureRegions
 }

pkg/cloud/csvprovider.go

@@ -367,14 +367,6 @@ func (c *CSVProvider) CombinedDiscountForNode(instanceType string, isPreemptible
 	return 1.0 - ((1.0 - defaultDiscount) * (1.0 - negotiatedDiscount))
 }
 
-func (c *CSVProvider) ParseID(id string) string {
-	return id
-}
-
-func (c *CSVProvider) ParsePVID(id string) string {
-	return id
-}
-
-func (c *CSVProvider) ParseLBID(id string) string {
-	return id
+func (c *CSVProvider) Regions() []string {
+	return []string{}
 }

pkg/cloud/customprovider.go

@@ -1,10 +1,12 @@
 package cloud
 
 import (
+	"fmt"
 	"io"
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/kubecost/cost-model/pkg/clustercache"
 	"github.com/kubecost/cost-model/pkg/env"
@@ -42,7 +44,7 @@ func (*CustomProvider) ClusterManagementPricing() (string, float64, error) {
 	return "", 0.0, nil
 }
 
-func (*CustomProvider) GetLocalStorageQuery(window, offset string, rate bool, used bool) string {
+func (*CustomProvider) GetLocalStorageQuery(window, offset time.Duration, rate bool, used bool) string {
 	return ""
 }
 
@@ -81,12 +83,7 @@ func (cp *CustomProvider) UpdateConfig(r io.Reader, updateType string) (*CustomP
 					return err
 				}
 			} else {
-				sci := v.(map[string]interface{})
-				sc := make(map[string]string)
-				for k, val := range sci {
-					sc[k] = val.(string)
-				}
-				c.SharedCosts = sc //todo: support reflection/multiple map fields
+				return fmt.Errorf("type error while updating config for %s", kUpper)
 			}
 		}
 
@@ -312,14 +309,6 @@ func (cp *CustomProvider) CombinedDiscountForNode(instanceType string, isPreempt
 	return 1.0 - ((1.0 - defaultDiscount) * (1.0 - negotiatedDiscount))
 }
 
-func (cp *CustomProvider) ParseID(id string) string {
-	return id
-}
-
-func (cp *CustomProvider) ParsePVID(id string) string {
-	return id
-}
-
-func (cp *CustomProvider) ParseLBID(id string) string {
-	return id
+func (cp *CustomProvider) Regions() []string {
+	return []string{}
 }

pkg/cloud/gcpprovider.go

@@ -13,27 +13,61 @@ import (
 	"sync"
 	"time"
 
-	"k8s.io/klog"
-
-	"cloud.google.com/go/bigquery"
-	"cloud.google.com/go/compute/metadata"
-
 	"github.com/kubecost/cost-model/pkg/clustercache"
 	"github.com/kubecost/cost-model/pkg/env"
 	"github.com/kubecost/cost-model/pkg/log"
 	"github.com/kubecost/cost-model/pkg/util"
+	"github.com/kubecost/cost-model/pkg/util/fileutil"
 	"github.com/kubecost/cost-model/pkg/util/json"
+	"github.com/kubecost/cost-model/pkg/util/timeutil"
 
+	"cloud.google.com/go/bigquery"
+	"cloud.google.com/go/compute/metadata"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	compute "google.golang.org/api/compute/v1"
 	"google.golang.org/api/iterator"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/klog"
 )
 
 const GKE_GPU_TAG = "cloud.google.com/gke-accelerator"
 const BigqueryUpdateType = "bigqueryupdate"
 
+// List obtained by installing the `gcloud` CLI tool,
+// logging into gcp account, and running command
+// `gcloud compute regions list`
+var gcpRegions = []string{
+	"asia-east1",
+	"asia-east2",
+	"asia-northeast1",
+	"asia-northeast2",
+	"asia-northeast3",
+	"asia-south1",
+	"asia-south2",
+	"asia-southeast1",
+	"asia-southeast2",
+	"australia-southeast1",
+	"australia-southeast2",
+	"europe-central2",
+	"europe-north1",
+	"europe-west1",
+	"europe-west2",
+	"europe-west3",
+	"europe-west4",
+	"europe-west6",
+	"northamerica-northeast1",
+	"northamerica-northeast2",
+	"southamerica-east1",
+	"us-central1",
+	"us-east1",
+	"us-east4",
+	"us-west1",
+	"us-west2",
+	"us-west3",
+	"us-west4",
+}
+
 type userAgentTransport struct {
 	userAgent string
 	base      http.RoundTripper
@@ -55,7 +89,7 @@ type GCP struct {
 	DownloadPricingDataLock sync.RWMutex
 	ReservedInstances       []*GCPReservedInstance
 	Config                  *ProviderConfig
-	serviceKeyProvided      bool
+	ServiceKeyProvided      bool
 	ValidPricingKeys        map[string]bool
 	clusterManagementPrice  float64
 	clusterProvisioner      string
@@ -124,7 +158,7 @@ func gcpAllocationToOutOfClusterAllocation(gcpAlloc gcpAllocation) *OutOfCluster
 
 // GetLocalStorageQuery returns the cost of local storage for the given window. Setting rate=true
 // returns hourly spend. Setting used=true only tracks used storage, not total.
-func (gcp *GCP) GetLocalStorageQuery(window, offset string, rate bool, used bool) string {
+func (gcp *GCP) GetLocalStorageQuery(window, offset time.Duration, rate bool, used bool) string {
 	// TODO Set to the price for the appropriate storage class. It's not trivial to determine the local storage disk type
 	// See https://cloud.google.com/compute/disks-image-pricing#persistentdisk
 	localStorageCost := 0.04
@@ -134,10 +168,7 @@ func (gcp *GCP) GetLocalStorageQuery(window, offset string, rate bool, used bool
 		baseMetric = "container_fs_usage_bytes"
 	}
 
-	fmtOffset := ""
-	if offset != "" {
-		fmtOffset = fmt.Sprintf("offset %s", offset)
-	}
+	fmtOffset := timeutil.DurationToPromOffsetString(offset)
 
 	fmtCumulativeQuery := `sum(
 		sum_over_time(%s{device!="tmpfs", id="/"}[%s:1m]%s)
@@ -151,8 +182,9 @@ func (gcp *GCP) GetLocalStorageQuery(window, offset string, rate bool, used bool
 	if rate {
 		fmtQuery = fmtMonthlyQuery
 	}
+	fmtWindow := timeutil.DurationString(window)
 
-	return fmt.Sprintf(fmtQuery, baseMetric, window, fmtOffset, env.GetPromClusterLabel(), localStorageCost)
+	return fmt.Sprintf(fmtQuery, baseMetric, fmtWindow, fmtOffset, env.GetPromClusterLabel(), localStorageCost)
 }
 
 func (gcp *GCP) GetConfig() (*CustomPricing, error) {
@@ -169,6 +201,9 @@ func (gcp *GCP) GetConfig() (*CustomPricing, error) {
 	if c.CurrencyCode == "" {
 		c.CurrencyCode = "USD"
 	}
+	if c.ShareTenancyCosts == "" {
+		c.ShareTenancyCosts = defaultShareTenancyCost
+	}
 	return c, nil
 }
 
@@ -196,13 +231,13 @@ func (*GCP) loadGCPAuthSecret() {
 	path := env.GetConfigPathWithDefault("/models/")
 
 	keyPath := path + "key.json"
-	keyExists, _ := util.FileExists(keyPath)
+	keyExists, _ := fileutil.FileExists(keyPath)
 	if keyExists {
 		klog.V(1).Infof("GCP Auth Key already exists, no need to load from secret")
 		return
 	}
 
-	exists, err := util.FileExists(authSecretPath)
+	exists, err := fileutil.FileExists(authSecretPath)
 	if !exists || err != nil {
 		errMessage := "Secret does not exist"
 		if err != nil {
@@ -254,7 +289,7 @@ func (gcp *GCP) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, er
 				if err != nil {
 					return err
 				}
-				gcp.serviceKeyProvided = true
+				gcp.ServiceKeyProvided = true
 			}
 		} else if updateType == AthenaInfoUpdateType {
 			a := AwsAthenaInfo{}
@@ -284,12 +319,7 @@ func (gcp *GCP) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, er
 						return err
 					}
 				} else {
-					sci := v.(map[string]interface{})
-					sc := make(map[string]string)
-					for k, val := range sci {
-						sc[k] = val.(string)
-					}
-					c.SharedCosts = sc //todo: support reflection/multiple map fields
+					return fmt.Errorf("type error while updating config for %s", kUpper)
 				}
 			}
 		}
@@ -405,7 +435,7 @@ func (gcp *GCP) ExternalAllocations(start string, end string, aggregators []stri
 		s = append(s, gcpOOC...)
 		qerr = err
 	}
-	if qerr != nil && gcp.serviceKeyProvided {
+	if qerr != nil && gcp.ServiceKeyProvided {
 		klog.Infof("Error querying gcp: %s", qerr)
 	}
 	return s, qerr
@@ -1459,6 +1489,10 @@ func (gcp *GCP) CombinedDiscountForNode(instanceType string, isPreemptible bool,
 	return 1.0 - ((1.0 - sustainedUseDiscount(class, defaultDiscount, isPreemptible)) * (1.0 - negotiatedDiscount))
 }
 
+func (gcp *GCP) Regions() []string {
+	return gcpRegions
+}
+
 func sustainedUseDiscount(class string, defaultDiscount float64, isPreemptible bool) float64 {
 	if isPreemptible {
 		return 0.0
@@ -1472,26 +1506,3 @@ func sustainedUseDiscount(class string, defaultDiscount float64, isPreemptible b
 	}
 	return discount
 }
-
-func (gcp *GCP) ParseID(id string) string {
-	// gce://guestbook-227502/us-central1-a/gke-niko-n1-standard-2-wljla-8df8e58a-hfy7
-	//  => gke-niko-n1-standard-2-wljla-8df8e58a-hfy7
-	rx := regexp.MustCompile("gce://[^/]*/[^/]*/([^/]+)")
-	match := rx.FindStringSubmatch(id)
-	if len(match) < 2 {
-		if id != "" {
-			log.Infof("gcpprovider.ParseID: failed to parse %s", id)
-		}
-		return id
-	}
-
-	return match[1]
-}
-
-func (gcp *GCP) ParsePVID(id string) string {
-	return id
-}
-
-func (gcp *GCP) ParseLBID(id string) string {
-	return id
-}

pkg/cloud/provider.go

@@ -5,7 +5,10 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"regexp"
+	"strconv"
 	"strings"
+	"time"
 
 	"k8s.io/klog"
 
@@ -13,12 +16,15 @@ import (
 
 	"github.com/kubecost/cost-model/pkg/clustercache"
 	"github.com/kubecost/cost-model/pkg/env"
+	"github.com/kubecost/cost-model/pkg/log"
+	"github.com/kubecost/cost-model/pkg/util/watcher"
 
 	v1 "k8s.io/api/core/v1"
 )
 
 const authSecretPath = "/var/secrets/service-key.json"
 const storageConfigSecretPath = "/var/azure-storage-config/azure-storage-config.json"
+const defaultShareTenancyCost = "true"
 
 var createTableStatements = []string{
 	`CREATE TABLE IF NOT EXISTS names (
@@ -127,55 +133,75 @@ type OutOfClusterAllocation struct {
 }
 
 type CustomPricing struct {
-	Provider                     string            `json:"provider"`
-	Description                  string            `json:"description"`
-	CPU                          string            `json:"CPU"`
-	SpotCPU                      string            `json:"spotCPU"`
-	RAM                          string            `json:"RAM"`
-	SpotRAM                      string            `json:"spotRAM"`
-	GPU                          string            `json:"GPU"`
-	SpotGPU                      string            `json:"spotGPU"`
-	Storage                      string            `json:"storage"`
-	ZoneNetworkEgress            string            `json:"zoneNetworkEgress"`
-	RegionNetworkEgress          string            `json:"regionNetworkEgress"`
-	InternetNetworkEgress        string            `json:"internetNetworkEgress"`
-	FirstFiveForwardingRulesCost string            `json:"firstFiveForwardingRulesCost"`
-	AdditionalForwardingRuleCost string            `json:"additionalForwardingRuleCost"`
-	LBIngressDataCost            string            `json:"LBIngressDataCost"`
-	SpotLabel                    string            `json:"spotLabel,omitempty"`
-	SpotLabelValue               string            `json:"spotLabelValue,omitempty"`
-	GpuLabel                     string            `json:"gpuLabel,omitempty"`
-	GpuLabelValue                string            `json:"gpuLabelValue,omitempty"`
-	ServiceKeyName               string            `json:"awsServiceKeyName,omitempty"`
-	ServiceKeySecret             string            `json:"awsServiceKeySecret,omitempty"`
-	SpotDataRegion               string            `json:"awsSpotDataRegion,omitempty"`
-	SpotDataBucket               string            `json:"awsSpotDataBucket,omitempty"`
-	SpotDataPrefix               string            `json:"awsSpotDataPrefix,omitempty"`
-	ProjectID                    string            `json:"projectID,omitempty"`
-	AthenaProjectID              string            `json:"athenaProjectID,omitempty"`
-	AthenaBucketName             string            `json:"athenaBucketName"`
-	AthenaRegion                 string            `json:"athenaRegion"`
-	AthenaDatabase               string            `json:"athenaDatabase"`
-	AthenaTable                  string            `json:"athenaTable"`
-	MasterPayerARN               string            `json:"masterPayerARN"`
-	BillingDataDataset           string            `json:"billingDataDataset,omitempty"`
-	CustomPricesEnabled          string            `json:"customPricesEnabled"`
-	DefaultIdle                  string            `json:"defaultIdle"`
-	AzureSubscriptionID          string            `json:"azureSubscriptionID"`
-	AzureClientID                string            `json:"azureClientID"`
-	AzureClientSecret            string            `json:"azureClientSecret"`
-	AzureTenantID                string            `json:"azureTenantID"`
-	AzureBillingRegion           string            `json:"azureBillingRegion"`
-	CurrencyCode                 string            `json:"currencyCode"`
-	Discount                     string            `json:"discount"`
-	NegotiatedDiscount           string            `json:"negotiatedDiscount"`
-	SharedCosts                  map[string]string `json:"sharedCost"`
-	ClusterName                  string            `json:"clusterName"`
-	SharedNamespaces             string            `json:"sharedNamespaces"`
-	SharedLabelNames             string            `json:"sharedLabelNames"`
-	SharedLabelValues            string            `json:"sharedLabelValues"`
-	ReadOnly                     string            `json:"readOnly"`
-	KubecostToken                string            `json:"kubecostToken"`
+	Provider                     string `json:"provider"`
+	Description                  string `json:"description"`
+	CPU                          string `json:"CPU"`
+	SpotCPU                      string `json:"spotCPU"`
+	RAM                          string `json:"RAM"`
+	SpotRAM                      string `json:"spotRAM"`
+	GPU                          string `json:"GPU"`
+	SpotGPU                      string `json:"spotGPU"`
+	Storage                      string `json:"storage"`
+	ZoneNetworkEgress            string `json:"zoneNetworkEgress"`
+	RegionNetworkEgress          string `json:"regionNetworkEgress"`
+	InternetNetworkEgress        string `json:"internetNetworkEgress"`
+	FirstFiveForwardingRulesCost string `json:"firstFiveForwardingRulesCost"`
+	AdditionalForwardingRuleCost string `json:"additionalForwardingRuleCost"`
+	LBIngressDataCost            string `json:"LBIngressDataCost"`
+	SpotLabel                    string `json:"spotLabel,omitempty"`
+	SpotLabelValue               string `json:"spotLabelValue,omitempty"`
+	GpuLabel                     string `json:"gpuLabel,omitempty"`
+	GpuLabelValue                string `json:"gpuLabelValue,omitempty"`
+	ServiceKeyName               string `json:"awsServiceKeyName,omitempty"`
+	ServiceKeySecret             string `json:"awsServiceKeySecret,omitempty"`
+	SpotDataRegion               string `json:"awsSpotDataRegion,omitempty"`
+	SpotDataBucket               string `json:"awsSpotDataBucket,omitempty"`
+	SpotDataPrefix               string `json:"awsSpotDataPrefix,omitempty"`
+	ProjectID                    string `json:"projectID,omitempty"`
+	AthenaProjectID              string `json:"athenaProjectID,omitempty"`
+	AthenaBucketName             string `json:"athenaBucketName"`
+	AthenaRegion                 string `json:"athenaRegion"`
+	AthenaDatabase               string `json:"athenaDatabase"`
+	AthenaTable                  string `json:"athenaTable"`
+	MasterPayerARN               string `json:"masterPayerARN"`
+	BillingDataDataset           string `json:"billingDataDataset,omitempty"`
+	CustomPricesEnabled          string `json:"customPricesEnabled"`
+	DefaultIdle                  string `json:"defaultIdle"`
+	AzureSubscriptionID          string `json:"azureSubscriptionID"`
+	AzureClientID                string `json:"azureClientID"`
+	AzureClientSecret            string `json:"azureClientSecret"`
+	AzureTenantID                string `json:"azureTenantID"`
+	AzureBillingRegion           string `json:"azureBillingRegion"`
+	CurrencyCode                 string `json:"currencyCode"`
+	Discount                     string `json:"discount"`
+	NegotiatedDiscount           string `json:"negotiatedDiscount"`
+	SharedOverhead               string `json:"sharedOverhead"`
+	ClusterName                  string `json:"clusterName"`
+	SharedNamespaces             string `json:"sharedNamespaces"`
+	SharedLabelNames             string `json:"sharedLabelNames"`
+	SharedLabelValues            string `json:"sharedLabelValues"`
+	ShareTenancyCosts            string `json:"shareTenancyCosts"` // TODO clean up configuration so we can use a type other that string (this should be a bool, but the app panics if it's not a string)
+	ReadOnly                     string `json:"readOnly"`
+	KubecostToken                string `json:"kubecostToken"`
+}
+
+// GetSharedOverheadCostPerMonth parses and returns a float64 representation
+// of the configured monthly shared overhead cost. If the string version cannot
+// be parsed into a float, an error is logged and 0.0 is returned.
+func (cp *CustomPricing) GetSharedOverheadCostPerMonth() float64 {
+	// Empty string should be interpreted as "no cost", i.e. 0.0
+	if cp.SharedOverhead == "" {
+		return 0.0
+	}
+
+	// Attempt to parse, but log and return 0.0 if that fails.
+	sharedCostPerMonth, err := strconv.ParseFloat(cp.SharedOverhead, 64)
+	if err != nil {
+		log.Errorf("SharedOverhead: failed to parse shared overhead \"%s\": %s", cp.SharedOverhead, err)
+		return 0.0
+	}
+
+	return sharedCostPerMonth
 }
 
 type ServiceAccountStatus struct {
@@ -185,7 +211,7 @@ type ServiceAccountStatus struct {
 type ServiceAccountCheck struct {
 	Message        string `json:"message"`
 	Status         bool   `json:"status"`
-	AdditionalInfo string `json:additionalInfo`
+	AdditionalInfo string `json:"additionalInfo"`
 }
 
 type PricingSources struct {
@@ -232,16 +258,14 @@ type Provider interface {
 	UpdateConfigFromConfigMap(map[string]string) (*CustomPricing, error)
 	GetConfig() (*CustomPricing, error)
 	GetManagementPlatform() (string, error)
-	GetLocalStorageQuery(string, string, bool, bool) string
+	GetLocalStorageQuery(time.Duration, time.Duration, bool, bool) string
 	ExternalAllocations(string, string, []string, string, string, bool) ([]*OutOfClusterAllocation, error)
 	ApplyReservedInstancePricing(map[string]*Node)
 	ServiceAccountStatus() *ServiceAccountStatus
 	PricingSourceStatus() map[string]*PricingSource
 	ClusterManagementPricing() (string, float64, error)
 	CombinedDiscountForNode(string, bool, float64, float64) float64
-	ParseID(string) string
-	ParsePVID(string) string
-	ParseLBID(string) string
+	Regions() []string
 }
 
 // ClusterName returns the name defined in cluster info, defaulting to the
@@ -275,6 +299,18 @@ func CustomPricesEnabled(p Provider) bool {
 	return config.CustomPricesEnabled == "true"
 }
 
+// ConfigWatcherFor returns a new ConfigWatcher instance which watches changes to the "pricing-configs"
+// configmap
+func ConfigWatcherFor(p Provider) *watcher.ConfigMapWatcher {
+	return &watcher.ConfigMapWatcher{
+		ConfigMapName: env.GetPricingConfigmapName(),
+		WatchFunc: func(name string, data map[string]string) error {
+			_, err := p.UpdateConfigFromConfigMap(data)
+			return err
+		},
+	}
+}
+
 // AllocateIdleByDefault returns true if the application settings specify to allocate idle by default
 func AllocateIdleByDefault(p Provider) bool {
 	config, err := p.GetConfig()
@@ -335,6 +371,17 @@ func SharedLabels(p Provider) ([]string, []string) {
 	return names, values
 }
 
+// ShareTenancyCosts returns true if the application settings specify to share
+// tenancy costs by default.
+func ShareTenancyCosts(p Provider) bool {
+	config, err := p.GetConfig()
+	if err != nil {
+		return false
+	}
+
+	return config.ShareTenancyCosts == "true"
+}
+
 func NewCrossClusterProvider(ctype string, overrideConfigPath string, cache clustercache.ClusterCache) (Provider, error) {
 	if ctype == "aws" {
 		return &AWS{
@@ -346,6 +393,11 @@ func NewCrossClusterProvider(ctype string, overrideConfigPath string, cache clus
 			Clientset: cache,
 			Config:    NewProviderConfig(overrideConfigPath),
 		}, nil
+	} else if ctype == "azure" {
+		return &Azure{
+			Clientset: cache,
+			Config:    NewProviderConfig(overrideConfigPath),
+		}, nil
 	}
 	return &CustomProvider{
 		Clientset: cache,
@@ -501,3 +553,52 @@ func GetOrCreateClusterMeta(cluster_id, cluster_name string) (string, string, er
 
 	return id, name, nil
 }
+
+// ParseID attempts to parse a ProviderId from a string based on formats from the various providers and
+// returns the string as is if it cannot find a match
+func ParseID(id string) string {
+	// It's of the form aws:///us-east-2a/i-0fea4fd46592d050b and we want i-0fea4fd46592d050b, if it exists
+	rx := regexp.MustCompile("aws://[^/]*/[^/]*/([^/]+)")
+	match := rx.FindStringSubmatch(id)
+	if len(match) >= 2 {
+		return match[1]
+	}
+
+	// gce://guestbook-227502/us-central1-a/gke-niko-n1-standard-2-wljla-8df8e58a-hfy7
+	//  => gke-niko-n1-standard-2-wljla-8df8e58a-hfy7
+	rx = regexp.MustCompile("gce://[^/]*/[^/]*/([^/]+)")
+	match = rx.FindStringSubmatch(id)
+	if len(match) >= 2 {
+		return match[1]
+	}
+
+	// Return id for Azure Provider, CSV Provider and Custom Provider
+	return id
+}
+
+// ParsePVID attempts to parse a PV ProviderId from a string based on formats from the various providers and
+// returns the string as is if it cannot find a match
+func ParsePVID(id string) string {
+	// Capture "vol-0fc54c5e83b8d2b76" from "aws://us-east-2a/vol-0fc54c5e83b8d2b76"
+	rx := regexp.MustCompile("aws:/[^/]*/[^/]*/([^/]+)")
+	match := rx.FindStringSubmatch(id)
+	if len(match) >= 2 {
+		return match[1]
+	}
+
+	// Return id for GCP Provider, Azure Provider, CSV Provider and Custom Provider
+	return id
+}
+
+// ParseLBID attempts to parse a LB ProviderId from a string based on formats from the various providers and
+// returns the string as is if it cannot find a match
+func ParseLBID(id string) string {
+	rx := regexp.MustCompile("^([^-]+)-.+amazonaws\\.com$") // Capture "ad9d88195b52a47c89b5055120f28c58" from "ad9d88195b52a47c89b5055120f28c58-1037804914.us-east-2.elb.amazonaws.com"
+	match := rx.FindStringSubmatch(id)
+	if len(match) >= 2 {
+		return match[1]
+	}
+
+	// Return id for GCP Provider, Azure Provider, CSV Provider and Custom Provider
+	return id
+}

pkg/cloud/providerconfig.go

@@ -9,7 +9,7 @@ import (
 	"sync"
 
 	"github.com/kubecost/cost-model/pkg/env"
-	"github.com/kubecost/cost-model/pkg/util"
+	"github.com/kubecost/cost-model/pkg/util/fileutil"
 	"github.com/kubecost/cost-model/pkg/util/json"
 	"github.com/microcosm-cc/bluemonday"
 
@@ -92,6 +92,11 @@ func (pc *ProviderConfig) loadConfig(writeIfNotExists bool) (*CustomPricing, err
 	if pc.customPricing.SpotGPU == "" {
 		pc.customPricing.SpotGPU = DefaultPricing().SpotGPU // Migration for users without this value set by default.
 	}
+
+	if pc.customPricing.ShareTenancyCosts == "" {
+		pc.customPricing.ShareTenancyCosts = defaultShareTenancyCost
+	}
+
 	return pc.customPricing, nil
 }
 
@@ -177,10 +182,12 @@ func DefaultPricing() *CustomPricing {
 		RegionNetworkEgress:   "0.01",
 		InternetNetworkEgress: "0.12",
 		CustomPricesEnabled:   "false",
+		ShareTenancyCosts:     "true",
 	}
 }
 
 func SetCustomPricingField(obj *CustomPricing, name string, value string) error {
+
 	structValue := reflect.ValueOf(obj).Elem()
 	structFieldValue := structValue.FieldByName(name)
 
@@ -211,7 +218,7 @@ func SetCustomPricingField(obj *CustomPricing, name string, value string) error
 // but the error isn't relevant to the path. This can happen when the current
 // user doesn't have permission to access the file.
 func fileExists(filename string) (bool, error) {
-	return util.FileExists(filename) // delegate to utility method
+	return fileutil.FileExists(filename) // delegate to utility method
 }
 
 // Returns the configuration directory concatenated with a specific config file name

pkg/clustercache/clustercache.go

@@ -7,6 +7,8 @@ import (
 	"k8s.io/klog"
 
 	appsv1 "k8s.io/api/apps/v1"
+	autoscaling "k8s.io/api/autoscaling/v2beta1"
+	batchv1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
 	stv1 "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/fields"
@@ -53,9 +55,18 @@ type ClusterCache interface {
 	// GetAllPersistentVolumes returns all the cached persistent volumes
 	GetAllPersistentVolumes() []*v1.PersistentVolume
 
+	// GetAllPersistentVolumeClaims returns all the cached persistent volume claims
+	GetAllPersistentVolumeClaims() []*v1.PersistentVolumeClaim
+
 	// GetAllStorageClasses returns all the cached storage classes
 	GetAllStorageClasses() []*stv1.StorageClass
 
+	// GetAllJobs returns all the cached jobs
+	GetAllJobs() []*batchv1.Job
+
+	// GetAllHorizontalPodAutoscalers() returns all cached horizontal pod autoscalers
+	GetAllHorizontalPodAutoscalers() []*autoscaling.HorizontalPodAutoscaler
+
 	// SetConfigMapUpdateFunc sets the configmap update function
 	SetConfigMapUpdateFunc(func(interface{}))
 }
@@ -74,7 +85,10 @@ type KubernetesClusterCache struct {
 	statefulsetWatch       WatchController
 	replicasetWatch        WatchController
 	pvWatch                WatchController
+	pvcWatch               WatchController
 	storageClassWatch      WatchController
+	jobsWatch              WatchController
+	hpaWatch               WatchController
 	stop                   chan struct{}
 }
 
@@ -87,6 +101,8 @@ func NewKubernetesClusterCache(client kubernetes.Interface) ClusterCache {
 	coreRestClient := client.CoreV1().RESTClient()
 	appsRestClient := client.AppsV1().RESTClient()
 	storageRestClient := client.StorageV1().RESTClient()
+	batchClient := client.BatchV1().RESTClient()
+	autoscalingClient := client.AutoscalingV2beta1().RESTClient()
 
 	kubecostNamespace := env.GetKubecostNamespace()
 	klog.Infof("NAMESPACE: %s", kubecostNamespace)
@@ -103,12 +119,15 @@ func NewKubernetesClusterCache(client kubernetes.Interface) ClusterCache {
 		statefulsetWatch:       NewCachingWatcher(appsRestClient, "statefulsets", &appsv1.StatefulSet{}, "", fields.Everything()),
 		replicasetWatch:        NewCachingWatcher(appsRestClient, "replicasets", &appsv1.ReplicaSet{}, "", fields.Everything()),
 		pvWatch:                NewCachingWatcher(coreRestClient, "persistentvolumes", &v1.PersistentVolume{}, "", fields.Everything()),
+		pvcWatch:               NewCachingWatcher(coreRestClient, "persistentvolumeclaims", &v1.PersistentVolumeClaim{}, "", fields.Everything()),
 		storageClassWatch:      NewCachingWatcher(storageRestClient, "storageclasses", &stv1.StorageClass{}, "", fields.Everything()),
+		jobsWatch:              NewCachingWatcher(batchClient, "jobs", &batchv1.Job{}, "", fields.Everything()),
+		hpaWatch:               NewCachingWatcher(autoscalingClient, "horizontalpodautoscalers", &autoscaling.HorizontalPodAutoscaler{}, "", fields.Everything()),
 	}
 
 	// Wait for each caching watcher to initialize
 	var wg sync.WaitGroup
-	wg.Add(11)
+	wg.Add(14)
 
 	cancel := make(chan struct{})
 
@@ -122,7 +141,10 @@ func NewKubernetesClusterCache(client kubernetes.Interface) ClusterCache {
 	go initializeCache(kcc.statefulsetWatch, &wg, cancel)
 	go initializeCache(kcc.replicasetWatch, &wg, cancel)
 	go initializeCache(kcc.pvWatch, &wg, cancel)
+	go initializeCache(kcc.pvcWatch, &wg, cancel)
 	go initializeCache(kcc.storageClassWatch, &wg, cancel)
+	go initializeCache(kcc.jobsWatch, &wg, cancel)
+	go initializeCache(kcc.hpaWatch, &wg, cancel)
 
 	wg.Wait()
 
@@ -145,7 +167,10 @@ func (kcc *KubernetesClusterCache) Run() {
 	go kcc.statefulsetWatch.Run(1, stopCh)
 	go kcc.replicasetWatch.Run(1, stopCh)
 	go kcc.pvWatch.Run(1, stopCh)
+	go kcc.pvcWatch.Run(1, stopCh)
 	go kcc.storageClassWatch.Run(1, stopCh)
+	go kcc.jobsWatch.Run(1, stopCh)
+	go kcc.hpaWatch.Run(1, stopCh)
 
 	kcc.stop = stopCh
 }
@@ -244,6 +269,15 @@ func (kcc *KubernetesClusterCache) GetAllPersistentVolumes() []*v1.PersistentVol
 	return pvs
 }
 
+func (kcc *KubernetesClusterCache) GetAllPersistentVolumeClaims() []*v1.PersistentVolumeClaim {
+	var pvcs []*v1.PersistentVolumeClaim
+	items := kcc.pvcWatch.GetAll()
+	for _, pvc := range items {
+		pvcs = append(pvcs, pvc.(*v1.PersistentVolumeClaim))
+	}
+	return pvcs
+}
+
 func (kcc *KubernetesClusterCache) GetAllStorageClasses() []*stv1.StorageClass {
 	var storageClasses []*stv1.StorageClass
 	items := kcc.storageClassWatch.GetAll()
@@ -253,6 +287,24 @@ func (kcc *KubernetesClusterCache) GetAllStorageClasses() []*stv1.StorageClass {
 	return storageClasses
 }
 
+func (kcc *KubernetesClusterCache) GetAllJobs() []*batchv1.Job {
+	var jobs []*batchv1.Job
+	items := kcc.jobsWatch.GetAll()
+	for _, job := range items {
+		jobs = append(jobs, job.(*batchv1.Job))
+	}
+	return jobs
+}
+
+func (kcc *KubernetesClusterCache) GetAllHorizontalPodAutoscalers() []*autoscaling.HorizontalPodAutoscaler {
+	var hpas []*autoscaling.HorizontalPodAutoscaler
+	items := kcc.hpaWatch.GetAll()
+	for _, hpa := range items {
+		hpas = append(hpas, hpa.(*autoscaling.HorizontalPodAutoscaler))
+	}
+	return hpas
+}
+
 func (kcc *KubernetesClusterCache) SetConfigMapUpdateFunc(f func(interface{})) {
 	kcc.kubecostConfigMapWatch.SetUpdateHandler(f)
 }

pkg/util/blockingqueue.go → pkg/collections/blockingqueue.go

@@ -1,4 +1,4 @@
-package util
+package collections
 
 import (
 	"sync"
@@ -17,11 +17,22 @@ type BlockingQueue interface {
 	// Dequeue removes the first item from the queue and returns it.
 	Dequeue() interface{}
 
+	// TryDequeue attempts to remove the first item from the queue and return it. This
+	// method does not block, and instead, returns true if the item was available and false
+	// otherwise
+	TryDequeue() (interface{}, bool)
+
+	// Each blocks modification and allows iteration of the queue.
+	Each(f func(int, interface{}))
+
 	// Length returns the length of the queue
 	Length() int
 
 	// IsEmpty returns true if the queue is empty
 	IsEmpty() bool
+
+	// Clear empties the queue
+	Clear()
 }
 
 // blockingSliceQueue is an implementation of BlockingQueue which uses a slice for storage.
@@ -70,6 +81,35 @@ func (q *blockingSliceQueue) Dequeue() interface{} {
 	return e
 }
 
+// TryDequeue attempts to remove the first item from the queue and return it. This
+// method does not block, and instead, returns true if the item was available and false
+// otherwise
+func (q *blockingSliceQueue) TryDequeue() (interface{}, bool) {
+	q.l.Lock()
+	defer q.l.Unlock()
+
+	if len(q.q) == 0 {
+		return nil, false
+	}
+
+	e := q.q[0]
+
+	// nil 0 index to prevent leak
+	q.q[0] = nil
+	q.q = q.q[1:]
+	return e, true
+}
+
+// Each blocks modification and allows iteration of the queue.
+func (q *blockingSliceQueue) Each(f func(int, interface{})) {
+	q.l.Lock()
+	defer q.l.Unlock()
+
+	for i, entry := range q.q {
+		f(i, entry)
+	}
+}
+
 // Length returns the length of the queue
 func (q *blockingSliceQueue) Length() int {
 	q.l.Lock()
@@ -82,3 +122,15 @@ func (q *blockingSliceQueue) Length() int {
 func (q *blockingSliceQueue) IsEmpty() bool {
 	return q.Length() == 0
 }
+
+// Clear empties the queue
+func (q *blockingSliceQueue) Clear() {
+	q.l.Lock()
+	defer q.l.Unlock()
+
+	// seems optimal here to create a new underlying slice/array to
+	// avoid capacity ballooning, but does feel like an implementation
+	// specific detail -- we can revisit if there are other relevant
+	// use-cases
+	q.q = []interface{}{}
+}

pkg/costmodel/aggregation.go

@@ -10,6 +10,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/kubecost/cost-model/pkg/util/httputil"
+	"github.com/kubecost/cost-model/pkg/util/timeutil"
+
 	"github.com/julienschmidt/httprouter"
 	"github.com/kubecost/cost-model/pkg/cloud"
 	"github.com/kubecost/cost-model/pkg/env"
@@ -118,9 +121,9 @@ func (a *Aggregation) RateCoefficient(rateStr string, resolutionHours float64) f
 	coeff := 1.0
 	switch rateStr {
 	case "daily":
-		coeff = util.HoursPerDay
+		coeff = timeutil.HoursPerDay
 	case "monthly":
-		coeff = util.HoursPerMonth
+		coeff = timeutil.HoursPerMonth
 	}
 
 	return coeff / a.TotalHours(resolutionHours)
@@ -195,7 +198,7 @@ func GetTotalContainerCost(costData map[string]*CostData, rate string, cp cloud.
 	return totalContainerCost
 }
 
-func (a *Accesses) ComputeIdleCoefficient(costData map[string]*CostData, cli prometheusClient.Client, cp cloud.Provider, discount float64, customDiscount float64, windowString, offset string) (map[string]float64, error) {
+func (a *Accesses) ComputeIdleCoefficient(costData map[string]*CostData, cli prometheusClient.Client, cp cloud.Provider, discount float64, customDiscount float64, window, offset time.Duration) (map[string]float64, error) {
 	coefficients := make(map[string]float64)
 
 	profileName := "ComputeIdleCoefficient: ComputeClusterCosts"
@@ -203,12 +206,12 @@ func (a *Accesses) ComputeIdleCoefficient(costData map[string]*CostData, cli pro
 
 	var clusterCosts map[string]*ClusterCosts
 	var err error
-
-	key := fmt.Sprintf("%s:%s", windowString, offset)
+	fmtWindow, fmtOffset := timeutil.DurationOffsetStrings(window, offset)
+	key := fmt.Sprintf("%s:%s", fmtWindow, fmtOffset)
 	if data, valid := a.ClusterCostsCache.Get(key); valid {
 		clusterCosts = data.(map[string]*ClusterCosts)
 	} else {
-		clusterCosts, err = a.ComputeClusterCosts(cli, cp, windowString, offset, false)
+		clusterCosts, err = a.ComputeClusterCosts(cli, cp, window, offset, false)
 		if err != nil {
 			return nil, err
 		}
@@ -224,7 +227,7 @@ func (a *Accesses) ComputeIdleCoefficient(costData map[string]*CostData, cli pro
 		}
 
 		if costs.TotalCumulative == 0 {
-			return nil, fmt.Errorf("TotalCumulative cluster cost for cluster '%s' returned 0 over window '%s' offset '%s'", cid, windowString, offset)
+			return nil, fmt.Errorf("TotalCumulative cluster cost for cluster '%s' returned 0 over window '%s' offset '%s'", cid, fmtWindow, fmtOffset)
 		}
 
 		totalContainerCost := 0.0
@@ -1450,16 +1453,11 @@ func (a *Accesses) ComputeAggregateCostModel(promClient prometheusClient.Client,
 
 	sc := make(map[string]*SharedCostInfo)
 	if !disableSharedOverhead {
-		for key, val := range c.SharedCosts {
-			cost, err := strconv.ParseFloat(val, 64)
-			durationCoefficient := window.Hours() / util.HoursPerMonth
-			if err != nil {
-				return nil, "", fmt.Errorf("unable to parse shared cost %s: %s", val, err)
-			}
-			sc[key] = &SharedCostInfo{
-				Name: key,
-				Cost: cost * durationCoefficient,
-			}
+		costPerMonth := c.GetSharedOverheadCostPerMonth()
+		durationCoefficient := window.Hours() / timeutil.HoursPerMonth
+		sc["total"] = &SharedCostInfo{
+			Name: "total",
+			Cost: costPerMonth * durationCoefficient,
 		}
 	}
 
@@ -1491,11 +1489,9 @@ func (a *Accesses) ComputeAggregateCostModel(promClient prometheusClient.Client,
 			}
 		}
 
-		// Convert to Prometheus-compatible strings
-		durStr, offStr := util.DurationOffsetStrings(dur, off)
-
-		idleCoefficients, err = a.ComputeIdleCoefficient(costData, promClient, a.CloudProvider, discount, customDiscount, durStr, offStr)
+		idleCoefficients, err = a.ComputeIdleCoefficient(costData, promClient, a.CloudProvider, discount, customDiscount, dur, off)
 		if err != nil {
+			durStr, offStr := timeutil.DurationOffsetStrings(dur, off)
 			log.Errorf("ComputeAggregateCostModel: error computing idle coefficient: duration=%s, offset=%s, err=%s", durStr, offStr, err)
 			return nil, "", err
 		}
@@ -1743,10 +1739,16 @@ func (a *Accesses) warmAggregateCostModelCache() {
 	// for the given duration. Cache is intentionally set to expire (i.e. noExpireCache=false) so that
 	// if the default parameters change, the old cached defaults with eventually expire. Thus, the
 	// timing of the cache expiry/refresh is the only mechanism ensuring 100% cache warmth.
-	warmFunc := func(duration, durationHrs, offset string, cacheEfficiencyData bool) (error, error) {
+	warmFunc := func(duration, offset time.Duration, cacheEfficiencyData bool) (error, error) {
+		if a.ThanosClient != nil {
+			duration = thanos.OffsetDuration()
+			log.Infof("Setting Offset to %s", duration)
+		}
+		fmtDuration, fmtOffset := timeutil.DurationOffsetStrings(duration, offset)
+		durationHrs, err := timeutil.FormatDurationStringDaysToHours(fmtDuration)
 		promClient := a.GetPrometheusClient(true)
 
-		windowStr := fmt.Sprintf("%s offset %s", duration, offset)
+		windowStr := fmt.Sprintf("%s offset %s", fmtDuration, fmtOffset)
 		window, err := kubecost.ParseWindowUTC(windowStr)
 		if err != nil {
 			return nil, fmt.Errorf("invalid window from window string: %s", windowStr)
@@ -1777,17 +1779,14 @@ func (a *Accesses) warmAggregateCostModelCache() {
 
 		aggKey := GenerateAggKey(window, field, subfields, aggOpts)
 		log.Infof("aggregation: cache warming defaults: %s", aggKey)
-		key := fmt.Sprintf("%s:%s", durationHrs, offset)
+		key := fmt.Sprintf("%s:%s", durationHrs, fmtOffset)
 
 		_, _, aggErr := a.ComputeAggregateCostModel(promClient, window, field, subfields, aggOpts)
 		if aggErr != nil {
 			log.Infof("Error building cache %s: %s", window, aggErr)
 		}
-		if a.ThanosClient != nil {
-			offset = thanos.Offset()
-			log.Infof("Setting offset to %s", offset)
-		}
-		totals, err := a.ComputeClusterCosts(promClient, a.CloudProvider, durationHrs, offset, cacheEfficiencyData)
+
+		totals, err := a.ComputeClusterCosts(promClient, a.CloudProvider, duration, offset, cacheEfficiencyData)
 		if err != nil {
 			log.Infof("Error building cluster costs cache %s", key)
 		}
@@ -1799,9 +1798,9 @@ func (a *Accesses) warmAggregateCostModelCache() {
 		}
 		if len(totals) > 0 && maxMinutesWithData > clusterCostsCacheMinutes {
 			a.ClusterCostsCache.Set(key, totals, a.GetCacheExpiration(window.Duration()))
-			log.Infof("caching %s cluster costs for %s", duration, a.GetCacheExpiration(window.Duration()))
+			log.Infof("caching %s cluster costs for %s", fmtDuration, a.GetCacheExpiration(window.Duration()))
 		} else {
-			log.Warningf("not caching %s cluster costs: no data or less than %f minutes data ", duration, clusterCostsCacheMinutes)
+			log.Warningf("not caching %s cluster costs: no data or less than %f minutes data ", fmtDuration, clusterCostsCacheMinutes)
 		}
 		return aggErr, err
 	}
@@ -1810,18 +1809,16 @@ func (a *Accesses) warmAggregateCostModelCache() {
 	go func(sem *util.Semaphore) {
 		defer errors.HandlePanic()
 
-		duration := "1d"
-		offset := "1m"
-		durHrs := "24h"
-		dur := 24 * time.Hour
+		offset := time.Minute
+		duration := 24 * time.Hour
 
 		for {
 			sem.Acquire()
-			warmFunc(duration, durHrs, offset, true)
+			warmFunc(duration, offset, true)
 			sem.Return()
 
-			log.Infof("aggregation: warm cache: %s", duration)
-			time.Sleep(a.GetCacheRefresh(dur))
+			log.Infof("aggregation: warm cache: %s", timeutil.DurationString(duration))
+			time.Sleep(a.GetCacheRefresh(duration))
 		}
 	}(sem)
 
@@ -1830,18 +1827,16 @@ func (a *Accesses) warmAggregateCostModelCache() {
 		go func(sem *util.Semaphore) {
 			defer errors.HandlePanic()
 
-			duration := "2d"
-			offset := "1m"
-			durHrs := "48h"
-			dur := 2 * 24 * time.Hour
+			offset := time.Minute
+			duration := 2 * 24 * time.Hour
 
 			for {
 				sem.Acquire()
-				warmFunc(duration, durHrs, offset, false)
+				warmFunc(duration, offset, false)
 				sem.Return()
 
-				log.Infof("aggregation: warm cache: %s", duration)
-				time.Sleep(a.GetCacheRefresh(dur))
+				log.Infof("aggregation: warm cache: %s", timeutil.DurationString(duration))
+				time.Sleep(a.GetCacheRefresh(duration))
 			}
 		}(sem)
 
@@ -1849,19 +1844,17 @@ func (a *Accesses) warmAggregateCostModelCache() {
 		go func(sem *util.Semaphore) {
 			defer errors.HandlePanic()
 
-			duration := "7d"
-			offset := "1m"
-			durHrs := "168h"
-			dur := 7 * 24 * time.Hour
+			offset := time.Minute
+			duration := 7 * 24 * time.Hour
 
 			for {
 				sem.Acquire()
-				aggErr, err := warmFunc(duration, durHrs, offset, false)
+				aggErr, err := warmFunc(duration, offset, false)
 				sem.Return()
 
-				log.Infof("aggregation: warm cache: %s", duration)
+				log.Infof("aggregation: warm cache: %s", timeutil.DurationString(duration))
 				if aggErr == nil && err == nil {
-					time.Sleep(a.GetCacheRefresh(dur))
+					time.Sleep(a.GetCacheRefresh(duration))
 				} else {
 					time.Sleep(5 * time.Minute)
 				}
@@ -1873,16 +1866,14 @@ func (a *Accesses) warmAggregateCostModelCache() {
 			defer errors.HandlePanic()
 
 			for {
-				duration := "30d"
-				offset := "1m"
-				durHrs := "720h"
-				dur := 30 * 24 * time.Hour
+				offset := time.Minute
+				duration := 30 * 24 * time.Hour
 
 				sem.Acquire()
-				aggErr, err := warmFunc(duration, durHrs, offset, false)
+				aggErr, err := warmFunc(duration, offset, false)
 				sem.Return()
 				if aggErr == nil && err == nil {
-					time.Sleep(a.GetCacheRefresh(dur))
+					time.Sleep(a.GetCacheRefresh(duration))
 				} else {
 					time.Sleep(5 * time.Minute)
 				}
@@ -2069,7 +2060,7 @@ func (a *Accesses) AggregateCostModelHandler(w http.ResponseWriter, r *http.Requ
 	data, message, err = a.AggAPI.ComputeAggregateCostModel(promClient, window, field, subfields, opts)
 
 	// Find any warnings in http request context
-	warning, _ := util.GetWarning(r)
+	warning, _ := httputil.GetWarning(r)
 
 	if err != nil {
 		if emptyErr, ok := err.(*EmptyDataError); ok {
@@ -2117,7 +2108,7 @@ func (a *Accesses) AggregateCostModelHandler(w http.ResponseWriter, r *http.Requ
 // ParseAggregationProperties attempts to parse and return aggregation properties
 // encoded under the given key. If none exist, or if parsing fails, an error
 // is returned with empty AllocationProperties.
-func ParseAggregationProperties(qp util.QueryParams, key string) ([]string, error) {
+func ParseAggregationProperties(qp httputil.QueryParams, key string) ([]string, error) {
 	aggregateBy := []string{}
 	for _, agg := range qp.GetList(key, ",") {
 		aggregate := strings.TrimSpace(agg)
@@ -2138,7 +2129,7 @@ func ParseAggregationProperties(qp util.QueryParams, key string) ([]string, erro
 func (a *Accesses) ComputeAllocationHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")
 
-	qp := util.NewQueryParams(r.URL.Query())
+	qp := httputil.NewQueryParams(r.URL.Query())
 
 	// Window is a required field describing the window of time over which to
 	// compute allocation data.

pkg/costmodel/allocation.go

@@ -7,63 +7,65 @@ import (
 	"strings"
 	"time"
 
+	"github.com/kubecost/cost-model/pkg/util/timeutil"
+
 	"github.com/kubecost/cost-model/pkg/cloud"
 	"github.com/kubecost/cost-model/pkg/env"
 	"github.com/kubecost/cost-model/pkg/kubecost"
 	"github.com/kubecost/cost-model/pkg/log"
 	"github.com/kubecost/cost-model/pkg/prom"
-	"github.com/kubecost/cost-model/pkg/util"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/klog"
 )
 
 const (
-	queryFmtPods              = `avg(kube_pod_container_status_running{}) by (pod, namespace, %s)[%s:%s]%s`
-	queryFmtRAMBytesAllocated = `avg(avg_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s, provider_id)`
-	queryFmtRAMRequests       = `avg(avg_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte", container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
-	queryFmtRAMUsageAvg       = `avg(avg_over_time(container_memory_working_set_bytes{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, %s)`
-	queryFmtRAMUsageMax       = `max(max_over_time(container_memory_working_set_bytes{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, %s)`
-	queryFmtCPUCoresAllocated = `avg(avg_over_time(container_cpu_allocation{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
-	queryFmtCPURequests       = `avg(avg_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
-	queryFmtCPUUsageAvg       = `avg(rate(container_cpu_usage_seconds_total{container_name!="", container_name!="POD", instance!=""}[%s]%s)) by (container_name, pod_name, namespace, instance, %s)`
-
-	// This query could be written without the recording rule
-	// "kubecost_savings_container_cpu_usage_seconds", but we should
-	// only do that when we're ready to incur the performance tradeoffs
-	// with subqueries which would probably be in the world of hourly
-	// ETL.
-	//
-	// See PromQL subquery documentation for a rate example:
-	// https://prometheus.io/blog/2019/01/28/subquery-support/#examples
-	queryFmtCPUUsageMax           = `max(max_over_time(kubecost_savings_container_cpu_usage_seconds[%s]%s)) by (container_name, pod_name, namespace, instance, %s)`
-	queryFmtGPUsRequested         = `avg(avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
-	queryFmtNodeCostPerCPUHr      = `avg(avg_over_time(node_cpu_hourly_cost[%s]%s)) by (node, %s, instance_type, provider_id)`
-	queryFmtNodeCostPerRAMGiBHr   = `avg(avg_over_time(node_ram_hourly_cost[%s]%s)) by (node, %s, instance_type, provider_id)`
-	queryFmtNodeCostPerGPUHr      = `avg(avg_over_time(node_gpu_hourly_cost[%s]%s)) by (node, %s, instance_type, provider_id)`
-	queryFmtNodeIsSpot            = `avg_over_time(kubecost_node_is_spot[%s]%s)`
-	queryFmtPVCInfo               = `avg(kube_persistentvolumeclaim_info{volumename != ""}) by (persistentvolumeclaim, storageclass, volumename, namespace, %s)[%s:%s]%s`
-	queryFmtPVBytes               = `avg(avg_over_time(kube_persistentvolume_capacity_bytes[%s]%s)) by (persistentvolume, %s)`
-	queryFmtPodPVCAllocation      = `avg(avg_over_time(pod_pvc_allocation[%s]%s)) by (persistentvolume, persistentvolumeclaim, pod, namespace, %s)`
-	queryFmtPVCBytesRequested     = `avg(avg_over_time(kube_persistentvolumeclaim_resource_requests_storage_bytes{}[%s]%s)) by (persistentvolumeclaim, namespace, %s)`
-	queryFmtPVCostPerGiBHour      = `avg(avg_over_time(pv_hourly_cost[%s]%s)) by (volumename, %s)`
-	queryFmtNetZoneGiB            = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="true"}[%s]%s)) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
-	queryFmtNetZoneCostPerGiB     = `avg(avg_over_time(kubecost_network_zone_egress_cost{}[%s]%s)) by (%s)`
-	queryFmtNetRegionGiB          = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="false"}[%s]%s)) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
-	queryFmtNetRegionCostPerGiB   = `avg(avg_over_time(kubecost_network_region_egress_cost{}[%s]%s)) by (%s)`
-	queryFmtNetInternetGiB        = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="true"}[%s]%s)) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
-	queryFmtNetInternetCostPerGiB = `avg(avg_over_time(kubecost_network_internet_egress_cost{}[%s]%s)) by (%s)`
-	queryFmtNamespaceLabels       = `avg_over_time(kube_namespace_labels[%s]%s)`
-	queryFmtNamespaceAnnotations  = `avg_over_time(kube_namespace_annotations[%s]%s)`
-	queryFmtPodLabels             = `avg_over_time(kube_pod_labels[%s]%s)`
-	queryFmtPodAnnotations        = `avg_over_time(kube_pod_annotations[%s]%s)`
-	queryFmtServiceLabels         = `avg_over_time(service_selector_labels[%s]%s)`
-	queryFmtDeploymentLabels      = `avg_over_time(deployment_match_labels[%s]%s)`
-	queryFmtStatefulSetLabels     = `avg_over_time(statefulSet_match_labels[%s]%s)`
-	queryFmtDaemonSetLabels       = `sum(avg_over_time(kube_pod_owner{owner_kind="DaemonSet"}[%s]%s)) by (pod, owner_name, namespace, %s)`
-	queryFmtJobLabels             = `sum(avg_over_time(kube_pod_owner{owner_kind="Job"}[%s]%s)) by (pod, owner_name, namespace ,%s)`
-	queryFmtLBCostPerHr           = `avg(avg_over_time(kubecost_load_balancer_cost[%s]%s)) by (namespace, service_name, %s)`
-	queryFmtLBActiveMins          = `count(kubecost_load_balancer_cost) by (namespace, service_name, %s)[%s:%s]%s`
+	queryFmtPods                     = `avg(kube_pod_container_status_running{}) by (pod, namespace, %s)[%s:%s]%s`
+	queryFmtRAMBytesAllocated        = `avg(avg_over_time(container_memory_allocation_bytes{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s, provider_id)`
+	queryFmtRAMRequests              = `avg(avg_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte", container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
+	queryFmtRAMUsageAvg              = `avg(avg_over_time(container_memory_working_set_bytes{container!="", container_name!="POD", container!="POD"}[%s]%s)) by (container_name, container, pod_name, pod, namespace, instance, %s)`
+	queryFmtRAMUsageMax              = `max(max_over_time(container_memory_working_set_bytes{container!="", container_name!="POD", container!="POD"}[%s]%s)) by (container_name, container, pod_name, pod, namespace, instance, %s)`
+	queryFmtCPUCoresAllocated        = `avg(avg_over_time(container_cpu_allocation{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
+	queryFmtCPURequests              = `avg(avg_over_time(kube_pod_container_resource_requests{resource="cpu", unit="core", container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
+	queryFmtCPUUsageAvg              = `avg(rate(container_cpu_usage_seconds_total{container!="", container_name!="POD", container!="POD"}[%s]%s)) by (container_name, container, pod_name, pod, namespace, instance, %s)`
+	queryFmtCPUUsageMax              = `max(rate(container_cpu_usage_seconds_total{container!="", container_name!="POD", container!="POD"}[%s]%s)) by (container_name, container, pod_name, pod, namespace, instance, %s)`
+	queryFmtGPUsRequested            = `avg(avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
+	queryFmtGPUsAllocated            = `avg(avg_over_time(container_gpu_allocation{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
+	queryFmtNodeCostPerCPUHr         = `avg(avg_over_time(node_cpu_hourly_cost[%s]%s)) by (node, %s, instance_type, provider_id)`
+	queryFmtNodeCostPerRAMGiBHr      = `avg(avg_over_time(node_ram_hourly_cost[%s]%s)) by (node, %s, instance_type, provider_id)`
+	queryFmtNodeCostPerGPUHr         = `avg(avg_over_time(node_gpu_hourly_cost[%s]%s)) by (node, %s, instance_type, provider_id)`
+	queryFmtNodeIsSpot               = `avg_over_time(kubecost_node_is_spot[%s]%s)`
+	queryFmtPVCInfo                  = `avg(kube_persistentvolumeclaim_info{volumename != ""}) by (persistentvolumeclaim, storageclass, volumename, namespace, %s)[%s:%s]%s`
+	queryFmtPVBytes                  = `avg(avg_over_time(kube_persistentvolume_capacity_bytes[%s]%s)) by (persistentvolume, %s)`
+	queryFmtPodPVCAllocation         = `avg(avg_over_time(pod_pvc_allocation[%s]%s)) by (persistentvolume, persistentvolumeclaim, pod, namespace, %s)`
+	queryFmtPVCBytesRequested        = `avg(avg_over_time(kube_persistentvolumeclaim_resource_requests_storage_bytes{}[%s]%s)) by (persistentvolumeclaim, namespace, %s)`
+	queryFmtPVCostPerGiBHour         = `avg(avg_over_time(pv_hourly_cost[%s]%s)) by (volumename, %s)`
+	queryFmtNetZoneGiB               = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="true"}[%s]%s)) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
+	queryFmtNetZoneCostPerGiB        = `avg(avg_over_time(kubecost_network_zone_egress_cost{}[%s]%s)) by (%s)`
+	queryFmtNetRegionGiB             = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="false", sameZone="false", sameRegion="false"}[%s]%s)) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
+	queryFmtNetRegionCostPerGiB      = `avg(avg_over_time(kubecost_network_region_egress_cost{}[%s]%s)) by (%s)`
+	queryFmtNetInternetGiB           = `sum(increase(kubecost_pod_network_egress_bytes_total{internet="true"}[%s]%s)) by (pod_name, namespace, %s) / 1024 / 1024 / 1024`
+	queryFmtNetInternetCostPerGiB    = `avg(avg_over_time(kubecost_network_internet_egress_cost{}[%s]%s)) by (%s)`
+	queryFmtNetReceiveBytes          = `sum(increase(container_network_receive_bytes_total{pod!="", container="POD"}[%s]%s)) by (pod_name, pod, namespace, %s)`
+	queryFmtNetTransferBytes         = `sum(increase(container_network_transmit_bytes_total{pod!="", container="POD"}[%s]%s)) by (pod_name, pod, namespace, %s)`
+	queryFmtNamespaceLabels          = `avg_over_time(kube_namespace_labels[%s]%s)`
+	queryFmtNamespaceAnnotations     = `avg_over_time(kube_namespace_annotations[%s]%s)`
+	queryFmtPodLabels                = `avg_over_time(kube_pod_labels[%s]%s)`
+	queryFmtPodAnnotations           = `avg_over_time(kube_pod_annotations[%s]%s)`
+	queryFmtServiceLabels            = `avg_over_time(service_selector_labels[%s]%s)`
+	queryFmtDeploymentLabels         = `avg_over_time(deployment_match_labels[%s]%s)`
+	queryFmtStatefulSetLabels        = `avg_over_time(statefulSet_match_labels[%s]%s)`
+	queryFmtDaemonSetLabels          = `sum(avg_over_time(kube_pod_owner{owner_kind="DaemonSet"}[%s]%s)) by (pod, owner_name, namespace, %s)`
+	queryFmtJobLabels                = `sum(avg_over_time(kube_pod_owner{owner_kind="Job"}[%s]%s)) by (pod, owner_name, namespace ,%s)`
+	queryFmtPodsWithReplicaSetOwner  = `sum(avg_over_time(kube_pod_owner{owner_kind="ReplicaSet"}[%s]%s)) by (pod, owner_name, namespace ,%s)`
+	queryFmtReplicaSetsWithoutOwners = `avg(avg_over_time(kube_replicaset_owner{owner_kind="<none>", owner_name="<none>"}[%s]%s)) by (replicaset, namespace, %s)`
+	queryFmtLBCostPerHr              = `avg(avg_over_time(kubecost_load_balancer_cost[%s]%s)) by (namespace, service_name, %s)`
+	queryFmtLBActiveMins             = `count(kubecost_load_balancer_cost) by (namespace, service_name, %s)[%s:%s]%s`
 )
 
+// This is a bit of a hack to work around garbage data from cadvisor
+// Ideally you cap each pod to the max CPU on its node, but that involves a bit more complexity, as it it would need to be done when allocations joins with asset data.
+const MAX_CPU_CAP = 512
+
 // CanCompute should return true if CostModel can act as a valid source for the
 // given time range. In the case of CostModel we want to attempt to compute as
 // long as the range starts in the past. If the CostModel ends up not having
@@ -122,9 +124,9 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	}
 
 	// Convert resolution duration to a query-ready string
-	resStr := util.DurationString(resolution)
+	resStr := timeutil.DurationString(resolution)
 
-	ctx := prom.NewContext(cm.PrometheusClient)
+	ctx := prom.NewNamedContext(cm.PrometheusClient, prom.AllocationContextName)
 
 	queryRAMBytesAllocated := fmt.Sprintf(queryFmtRAMBytesAllocated, durStr, offStr, env.GetPromClusterLabel())
 	resChRAMBytesAllocated := ctx.Query(queryRAMBytesAllocated)
@@ -153,6 +155,9 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	queryGPUsRequested := fmt.Sprintf(queryFmtGPUsRequested, durStr, offStr, env.GetPromClusterLabel())
 	resChGPUsRequested := ctx.Query(queryGPUsRequested)
 
+	queryGPUsAllocated := fmt.Sprintf(queryFmtGPUsAllocated, durStr, offStr, env.GetPromClusterLabel())
+	resChGPUsAllocated := ctx.Query(queryGPUsAllocated)
+
 	queryNodeCostPerCPUHr := fmt.Sprintf(queryFmtNodeCostPerCPUHr, durStr, offStr, env.GetPromClusterLabel())
 	resChNodeCostPerCPUHr := ctx.Query(queryNodeCostPerCPUHr)
 
@@ -180,6 +185,12 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	queryPVCostPerGiBHour := fmt.Sprintf(queryFmtPVCostPerGiBHour, durStr, offStr, env.GetPromClusterLabel())
 	resChPVCostPerGiBHour := ctx.Query(queryPVCostPerGiBHour)
 
+	queryNetTransferBytes := fmt.Sprintf(queryFmtNetTransferBytes, durStr, offStr, env.GetPromClusterLabel())
+	resChNetTransferBytes := ctx.Query(queryNetTransferBytes)
+
+	queryNetReceiveBytes := fmt.Sprintf(queryFmtNetReceiveBytes, durStr, offStr, env.GetPromClusterLabel())
+	resChNetReceiveBytes := ctx.Query(queryNetReceiveBytes)
+
 	queryNetZoneGiB := fmt.Sprintf(queryFmtNetZoneGiB, durStr, offStr, env.GetPromClusterLabel())
 	resChNetZoneGiB := ctx.Query(queryNetZoneGiB)
 
@@ -222,6 +233,12 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	queryDaemonSetLabels := fmt.Sprintf(queryFmtDaemonSetLabels, durStr, offStr, env.GetPromClusterLabel())
 	resChDaemonSetLabels := ctx.Query(queryDaemonSetLabels)
 
+	queryPodsWithReplicaSetOwner := fmt.Sprintf(queryFmtPodsWithReplicaSetOwner, durStr, offStr, env.GetPromClusterLabel())
+	resChPodsWithReplicaSetOwner := ctx.Query(queryPodsWithReplicaSetOwner)
+
+	queryReplicaSetsWithoutOwners := fmt.Sprintf(queryFmtReplicaSetsWithoutOwners, durStr, offStr, env.GetPromClusterLabel())
+	resChReplicaSetsWithoutOwners := ctx.Query(queryReplicaSetsWithoutOwners)
+
 	queryJobLabels := fmt.Sprintf(queryFmtJobLabels, durStr, offStr, env.GetPromClusterLabel())
 	resChJobLabels := ctx.Query(queryJobLabels)
 
@@ -240,6 +257,7 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	resRAMUsageAvg, _ := resChRAMUsageAvg.Await()
 	resRAMUsageMax, _ := resChRAMUsageMax.Await()
 	resGPUsRequested, _ := resChGPUsRequested.Await()
+	resGPUsAllocated, _ := resChGPUsAllocated.Await()
 
 	resNodeCostPerCPUHr, _ := resChNodeCostPerCPUHr.Await()
 	resNodeCostPerRAMGiBHr, _ := resChNodeCostPerRAMGiBHr.Await()
@@ -253,6 +271,8 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	resPVCBytesRequested, _ := resChPVCBytesRequested.Await()
 	resPodPVCAllocation, _ := resChPodPVCAllocation.Await()
 
+	resNetTransferBytes, _ := resChNetTransferBytes.Await()
+	resNetReceiveBytes, _ := resChNetReceiveBytes.Await()
 	resNetZoneGiB, _ := resChNetZoneGiB.Await()
 	resNetZoneCostPerGiB, _ := resChNetZoneCostPerGiB.Await()
 	resNetRegionGiB, _ := resChNetRegionGiB.Await()
@@ -268,6 +288,8 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	resDeploymentLabels, _ := resChDeploymentLabels.Await()
 	resStatefulSetLabels, _ := resChStatefulSetLabels.Await()
 	resDaemonSetLabels, _ := resChDaemonSetLabels.Await()
+	resPodsWithReplicaSetOwner, _ := resChPodsWithReplicaSetOwner.Await()
+	resReplicaSetsWithoutOwners, _ := resChReplicaSetsWithoutOwners.Await()
 	resJobLabels, _ := resChJobLabels.Await()
 	resLBCostPerHr, _ := resChLBCostPerHr.Await()
 	resLBActiveMins, _ := resChLBActiveMins.Await()
@@ -291,7 +313,8 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	applyRAMBytesRequested(podMap, resRAMRequests)
 	applyRAMBytesUsedAvg(podMap, resRAMUsageAvg)
 	applyRAMBytesUsedMax(podMap, resRAMUsageMax)
-	applyGPUsRequested(podMap, resGPUsRequested)
+	applyGPUsAllocated(podMap, resGPUsRequested, resGPUsAllocated)
+	applyNetworkTotals(podMap, resNetTransferBytes, resNetReceiveBytes)
 	applyNetworkAllocation(podMap, resNetZoneGiB, resNetZoneCostPerGiB)
 	applyNetworkAllocation(podMap, resNetRegionGiB, resNetRegionCostPerGiB)
 	applyNetworkAllocation(podMap, resNetInternetGiB, resNetInternetCostPerGiB)
@@ -311,10 +334,12 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	podStatefulSetMap := labelsToPodControllerMap(podLabels, resToStatefulSetLabels(resStatefulSetLabels))
 	podDaemonSetMap := resToPodDaemonSetMap(resDaemonSetLabels)
 	podJobMap := resToPodJobMap(resJobLabels)
+	podReplicaSetMap := resToPodReplicaSetMap(resPodsWithReplicaSetOwner, resReplicaSetsWithoutOwners)
 	applyControllersToPods(podMap, podDeploymentMap)
 	applyControllersToPods(podMap, podStatefulSetMap)
 	applyControllersToPods(podMap, podDaemonSetMap)
 	applyControllersToPods(podMap, podJobMap)
+	applyControllersToPods(podMap, podReplicaSetMap)
 
 	// TODO breakdown network costs?
 
@@ -322,9 +347,9 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	// for converting resource allocation data to cumulative costs.
 	nodeMap := map[nodeKey]*NodePricing{}
 
-	applyNodeCostPerCPUHr(nodeMap, resNodeCostPerCPUHr, cm.Provider.ParseID)
-	applyNodeCostPerRAMGiBHr(nodeMap, resNodeCostPerRAMGiBHr, cm.Provider.ParseID)
-	applyNodeCostPerGPUHr(nodeMap, resNodeCostPerGPUHr, cm.Provider.ParseID)
+	applyNodeCostPerCPUHr(nodeMap, resNodeCostPerCPUHr)
+	applyNodeCostPerRAMGiBHr(nodeMap, resNodeCostPerRAMGiBHr)
+	applyNodeCostPerGPUHr(nodeMap, resNodeCostPerGPUHr)
 	applyNodeSpot(nodeMap, resNodeIsSpot)
 	applyNodeDiscount(nodeMap, cm)
 
@@ -430,9 +455,9 @@ func (cm *CostModel) buildPodMap(window kubecost.Window, resolution, maxBatchSiz
 	start, end := *window.Start(), *window.End()
 
 	// Convert resolution duration to a query-ready string
-	resStr := util.DurationString(resolution)
+	resStr := timeutil.DurationString(resolution)
 
-	ctx := prom.NewContext(cm.PrometheusClient)
+	ctx := prom.NewNamedContext(cm.PrometheusClient, prom.AllocationContextName)
 
 	// Query for (start, end) by (pod, namespace, cluster) over the given
 	// window, using the given resolution, and if necessary in batches no
@@ -608,7 +633,7 @@ func applyPodResults(window kubecost.Window, resolution time.Duration, podMap ma
 
 func applyCPUCoresAllocated(podMap map[podKey]*Pod, resCPUCoresAllocated []*prom.QueryResult) {
 	for _, res := range resCPUCoresAllocated {
-		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace", "pod")
+		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
 		if err != nil {
 			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU allocation result missing field: %s", err)
 			continue
@@ -630,6 +655,10 @@ func applyCPUCoresAllocated(podMap map[podKey]*Pod, resCPUCoresAllocated []*prom
 		}
 
 		cpuCores := res.Values[0].Value
+		if cpuCores > MAX_CPU_CAP {
+			klog.Infof("[WARNING] Very large cpu allocation, clamping to %f", res.Values[0].Value*(pod.Allocations[container].Minutes()/60.0))
+			cpuCores = 0.0
+		}
 		hours := pod.Allocations[container].Minutes() / 60.0
 		pod.Allocations[container].CPUCoreHours = cpuCores * hours
 
@@ -644,7 +673,7 @@ func applyCPUCoresAllocated(podMap map[podKey]*Pod, resCPUCoresAllocated []*prom
 
 func applyCPUCoresRequested(podMap map[podKey]*Pod, resCPUCoresRequested []*prom.QueryResult) {
 	for _, res := range resCPUCoresRequested {
-		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace", "pod")
+		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
 		if err != nil {
 			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU request result missing field: %s", err)
 			continue
@@ -672,6 +701,10 @@ func applyCPUCoresRequested(podMap map[podKey]*Pod, resCPUCoresRequested []*prom
 		if pod.Allocations[container].CPUCores() < res.Values[0].Value {
 			pod.Allocations[container].CPUCoreHours = res.Values[0].Value * (pod.Allocations[container].Minutes() / 60.0)
 		}
+		if pod.Allocations[container].CPUCores() > MAX_CPU_CAP {
+			klog.Infof("[WARNING] Very large cpu allocation, clamping! to %f", res.Values[0].Value*(pod.Allocations[container].Minutes()/60.0))
+			pod.Allocations[container].CPUCoreHours = res.Values[0].Value * (pod.Allocations[container].Minutes() / 60.0)
+		}
 
 		node, err := res.GetString("node")
 		if err != nil {
@@ -684,7 +717,7 @@ func applyCPUCoresRequested(podMap map[podKey]*Pod, resCPUCoresRequested []*prom
 
 func applyCPUCoresUsedAvg(podMap map[podKey]*Pod, resCPUCoresUsedAvg []*prom.QueryResult) {
 	for _, res := range resCPUCoresUsedAvg {
-		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace", "pod_name")
+		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
 		if err != nil {
 			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage avg result missing field: %s", err)
 			continue
@@ -694,11 +727,13 @@ func applyCPUCoresUsedAvg(podMap map[podKey]*Pod, resCPUCoresUsedAvg []*prom.Que
 		if !ok {
 			continue
 		}
-
-		container, err := res.GetString("container_name")
-		if err != nil {
-			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage avg query result missing 'container': %s", key)
-			continue
+		container, err := res.GetString("container")
+		if container == "" || err != nil {
+			container, err = res.GetString("container_name")
+			if err != nil {
+				log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage avg query result missing 'container': %s", key)
+				continue
+			}
 		}
 
 		if _, ok := pod.Allocations[container]; !ok {
@@ -706,12 +741,16 @@ func applyCPUCoresUsedAvg(podMap map[podKey]*Pod, resCPUCoresUsedAvg []*prom.Que
 		}
 
 		pod.Allocations[container].CPUCoreUsageAverage = res.Values[0].Value
+		if res.Values[0].Value > MAX_CPU_CAP {
+			klog.Infof("[WARNING] Very large cpu USAGE, dropping outlier")
+			pod.Allocations[container].CPUCoreUsageAverage = 0.0
+		}
 	}
 }
 
 func applyCPUCoresUsedMax(podMap map[podKey]*Pod, resCPUCoresUsedMax []*prom.QueryResult) {
 	for _, res := range resCPUCoresUsedMax {
-		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace", "pod_name")
+		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
 		if err != nil {
 			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage max result missing field: %s", err)
 			continue
@@ -722,10 +761,13 @@ func applyCPUCoresUsedMax(podMap map[podKey]*Pod, resCPUCoresUsedMax []*prom.Que
 			continue
 		}
 
-		container, err := res.GetString("container_name")
-		if err != nil {
-			log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage max query result missing 'container': %s", key)
-			continue
+		container, err := res.GetString("container")
+		if container == "" || err != nil {
+			container, err = res.GetString("container_name")
+			if err != nil {
+				log.DedupedWarningf(10, "CostModel.ComputeAllocation: CPU usage max query result missing 'container': %s", key)
+				continue
+			}
 		}
 
 		if _, ok := pod.Allocations[container]; !ok {
@@ -744,7 +786,7 @@ func applyCPUCoresUsedMax(podMap map[podKey]*Pod, resCPUCoresUsedMax []*prom.Que
 
 func applyRAMBytesAllocated(podMap map[podKey]*Pod, resRAMBytesAllocated []*prom.QueryResult) {
 	for _, res := range resRAMBytesAllocated {
-		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace", "pod")
+		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
 		if err != nil {
 			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM allocation result missing field: %s", err)
 			continue
@@ -780,7 +822,7 @@ func applyRAMBytesAllocated(podMap map[podKey]*Pod, resRAMBytesAllocated []*prom
 
 func applyRAMBytesRequested(podMap map[podKey]*Pod, resRAMBytesRequested []*prom.QueryResult) {
 	for _, res := range resRAMBytesRequested {
-		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace", "pod")
+		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
 		if err != nil {
 			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM request result missing field: %s", err)
 			continue
@@ -820,7 +862,7 @@ func applyRAMBytesRequested(podMap map[podKey]*Pod, resRAMBytesRequested []*prom
 
 func applyRAMBytesUsedAvg(podMap map[podKey]*Pod, resRAMBytesUsedAvg []*prom.QueryResult) {
 	for _, res := range resRAMBytesUsedAvg {
-		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace", "pod_name")
+		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
 		if err != nil {
 			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM avg usage result missing field: %s", err)
 			continue
@@ -831,10 +873,13 @@ func applyRAMBytesUsedAvg(podMap map[podKey]*Pod, resRAMBytesUsedAvg []*prom.Que
 			continue
 		}
 
-		container, err := res.GetString("container_name")
-		if err != nil {
-			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM usage avg query result missing 'container': %s", key)
-			continue
+		container, err := res.GetString("container")
+		if container == "" || err != nil {
+			container, err = res.GetString("container_name")
+			if err != nil {
+				log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM usage avg query result missing 'container': %s", key)
+				continue
+			}
 		}
 
 		if _, ok := pod.Allocations[container]; !ok {
@@ -847,7 +892,7 @@ func applyRAMBytesUsedAvg(podMap map[podKey]*Pod, resRAMBytesUsedAvg []*prom.Que
 
 func applyRAMBytesUsedMax(podMap map[podKey]*Pod, resRAMBytesUsedMax []*prom.QueryResult) {
 	for _, res := range resRAMBytesUsedMax {
-		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace", "pod_name")
+		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
 		if err != nil {
 			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM usage max result missing field: %s", err)
 			continue
@@ -858,10 +903,13 @@ func applyRAMBytesUsedMax(podMap map[podKey]*Pod, resRAMBytesUsedMax []*prom.Que
 			continue
 		}
 
-		container, err := res.GetString("container_name")
-		if err != nil {
-			log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM usage max query result missing 'container': %s", key)
-			continue
+		container, err := res.GetString("container")
+		if container == "" || err != nil {
+			container, err = res.GetString("container_name")
+			if err != nil {
+				log.DedupedWarningf(10, "CostModel.ComputeAllocation: RAM usage max query result missing 'container': %s", key)
+				continue
+			}
 		}
 
 		if _, ok := pod.Allocations[container]; !ok {
@@ -878,9 +926,12 @@ func applyRAMBytesUsedMax(podMap map[podKey]*Pod, resRAMBytesUsedMax []*prom.Que
 	}
 }
 
-func applyGPUsRequested(podMap map[podKey]*Pod, resGPUsRequested []*prom.QueryResult) {
+func applyGPUsAllocated(podMap map[podKey]*Pod, resGPUsRequested []*prom.QueryResult, resGPUsAllocated []*prom.QueryResult) {
+	if len(resGPUsAllocated) > 0 { // Use the new query, when it's become available in a window
+		resGPUsRequested = resGPUsAllocated
+	}
 	for _, res := range resGPUsRequested {
-		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace", "pod")
+		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
 		if err != nil {
 			log.DedupedWarningf(10, "CostModel.ComputeAllocation: GPU request result missing field: %s", err)
 			continue
@@ -906,6 +957,41 @@ func applyGPUsRequested(podMap map[podKey]*Pod, resGPUsRequested []*prom.QueryRe
 	}
 }
 
+func applyNetworkTotals(podMap map[podKey]*Pod, resNetworkTransferBytes []*prom.QueryResult, resNetworkReceiveBytes []*prom.QueryResult) {
+	for _, res := range resNetworkTransferBytes {
+		podKey, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: Network Transfer Bytes query result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[podKey]
+		if !ok {
+			continue
+		}
+
+		for _, alloc := range pod.Allocations {
+			alloc.NetworkTransferBytes = res.Values[0].Value / float64(len(pod.Allocations))
+		}
+	}
+	for _, res := range resNetworkReceiveBytes {
+		podKey, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
+		if err != nil {
+			log.DedupedWarningf(10, "CostModel.ComputeAllocation: Network Receive Bytes query result missing field: %s", err)
+			continue
+		}
+
+		pod, ok := podMap[podKey]
+		if !ok {
+			continue
+		}
+
+		for _, alloc := range pod.Allocations {
+			alloc.NetworkReceiveBytes = res.Values[0].Value / float64(len(pod.Allocations))
+		}
+	}
+}
+
 func applyNetworkAllocation(podMap map[podKey]*Pod, resNetworkGiB []*prom.QueryResult, resNetworkCostPerGiB []*prom.QueryResult) {
 	costPerGiBByCluster := map[string]float64{}
 
@@ -919,7 +1005,7 @@ func applyNetworkAllocation(podMap map[podKey]*Pod, resNetworkGiB []*prom.QueryR
 	}
 
 	for _, res := range resNetworkGiB {
-		podKey, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace", "pod_name")
+		podKey, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
 		if err != nil {
 			log.DedupedWarningf(10, "CostModel.ComputeAllocation: Network allocation query result missing field: %s", err)
 			continue
@@ -963,7 +1049,7 @@ func resToPodLabels(resPodLabels []*prom.QueryResult) map[podKey]map[string]stri
 	podLabels := map[podKey]map[string]string{}
 
 	for _, res := range resPodLabels {
-		podKey, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace", "pod")
+		podKey, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
 		if err != nil {
 			continue
 		}
@@ -1005,7 +1091,7 @@ func resToPodAnnotations(resPodAnnotations []*prom.QueryResult) map[podKey]map[s
 	podAnnotations := map[podKey]map[string]string{}
 
 	for _, res := range resPodAnnotations {
-		podKey, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace", "pod")
+		podKey, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
 		if err != nil {
 			continue
 		}
@@ -1031,7 +1117,7 @@ func applyLabels(podMap map[podKey]*Pod, namespaceLabels map[namespaceKey]map[st
 			}
 			// Apply namespace labels first, then pod labels so that pod labels
 			// overwrite namespace labels.
-			nsKey := newNamespaceKey(podKey.Cluster, podKey.Namespace)
+			nsKey := podKey.namespaceKey // newNamespaceKey(podKey.Cluster, podKey.Namespace)
 			if labels, ok := namespaceLabels[nsKey]; ok {
 				for k, v := range labels {
 					allocLabels[k] = v
@@ -1252,6 +1338,48 @@ func resToPodJobMap(resJobLabels []*prom.QueryResult) map[podKey]controllerKey {
 	return jobLabels
 }
 
+func resToPodReplicaSetMap(resPodsWithReplicaSetOwner []*prom.QueryResult, resReplicaSetsWithoutOwners []*prom.QueryResult) map[podKey]controllerKey {
+	// Build out set of ReplicaSets that have no owners, themselves, such that
+	// the ReplicaSet should be used as the owner of the Pods it controls.
+	// (This should exclude, for example, ReplicaSets that are controlled by
+	// Deployments, in which case the Deployment should be the Pod's owner.)
+	replicaSets := map[controllerKey]struct{}{}
+
+	for _, res := range resReplicaSetsWithoutOwners {
+		controllerKey, err := resultReplicaSetKey(res, env.GetPromClusterLabel(), "namespace", "replicaset")
+		if err != nil {
+			continue
+		}
+
+		replicaSets[controllerKey] = struct{}{}
+	}
+
+	// Create the mapping of Pods to ReplicaSets, ignoring any ReplicaSets that
+	// to not appear in the set of uncontrolled ReplicaSets above.
+	podToReplicaSet := map[podKey]controllerKey{}
+
+	for _, res := range resPodsWithReplicaSetOwner {
+		controllerKey, err := resultReplicaSetKey(res, env.GetPromClusterLabel(), "namespace", "owner_name")
+		if err != nil {
+			continue
+		}
+		if _, ok := replicaSets[controllerKey]; !ok {
+			continue
+		}
+
+		pod, err := res.GetString("pod")
+		if err != nil {
+			log.Warningf("CostModel.ComputeAllocation: ReplicaSet result without pod: %s", controllerKey)
+		}
+
+		podKey := newPodKey(controllerKey.Cluster, controllerKey.Namespace, pod)
+
+		podToReplicaSet[podKey] = controllerKey
+	}
+
+	return podToReplicaSet
+}
+
 func applyServicesToPods(podMap map[podKey]*Pod, podLabels map[podKey]map[string]string, allocsByService map[serviceKey][]*kubecost.Allocation, serviceLabels map[serviceKey]map[string]string) {
 	podServicesMap := map[podKey][]serviceKey{}
 
@@ -1306,8 +1434,7 @@ func applyControllersToPods(podMap map[podKey]*Pod, podControllerMap map[podKey]
 	}
 }
 
-func applyNodeCostPerCPUHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerCPUHr []*prom.QueryResult,
-	providerIDParser func(string) string) {
+func applyNodeCostPerCPUHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerCPUHr []*prom.QueryResult) {
 	for _, res := range resNodeCostPerCPUHr {
 		cluster, err := res.GetString(env.GetPromClusterLabel())
 		if err != nil {
@@ -1337,7 +1464,7 @@ func applyNodeCostPerCPUHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerCPUHr
 			nodeMap[key] = &NodePricing{
 				Name:       node,
 				NodeType:   instanceType,
-				ProviderID: providerIDParser(providerID),
+				ProviderID: cloud.ParseID(providerID),
 			}
 		}
 
@@ -1345,8 +1472,7 @@ func applyNodeCostPerCPUHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerCPUHr
 	}
 }
 
-func applyNodeCostPerRAMGiBHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerRAMGiBHr []*prom.QueryResult,
-	providerIDParser func(string) string) {
+func applyNodeCostPerRAMGiBHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerRAMGiBHr []*prom.QueryResult) {
 	for _, res := range resNodeCostPerRAMGiBHr {
 		cluster, err := res.GetString(env.GetPromClusterLabel())
 		if err != nil {
@@ -1376,7 +1502,7 @@ func applyNodeCostPerRAMGiBHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerRA
 			nodeMap[key] = &NodePricing{
 				Name:       node,
 				NodeType:   instanceType,
-				ProviderID: providerIDParser(providerID),
+				ProviderID: cloud.ParseID(providerID),
 			}
 		}
 
@@ -1384,8 +1510,7 @@ func applyNodeCostPerRAMGiBHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerRA
 	}
 }
 
-func applyNodeCostPerGPUHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerGPUHr []*prom.QueryResult,
-	providerIDParser func(string) string) {
+func applyNodeCostPerGPUHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerGPUHr []*prom.QueryResult) {
 	for _, res := range resNodeCostPerGPUHr {
 		cluster, err := res.GetString(env.GetPromClusterLabel())
 		if err != nil {
@@ -1415,7 +1540,7 @@ func applyNodeCostPerGPUHr(nodeMap map[nodeKey]*NodePricing, resNodeCostPerGPUHr
 			nodeMap[key] = &NodePricing{
 				Name:       node,
 				NodeType:   instanceType,
-				ProviderID: providerIDParser(providerID),
+				ProviderID: cloud.ParseID(providerID),
 			}
 		}
 

pkg/costmodel/cluster.go

@@ -4,11 +4,12 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/kubecost/cost-model/pkg/util/timeutil"
+
 	"github.com/kubecost/cost-model/pkg/cloud"
 	"github.com/kubecost/cost-model/pkg/env"
 	"github.com/kubecost/cost-model/pkg/log"
 	"github.com/kubecost/cost-model/pkg/prom"
-	"github.com/kubecost/cost-model/pkg/util"
 
 	prometheus "github.com/prometheus/client_golang/api"
 	"k8s.io/klog"
@@ -70,15 +71,12 @@ type ClusterCostsBreakdown struct {
 
 // NewClusterCostsFromCumulative takes cumulative cost data over a given time range, computes
 // the associated monthly rate data, and returns the Costs.
-func NewClusterCostsFromCumulative(cpu, gpu, ram, storage float64, window, offset string, dataHours float64) (*ClusterCosts, error) {
-	start, end, err := util.ParseTimeRange(window, offset)
-	if err != nil {
-		return nil, err
-	}
+func NewClusterCostsFromCumulative(cpu, gpu, ram, storage float64, window, offset time.Duration, dataHours float64) (*ClusterCosts, error) {
+	start, end := timeutil.ParseTimeRange(window, offset)
 
 	// If the number of hours is not given (i.e. is zero) compute one from the window and offset
 	if dataHours == 0 {
-		dataHours = end.Sub(*start).Hours()
+		dataHours = end.Sub(start).Hours()
 	}
 
 	// Do not allow zero-length windows to prevent divide-by-zero issues
@@ -87,17 +85,17 @@ func NewClusterCostsFromCumulative(cpu, gpu, ram, storage float64, window, offse
 	}
 
 	cc := &ClusterCosts{
-		Start:             start,
-		End:               end,
+		Start:             &start,
+		End:               &end,
 		CPUCumulative:     cpu,
 		GPUCumulative:     gpu,
 		RAMCumulative:     ram,
 		StorageCumulative: storage,
 		TotalCumulative:   cpu + gpu + ram + storage,
-		CPUMonthly:        cpu / dataHours * (util.HoursPerMonth),
-		GPUMonthly:        gpu / dataHours * (util.HoursPerMonth),
-		RAMMonthly:        ram / dataHours * (util.HoursPerMonth),
-		StorageMonthly:    storage / dataHours * (util.HoursPerMonth),
+		CPUMonthly:        cpu / dataHours * (timeutil.HoursPerMonth),
+		GPUMonthly:        gpu / dataHours * (timeutil.HoursPerMonth),
+		RAMMonthly:        ram / dataHours * (timeutil.HoursPerMonth),
+		StorageMonthly:    storage / dataHours * (timeutil.HoursPerMonth),
 	}
 	cc.TotalMonthly = cc.CPUMonthly + cc.GPUMonthly + cc.RAMMonthly + cc.StorageMonthly
 
@@ -138,9 +136,9 @@ func ClusterDisks(client prometheus.Client, provider cloud.Provider, duration, o
 	// TODO niko/assets how do we not hard-code this price?
 	costPerGBHr := 0.04 / 730.0
 
-	ctx := prom.NewContext(client)
-	queryPVCost := fmt.Sprintf(`sum_over_time((avg(kube_persistentvolume_capacity_bytes) by (%s, persistentvolume)  * on(%s, persistentvolume) group_right avg(pv_hourly_cost) by (%s, persistentvolume,provider_id))[%s:%dm]%s)/1024/1024/1024 * %f`, env.GetPromClusterLabel(), env.GetPromClusterLabel(), env.GetPromClusterLabel(), durationStr, minsPerResolution, offsetStr, hourlyToCumulative)
-	queryPVSize := fmt.Sprintf(`avg_over_time(kube_persistentvolume_capacity_bytes[%s:%dm]%s)`, durationStr, minsPerResolution, offsetStr)
+	ctx := prom.NewNamedContext(client, prom.ClusterContextName)
+	queryPVCost := fmt.Sprintf(`avg(avg_over_time(pv_hourly_cost[%s]%s)) by (%s, persistentvolume,provider_id)`, durationStr, offsetStr, env.GetPromClusterLabel())
+	queryPVSize := fmt.Sprintf(`avg(avg_over_time(kube_persistentvolume_capacity_bytes[%s]%s)) by (%s, persistentvolume)`, durationStr, offsetStr, env.GetPromClusterLabel())
 	queryActiveMins := fmt.Sprintf(`count(pv_hourly_cost) by (%s, persistentvolume)[%s:%dm]%s`, env.GetPromClusterLabel(), durationStr, minsPerResolution, offsetStr)
 
 	queryLocalStorageCost := fmt.Sprintf(`sum_over_time(sum(container_fs_limit_bytes{device!="tmpfs", id="/"}) by (instance, %s)[%s:%dm]%s) / 1024 / 1024 / 1024 * %f * %f`, env.GetPromClusterLabel(), durationStr, minsPerResolution, offsetStr, hourlyToCumulative, costPerGBHr)
@@ -169,61 +167,7 @@ func ClusterDisks(client prometheus.Client, provider cloud.Provider, duration, o
 
 	diskMap := map[string]*Disk{}
 
-	for _, result := range resPVCost {
-		cluster, err := result.GetString(env.GetPromClusterLabel())
-		if err != nil {
-			cluster = env.GetClusterID()
-		}
-
-		name, err := result.GetString("persistentvolume")
-		if err != nil {
-			log.Warningf("ClusterDisks: PV cost data missing persistentvolume")
-			continue
-		}
-
-		// TODO niko/assets storage class
-
-		cost := result.Values[0].Value
-		key := fmt.Sprintf("%s/%s", cluster, name)
-		if _, ok := diskMap[key]; !ok {
-			diskMap[key] = &Disk{
-				Cluster:   cluster,
-				Name:      name,
-				Breakdown: &ClusterCostsBreakdown{},
-			}
-		}
-		diskMap[key].Cost += cost
-		providerID, _ := result.GetString("provider_id") // just put the providerID set up here, it's the simplest query.
-		if providerID != "" {
-			diskMap[key].ProviderID = provider.ParsePVID(providerID)
-		}
-	}
-
-	for _, result := range resPVSize {
-		cluster, err := result.GetString(env.GetPromClusterLabel())
-		if err != nil {
-			cluster = env.GetClusterID()
-		}
-
-		name, err := result.GetString("persistentvolume")
-		if err != nil {
-			log.Warningf("ClusterDisks: PV size data missing persistentvolume")
-			continue
-		}
-
-		// TODO niko/assets storage class
-
-		bytes := result.Values[0].Value
-		key := fmt.Sprintf("%s/%s", cluster, name)
-		if _, ok := diskMap[key]; !ok {
-			diskMap[key] = &Disk{
-				Cluster:   cluster,
-				Name:      name,
-				Breakdown: &ClusterCostsBreakdown{},
-			}
-		}
-		diskMap[key].Bytes = bytes
-	}
+	pvCosts(diskMap, resolution, resActiveMins, resPVSize, resPVCost)
 
 	for _, result := range resLocalStorageCost {
 		cluster, err := result.GetString(env.GetPromClusterLabel())
@@ -300,39 +244,6 @@ func ClusterDisks(client prometheus.Client, provider cloud.Provider, duration, o
 		diskMap[key].Bytes = bytes
 	}
 
-	for _, result := range resActiveMins {
-		cluster, err := result.GetString(env.GetPromClusterLabel())
-		if err != nil {
-			cluster = env.GetClusterID()
-		}
-
-		name, err := result.GetString("persistentvolume")
-		if err != nil {
-			log.Warningf("ClusterDisks: active mins missing instance")
-			continue
-		}
-
-		key := fmt.Sprintf("%s/%s", cluster, name)
-		if _, ok := diskMap[key]; !ok {
-			log.DedupedWarningf(5, "ClusterDisks: active mins for unidentified disk")
-			continue
-		}
-
-		if len(result.Values) == 0 {
-			continue
-		}
-
-		s := time.Unix(int64(result.Values[0].Timestamp), 0)
-		e := time.Unix(int64(result.Values[len(result.Values)-1].Timestamp), 0).Add(resolution)
-		mins := e.Sub(s).Minutes()
-
-		// TODO niko/assets if mins >= threshold, interpolate for missing data?
-
-		diskMap[key].End = e
-		diskMap[key].Start = s
-		diskMap[key].Minutes = mins
-	}
-
 	for _, result := range resLocalActiveMins {
 		cluster, err := result.GetString(env.GetPromClusterLabel())
 		if err != nil {
@@ -464,8 +375,8 @@ func ClusterNodes(cp cloud.Provider, client prometheus.Client, duration, offset
 	minsPerResolution := 1
 	resolution := time.Duration(minsPerResolution) * time.Minute
 
-	requiredCtx := prom.NewContext(client)
-	optionalCtx := prom.NewContext(client)
+	requiredCtx := prom.NewNamedContext(client, prom.ClusterContextName)
+	optionalCtx := prom.NewNamedContext(client, prom.ClusterOptionalContextName)
 
 	queryNodeCPUHourlyCost := fmt.Sprintf(`avg(avg_over_time(node_cpu_hourly_cost[%s]%s)) by (%s, node, instance_type, provider_id)`, durationStr, offsetStr, env.GetPromClusterLabel())
 	queryNodeCPUCores := fmt.Sprintf(`avg(avg_over_time(kube_node_status_capacity_cpu_cores[%s]%s)) by (%s, node)`, durationStr, offsetStr, env.GetPromClusterLabel())
@@ -522,18 +433,18 @@ func ClusterNodes(cp cloud.Provider, client prometheus.Client, duration, offset
 		return nil, requiredCtx.ErrorCollection()
 	}
 
-	activeDataMap := buildActiveDataMap(resActiveMins, resolution, cp.ParseID)
+	activeDataMap := buildActiveDataMap(resActiveMins, resolution)
 
-	gpuCountMap := buildGPUCountMap(resNodeGPUCount, cp.ParseID)
+	gpuCountMap := buildGPUCountMap(resNodeGPUCount)
 
-	cpuCostMap, clusterAndNameToType1 := buildCPUCostMap(resNodeCPUHourlyCost, cp.ParseID)
-	ramCostMap, clusterAndNameToType2 := buildRAMCostMap(resNodeRAMHourlyCost, cp.ParseID)
-	gpuCostMap, clusterAndNameToType3 := buildGPUCostMap(resNodeGPUHourlyCost, gpuCountMap, cp.ParseID)
+	cpuCostMap, clusterAndNameToType1 := buildCPUCostMap(resNodeCPUHourlyCost)
+	ramCostMap, clusterAndNameToType2 := buildRAMCostMap(resNodeRAMHourlyCost)
+	gpuCostMap, clusterAndNameToType3 := buildGPUCostMap(resNodeGPUHourlyCost, gpuCountMap)
 
 	clusterAndNameToTypeIntermediate := mergeTypeMaps(clusterAndNameToType1, clusterAndNameToType2)
 	clusterAndNameToType := mergeTypeMaps(clusterAndNameToTypeIntermediate, clusterAndNameToType3)
 
-	cpuCoresMap := buildCPUCoresMap(resNodeCPUCores, clusterAndNameToType)
+	cpuCoresMap := buildCPUCoresMap(resNodeCPUCores)
 
 	ramBytesMap := buildRAMBytesMap(resNodeRAMBytes)
 
@@ -541,7 +452,7 @@ func ClusterNodes(cp cloud.Provider, client prometheus.Client, duration, offset
 	ramSystemPctMap := buildRAMSystemPctMap(resNodeRAMSystemPct)
 
 	cpuBreakdownMap := buildCPUBreakdownMap(resNodeCPUModeTotal)
-	preemptibleMap := buildPreemptibleMap(resIsSpot, cp.ParseID)
+	preemptibleMap := buildPreemptibleMap(resIsSpot)
 	labelsMap := buildLabelsMap(resLabels)
 
 	costTimesMinuteAndCount(activeDataMap, cpuCostMap, cpuCoresMap)
@@ -595,7 +506,7 @@ type LoadBalancer struct {
 	Minutes    float64
 }
 
-func ClusterLoadBalancers(cp cloud.Provider, client prometheus.Client, duration, offset time.Duration) (map[string]*LoadBalancer, error) {
+func ClusterLoadBalancers(client prometheus.Client, duration, offset time.Duration) (map[string]*LoadBalancer, error) {
 	durationStr := fmt.Sprintf("%dm", int64(duration.Minutes()))
 	offsetStr := fmt.Sprintf(" offset %dm", int64(offset.Minutes()))
 	if offset < time.Minute {
@@ -612,7 +523,7 @@ func ClusterLoadBalancers(cp cloud.Provider, client prometheus.Client, duration,
 	// [$/hr] * [min/res]*[hr/min] = [$/res]
 	hourlyToCumulative := float64(minsPerResolution) * (1.0 / 60.0)
 
-	ctx := prom.NewContext(client)
+	ctx := prom.NewNamedContext(client, prom.ClusterContextName)
 	queryLBCost := fmt.Sprintf(`sum_over_time((avg(kubecost_load_balancer_cost) by (namespace, service_name, %s, ingress_ip))[%s:%dm]%s) * %f`, env.GetPromClusterLabel(), durationStr, minsPerResolution, offsetStr, hourlyToCumulative)
 	queryActiveMins := fmt.Sprintf(`count(kubecost_load_balancer_cost) by (namespace, service_name, %s, ingress_ip)[%s:%dm]%s`, env.GetPromClusterLabel(), durationStr, minsPerResolution, offsetStr)
 
@@ -655,7 +566,7 @@ func ClusterLoadBalancers(cp cloud.Provider, client prometheus.Client, duration,
 			loadBalancerMap[key] = &LoadBalancer{
 				Cluster:    cluster,
 				Name:       namespace + "/" + serviceName,
-				ProviderID: cp.ParseLBID(providerID),
+				ProviderID: cloud.ParseLBID(providerID),
 			}
 		}
 		// Fill in Provider ID if it is available and missing in the loadBalancerMap
@@ -687,26 +598,26 @@ func ClusterLoadBalancers(cp cloud.Provider, client prometheus.Client, duration,
 			continue
 		}
 
-		s := time.Unix(int64(result.Values[0].Timestamp), 0)
-		e := time.Unix(int64(result.Values[len(result.Values)-1].Timestamp), 0)
-		mins := e.Sub(s).Minutes()
-
-		// TODO niko/assets if mins >= threshold, interpolate for missing data?
+		if lb, ok := loadBalancerMap[key]; ok {
+			s := time.Unix(int64(result.Values[0].Timestamp), 0)
+			e := time.Unix(int64(result.Values[len(result.Values)-1].Timestamp), 0)
+			mins := e.Sub(s).Minutes()
 
-		loadBalancerMap[key].Start = s
-		loadBalancerMap[key].Minutes = mins
+			lb.Start = s
+			lb.Minutes = mins
+		} else {
+			log.DedupedWarningf(20, "ClusterLoadBalancers: found minutes for key that does not exist: %s", key)
+		}
 	}
 	return loadBalancerMap, nil
 }
 
 // ComputeClusterCosts gives the cumulative and monthly-rate cluster costs over a window of time for all clusters.
-func (a *Accesses) ComputeClusterCosts(client prometheus.Client, provider cloud.Provider, window, offset string, withBreakdown bool) (map[string]*ClusterCosts, error) {
+func (a *Accesses) ComputeClusterCosts(client prometheus.Client, provider cloud.Provider, window, offset time.Duration, withBreakdown bool) (map[string]*ClusterCosts, error) {
 	// Compute number of minutes in the full interval, for use interpolating missed scrapes or scaling missing data
-	start, end, err := util.ParseTimeRange(window, offset)
-	if err != nil {
-		return nil, err
-	}
-	mins := end.Sub(*start).Minutes()
+	start, end := timeutil.ParseTimeRange(window, offset)
+
+	mins := end.Sub(start).Minutes()
 
 	// minsPerResolution determines accuracy and resource use for the following
 	// queries. Smaller values (higher resolution) result in better accuracy,
@@ -775,10 +686,7 @@ func (a *Accesses) ComputeClusterCosts(client prometheus.Client, provider cloud.
 		queryTotalLocalStorage = fmt.Sprintf(" + %s", queryTotalLocalStorage)
 	}
 
-	fmtOffset := ""
-	if offset != "" {
-		fmtOffset = fmt.Sprintf("offset %s", offset)
-	}
+	fmtOffset := timeutil.DurationToPromOffsetString(offset)
 
 	queryDataCount := fmt.Sprintf(fmtQueryDataCount, env.GetPromClusterLabel(), window, minsPerResolution, fmtOffset, minsPerResolution)
 	queryTotalGPU := fmt.Sprintf(fmtQueryTotalGPU, window, minsPerResolution, fmtOffset, hourlyToCumulative, env.GetPromClusterLabel())
@@ -786,7 +694,7 @@ func (a *Accesses) ComputeClusterCosts(client prometheus.Client, provider cloud.
 	queryTotalRAM := fmt.Sprintf(fmtQueryTotalRAM, env.GetPromClusterLabel(), window, minsPerResolution, fmtOffset, window, minsPerResolution, fmtOffset, env.GetPromClusterLabel(), hourlyToCumulative, env.GetPromClusterLabel())
 	queryTotalStorage := fmt.Sprintf(fmtQueryTotalStorage, env.GetPromClusterLabel(), window, minsPerResolution, fmtOffset, window, minsPerResolution, fmtOffset, env.GetPromClusterLabel(), hourlyToCumulative, env.GetPromClusterLabel())
 
-	ctx := prom.NewContext(client)
+	ctx := prom.NewNamedContext(client, prom.ClusterContextName)
 
 	resChs := ctx.QueryAll(
 		queryDataCount,
@@ -1001,7 +909,7 @@ func (a *Accesses) ComputeClusterCosts(client prometheus.Client, provider cloud.
 			dataMins = mins
 			klog.V(3).Infof("[Warning] cluster cost data count not found for cluster %s", id)
 		}
-		costs, err := NewClusterCostsFromCumulative(cd["cpu"], cd["gpu"], cd["ram"], cd["storage"]+cd["localstorage"], window, offset, dataMins/util.MinsPerHour)
+		costs, err := NewClusterCostsFromCumulative(cd["cpu"], cd["gpu"], cd["ram"], cd["storage"]+cd["localstorage"], window, offset, dataMins/timeutil.MinsPerHour)
 		if err != nil {
 			klog.V(3).Infof("[Warning] Failed to parse cluster costs on %s (%s) from cumulative data: %+v", window, offset, cd)
 			return nil, err
@@ -1052,8 +960,8 @@ func resultToTotals(qrs []*prom.QueryResult) ([][]string, error) {
 }
 
 // ClusterCostsOverTime gives the full cluster costs over time
-func ClusterCostsOverTime(cli prometheus.Client, provider cloud.Provider, startString, endString, windowString, offset string) (*Totals, error) {
-	localStorageQuery := provider.GetLocalStorageQuery(windowString, offset, true, false)
+func ClusterCostsOverTime(cli prometheus.Client, provider cloud.Provider, startString, endString string, window, offset time.Duration) (*Totals, error) {
+	localStorageQuery := provider.GetLocalStorageQuery(window, offset, true, false)
 	if localStorageQuery != "" {
 		localStorageQuery = fmt.Sprintf("+ %s", localStorageQuery)
 	}
@@ -1062,31 +970,30 @@ func ClusterCostsOverTime(cli prometheus.Client, provider cloud.Provider, startS
 
 	start, err := time.Parse(layout, startString)
 	if err != nil {
-		klog.V(1).Infof("Error parsing time " + startString + ". Error: " + err.Error())
+		klog.V(1).Infof("Error parsing time %s. Error: %s", startString, err.Error())
 		return nil, err
 	}
 	end, err := time.Parse(layout, endString)
 	if err != nil {
-		klog.V(1).Infof("Error parsing time " + endString + ". Error: " + err.Error())
+		klog.V(1).Infof("Error parsing time %s. Error: %s", endString, err.Error())
 		return nil, err
 	}
-	window, err := time.ParseDuration(windowString)
-	if err != nil {
-		klog.V(1).Infof("Error parsing time " + windowString + ". Error: " + err.Error())
+	fmtWindow := timeutil.DurationString(window)
+
+	if fmtWindow == "" {
+		err := fmt.Errorf("window value invalid or missing")
+		klog.V(1).Infof("Error parsing time %v. Error: %s", window, err.Error())
 		return nil, err
 	}
 
-	// turn offsets of the format "[0-9+]h" into the format "offset [0-9+]h" for use in query templatess
-	if offset != "" {
-		offset = fmt.Sprintf("offset %s", offset)
-	}
+	fmtOffset := timeutil.DurationToPromOffsetString(offset)
 
-	qCores := fmt.Sprintf(queryClusterCores, windowString, offset, env.GetPromClusterLabel(), windowString, offset, env.GetPromClusterLabel(), windowString, offset, env.GetPromClusterLabel(), env.GetPromClusterLabel())
-	qRAM := fmt.Sprintf(queryClusterRAM, windowString, offset, env.GetPromClusterLabel(), windowString, offset, env.GetPromClusterLabel(), env.GetPromClusterLabel())
-	qStorage := fmt.Sprintf(queryStorage, windowString, offset, env.GetPromClusterLabel(), windowString, offset, env.GetPromClusterLabel(), env.GetPromClusterLabel(), localStorageQuery)
+	qCores := fmt.Sprintf(queryClusterCores, fmtWindow, fmtOffset, env.GetPromClusterLabel(), fmtWindow, fmtOffset, env.GetPromClusterLabel(), fmtWindow, fmtOffset, env.GetPromClusterLabel(), env.GetPromClusterLabel())
+	qRAM := fmt.Sprintf(queryClusterRAM, fmtWindow, fmtOffset, env.GetPromClusterLabel(), fmtWindow, fmtOffset, env.GetPromClusterLabel(), env.GetPromClusterLabel())
+	qStorage := fmt.Sprintf(queryStorage, fmtWindow, fmtOffset, env.GetPromClusterLabel(), fmtWindow, fmtOffset, env.GetPromClusterLabel(), env.GetPromClusterLabel(), localStorageQuery)
 	qTotal := fmt.Sprintf(queryTotal, env.GetPromClusterLabel(), env.GetPromClusterLabel(), env.GetPromClusterLabel(), env.GetPromClusterLabel(), localStorageQuery)
 
-	ctx := prom.NewContext(cli)
+	ctx := prom.NewNamedContext(cli, prom.ClusterContextName)
 	resChClusterCores := ctx.QueryRange(qCores, start, end, window)
 	resChClusterRAM := ctx.QueryRange(qRAM, start, end, window)
 	resChStorage := ctx.QueryRange(qStorage, start, end, window)
@@ -1158,3 +1065,96 @@ func ClusterCostsOverTime(cli prometheus.Client, provider cloud.Provider, startS
 		StorageCost: storageTotal,
 	}, nil
 }
+
+func pvCosts(diskMap map[string]*Disk, resolution time.Duration, resActiveMins, resPVSize, resPVCost []*prom.QueryResult) {
+	for _, result := range resActiveMins {
+		cluster, err := result.GetString(env.GetPromClusterLabel())
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		name, err := result.GetString("persistentvolume")
+		if err != nil {
+			log.Warningf("ClusterDisks: active mins missing pv name")
+			continue
+		}
+
+		if len(result.Values) == 0 {
+			continue
+		}
+
+		key := fmt.Sprintf("%s/%s", cluster, name)
+		if _, ok := diskMap[key]; !ok {
+			diskMap[key] = &Disk{
+				Cluster:   cluster,
+				Name:      name,
+				Breakdown: &ClusterCostsBreakdown{},
+			}
+		}
+		s := time.Unix(int64(result.Values[0].Timestamp), 0)
+		e := time.Unix(int64(result.Values[len(result.Values)-1].Timestamp), 0).Add(resolution)
+		mins := e.Sub(s).Minutes()
+
+		// TODO niko/assets if mins >= threshold, interpolate for missing data?
+
+		diskMap[key].End = e
+		diskMap[key].Start = s
+		diskMap[key].Minutes = mins
+	}
+
+	for _, result := range resPVSize {
+		cluster, err := result.GetString(env.GetPromClusterLabel())
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		name, err := result.GetString("persistentvolume")
+		if err != nil {
+			log.Warningf("ClusterDisks: PV size data missing persistentvolume")
+			continue
+		}
+
+		// TODO niko/assets storage class
+
+		bytes := result.Values[0].Value
+		key := fmt.Sprintf("%s/%s", cluster, name)
+		if _, ok := diskMap[key]; !ok {
+			diskMap[key] = &Disk{
+				Cluster:   cluster,
+				Name:      name,
+				Breakdown: &ClusterCostsBreakdown{},
+			}
+		}
+		diskMap[key].Bytes = bytes
+	}
+
+	for _, result := range resPVCost {
+		cluster, err := result.GetString(env.GetPromClusterLabel())
+		if err != nil {
+			cluster = env.GetClusterID()
+		}
+
+		name, err := result.GetString("persistentvolume")
+		if err != nil {
+			log.Warningf("ClusterDisks: PV cost data missing persistentvolume")
+			continue
+		}
+
+		// TODO niko/assets storage class
+
+		cost := result.Values[0].Value
+		key := fmt.Sprintf("%s/%s", cluster, name)
+		if _, ok := diskMap[key]; !ok {
+			diskMap[key] = &Disk{
+				Cluster:   cluster,
+				Name:      name,
+				Breakdown: &ClusterCostsBreakdown{},
+			}
+		}
+		diskMap[key].Cost = cost * (diskMap[key].Bytes / 1024 / 1024 / 1024) * (diskMap[key].Minutes / 60)
+		providerID, _ := result.GetString("provider_id") // just put the providerID set up here, it's the simplest query.
+		if providerID != "" {
+			diskMap[key].ProviderID = cloud.ParsePVID(providerID)
+		}
+	}
+}

pkg/costmodel/cluster_helpers.go

@@ -1,6 +1,7 @@
 package costmodel
 
 import (
+	"github.com/kubecost/cost-model/pkg/cloud"
 	"time"
 
 	"github.com/kubecost/cost-model/pkg/env"
@@ -27,7 +28,6 @@ func mergeTypeMaps(clusterAndNameToType1, clusterAndNameToType2 map[nodeIdentifi
 
 func buildCPUCostMap(
 	resNodeCPUCost []*prom.QueryResult,
-	providerIDParser func(string) string,
 ) (
 	map[NodeIdentifier]float64,
 	map[nodeIdentifierNoProviderID]string,
@@ -56,7 +56,7 @@ func buildCPUCostMap(
 		key := NodeIdentifier{
 			Cluster:    cluster,
 			Name:       name,
-			ProviderID: providerIDParser(providerID),
+			ProviderID: cloud.ParseID(providerID),
 		}
 		keyNon := nodeIdentifierNoProviderID{
 			Cluster: cluster,
@@ -73,7 +73,6 @@ func buildCPUCostMap(
 
 func buildRAMCostMap(
 	resNodeRAMCost []*prom.QueryResult,
-	providerIDParser func(string) string,
 ) (
 	map[NodeIdentifier]float64,
 	map[nodeIdentifierNoProviderID]string,
@@ -102,7 +101,7 @@ func buildRAMCostMap(
 		key := NodeIdentifier{
 			Cluster:    cluster,
 			Name:       name,
-			ProviderID: providerIDParser(providerID),
+			ProviderID: cloud.ParseID(providerID),
 		}
 		keyNon := nodeIdentifierNoProviderID{
 			Cluster: cluster,
@@ -119,7 +118,6 @@ func buildRAMCostMap(
 func buildGPUCostMap(
 	resNodeGPUCost []*prom.QueryResult,
 	gpuCountMap map[NodeIdentifier]float64,
-	providerIDParser func(string) string,
 ) (
 	map[NodeIdentifier]float64,
 	map[nodeIdentifierNoProviderID]string,
@@ -148,7 +146,7 @@ func buildGPUCostMap(
 		key := NodeIdentifier{
 			Cluster:    cluster,
 			Name:       name,
-			ProviderID: providerIDParser(providerID),
+			ProviderID: cloud.ParseID(providerID),
 		}
 		keyNon := nodeIdentifierNoProviderID{
 			Cluster: cluster,
@@ -171,7 +169,6 @@ func buildGPUCostMap(
 
 func buildGPUCountMap(
 	resNodeGPUCount []*prom.QueryResult,
-	providerIDParser func(string) string,
 ) map[NodeIdentifier]float64 {
 
 	gpuCountMap := make(map[NodeIdentifier]float64)
@@ -194,7 +191,7 @@ func buildGPUCountMap(
 		key := NodeIdentifier{
 			Cluster:    cluster,
 			Name:       name,
-			ProviderID: providerIDParser(providerID),
+			ProviderID: cloud.ParseID(providerID),
 		}
 		gpuCountMap[key] = gpuCount
 	}
@@ -204,7 +201,6 @@ func buildGPUCountMap(
 
 func buildCPUCoresMap(
 	resNodeCPUCores []*prom.QueryResult,
-	clusterAndNameToType map[nodeIdentifierNoProviderID]string,
 ) map[nodeIdentifierNoProviderID]float64 {
 
 	m := make(map[nodeIdentifierNoProviderID]float64)
@@ -403,7 +399,7 @@ type activeData struct {
 	minutes float64
 }
 
-func buildActiveDataMap(resActiveMins []*prom.QueryResult, resolution time.Duration, providerIDParser func(string) string) map[NodeIdentifier]activeData {
+func buildActiveDataMap(resActiveMins []*prom.QueryResult, resolution time.Duration) map[NodeIdentifier]activeData {
 
 	m := make(map[NodeIdentifier]activeData)
 
@@ -424,7 +420,7 @@ func buildActiveDataMap(resActiveMins []*prom.QueryResult, resolution time.Durat
 		key := NodeIdentifier{
 			Cluster:    cluster,
 			Name:       name,
-			ProviderID: providerIDParser(providerID),
+			ProviderID: cloud.ParseID(providerID),
 		}
 
 		if len(result.Values) == 0 {
@@ -450,7 +446,6 @@ func buildActiveDataMap(resActiveMins []*prom.QueryResult, resolution time.Durat
 // node id -> is preemptible?
 func buildPreemptibleMap(
 	resIsSpot []*prom.QueryResult,
-	providerIDParser func(string) string,
 ) map[NodeIdentifier]bool {
 
 	m := make(map[NodeIdentifier]bool)
@@ -474,7 +469,7 @@ func buildPreemptibleMap(
 		key := NodeIdentifier{
 			Cluster:    cluster,
 			Name:       nodeName,
-			ProviderID: providerIDParser(providerID),
+			ProviderID: cloud.ParseID(providerID),
 		}
 
 		// TODO(michaelmdresser): check this condition at merge time?
@@ -503,7 +498,7 @@ func buildLabelsMap(
 		if err != nil {
 			cluster = env.GetClusterID()
 		}
-		node, err := result.GetString("kubernetes_node")
+		node, err := result.GetString("node")
 		if err != nil {
 			log.DedupedWarningf(5, "ClusterNodes: label data missing node")
 			continue

pkg/costmodel/cluster_helpers_test.go

@@ -699,7 +699,6 @@ func TestBuildNodeMap(t *testing.T) {
 }
 
 func TestBuildGPUCostMap(t *testing.T) {
-	providerIDParser := func(s string) string { return s }
 	cases := []struct {
 		name       string
 		promResult []*prom.QueryResult
@@ -850,7 +849,7 @@ func TestBuildGPUCostMap(t *testing.T) {
 
 	for _, testCase := range cases {
 		t.Run(testCase.name, func(t *testing.T) {
-			result, _ := buildGPUCostMap(testCase.promResult, testCase.countMap, providerIDParser)
+			result, _ := buildGPUCostMap(testCase.promResult, testCase.countMap)
 			if !reflect.DeepEqual(result, testCase.expected) {
 				t.Errorf("buildGPUCostMap case %s failed. Got %+v but expected %+v", testCase.name, result, testCase.expected)
 			}

pkg/costmodel/clusterinfo.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	cloudProvider "github.com/kubecost/cost-model/pkg/cloud"
+	"github.com/kubecost/cost-model/pkg/costmodel/clusters"
 	"github.com/kubecost/cost-model/pkg/env"
 	"github.com/kubecost/cost-model/pkg/thanos"
 
@@ -40,6 +41,27 @@ func writeThanosFlags(clusterInfo map[string]string) {
 	}
 }
 
+// default local cluster info provider implementation which provides an instanced object for
+// getting the local cluster info
+type defaultLocalClusterInfoProvider struct {
+	k8s      kubernetes.Interface
+	provider cloudProvider.Provider
+}
+
+// GetClusterInfo returns a string map containing the local cluster info
+func (dlcip *defaultLocalClusterInfoProvider) GetClusterInfo() map[string]string {
+	return GetClusterInfo(dlcip.k8s, dlcip.provider)
+}
+
+// NewLocalClusterInfoProvider creates a new clusters.LocalClusterInfoProvider implementation for providing local
+// cluster information
+func NewLocalClusterInfoProvider(k8s kubernetes.Interface, cloud cloudProvider.Provider) clusters.LocalClusterInfoProvider {
+	return &defaultLocalClusterInfoProvider{
+		k8s:      k8s,
+		provider: cloud,
+	}
+}
+
 // GetClusterInfo provides specific information about the cluster cloud provider as well as
 // generic configuration values.
 func GetClusterInfo(kubeClient kubernetes.Interface, cloud cloudProvider.Provider) map[string]string {

pkg/costmodel/clusters/clustermap.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/kubecost/cost-model/pkg/env"
 	"github.com/kubecost/cost-model/pkg/log"
 	"github.com/kubecost/cost-model/pkg/prom"
 	"github.com/kubecost/cost-model/pkg/thanos"
@@ -68,23 +69,31 @@ type ClusterMap interface {
 	StopRefresh()
 }
 
+// LocalClusterInfoProvider is a contract which is capable of performing local cluster info lookups.
+type LocalClusterInfoProvider interface {
+	// GetClusterInfo returns a string map containing the local cluster info
+	GetClusterInfo() map[string]string
+}
+
 // ClusterMap keeps records of all known cost-model clusters.
 type PrometheusClusterMap struct {
-	lock     *sync.RWMutex
-	client   prometheus.Client
-	clusters map[string]*ClusterInfo
-	stop     chan struct{}
+	lock         *sync.RWMutex
+	client       prometheus.Client
+	clusters     map[string]*ClusterInfo
+	localCluster LocalClusterInfoProvider
+	stop         chan struct{}
 }
 
 // NewClusterMap creates a new ClusterMap implementation using a prometheus or thanos client
-func NewClusterMap(client prometheus.Client, refresh time.Duration) ClusterMap {
+func NewClusterMap(client prometheus.Client, lcip LocalClusterInfoProvider, refresh time.Duration) ClusterMap {
 	stop := make(chan struct{})
 
 	cm := &PrometheusClusterMap{
-		lock:     new(sync.RWMutex),
-		client:   client,
-		clusters: make(map[string]*ClusterInfo),
-		stop:     stop,
+		lock:         new(sync.RWMutex),
+		client:       client,
+		clusters:     make(map[string]*ClusterInfo),
+		localCluster: lcip,
+		stop:         stop,
 	}
 
 	// Run an updater to ensure cluster data stays relevant over time
@@ -122,7 +131,7 @@ func (pcm *PrometheusClusterMap) loadClusters() (map[string]*ClusterInfo, error)
 
 	// Execute Query
 	tryQuery := func() (interface{}, error) {
-		ctx := prom.NewContext(pcm.client)
+		ctx := prom.NewNamedContext(pcm.client, prom.ClusterMapContextName)
 		r, _, e := ctx.QuerySync(clusterInfoQuery(offset))
 		return r, e
 	}
@@ -175,9 +184,61 @@ func (pcm *PrometheusClusterMap) loadClusters() (map[string]*ClusterInfo, error)
 		}
 	}
 
+	// populate the local cluster if it doesn't exist
+	localID := env.GetClusterID()
+	if _, ok := clusters[localID]; !ok {
+		localInfo, err := pcm.getLocalClusterInfo()
+		if err != nil {
+			log.Warningf("Failed to load local cluster info: %s", err)
+		} else {
+			clusters[localInfo.ID] = localInfo
+		}
+	}
+
 	return clusters, nil
 }
 
+// getLocalClusterInfo returns the local cluster info in the event there does not exist a metric available.
+func (pcm *PrometheusClusterMap) getLocalClusterInfo() (*ClusterInfo, error) {
+	info := pcm.localCluster.GetClusterInfo()
+
+	var id string
+	var name string
+
+	if i, ok := info["id"]; ok {
+		id = i
+	} else {
+		return nil, fmt.Errorf("Local Cluster Info Missing ID")
+	}
+	if n, ok := info["name"]; ok {
+		name = n
+	} else {
+		return nil, fmt.Errorf("Local Cluster Info Missing Name")
+	}
+
+	var clusterProfile string
+	var provider string
+	var provisioner string
+
+	if cp, ok := info["clusterProfile"]; ok {
+		clusterProfile = cp
+	}
+	if pvdr, ok := info["provider"]; ok {
+		provider = pvdr
+	}
+	if pvsr, ok := info["provisioner"]; ok {
+		provisioner = pvsr
+	}
+
+	return &ClusterInfo{
+		ID:          id,
+		Name:        name,
+		Profile:     clusterProfile,
+		Provider:    provider,
+		Provisioner: provisioner,
+	}, nil
+}
+
 // refreshClusters loads the clusters and updates the internal map
 func (pcm *PrometheusClusterMap) refreshClusters() {
 	updated, err := pcm.loadClusters()

pkg/costmodel/costmodel.go

@@ -233,8 +233,6 @@ const (
 func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, cp costAnalyzerCloud.Provider, window string, offset string, filterNamespace string) (map[string]*CostData, error) {
 	queryRAMUsage := fmt.Sprintf(queryRAMUsageStr, window, offset, window, offset, env.GetPromClusterLabel())
 	queryCPUUsage := fmt.Sprintf(queryCPUUsageStr, window, offset, env.GetPromClusterLabel())
-	queryGPURequests := fmt.Sprintf(queryGPURequestsStr, window, offset, window, offset, 1.0, env.GetPromClusterLabel(), env.GetPromClusterLabel(), env.GetPromClusterLabel(), window, offset, env.GetPromClusterLabel())
-	queryPVRequests := fmt.Sprintf(queryPVRequestsStr, env.GetPromClusterLabel(), env.GetPromClusterLabel(), env.GetPromClusterLabel(), env.GetPromClusterLabel())
 	queryNetZoneRequests := fmt.Sprintf(queryZoneNetworkUsage, window, "", env.GetPromClusterLabel())
 	queryNetRegionRequests := fmt.Sprintf(queryRegionNetworkUsage, window, "", env.GetPromClusterLabel())
 	queryNetInternetRequests := fmt.Sprintf(queryInternetNetworkUsage, window, "", env.GetPromClusterLabel())
@@ -244,11 +242,9 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, cp costAnalyze
 	clusterID := env.GetClusterID()
 
 	// Submit all Prometheus queries asynchronously
-	ctx := prom.NewContext(cli)
+	ctx := prom.NewNamedContext(cli, prom.ComputeCostDataContextName)
 	resChRAMUsage := ctx.Query(queryRAMUsage)
 	resChCPUUsage := ctx.Query(queryCPUUsage)
-	resChGPURequests := ctx.Query(queryGPURequests)
-	resChPVRequests := ctx.Query(queryPVRequests)
 	resChNetZoneRequests := ctx.Query(queryNetZoneRequests)
 	resChNetRegionRequests := ctx.Query(queryNetRegionRequests)
 	resChNetInternetRequests := ctx.Query(queryNetInternetRequests)
@@ -280,8 +276,6 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, cp costAnalyze
 	// Process Prometheus query results. Handle errors using ctx.Errors.
 	resRAMUsage, _ := resChRAMUsage.Await()
 	resCPUUsage, _ := resChCPUUsage.Await()
-	resGPURequests, _ := resChGPURequests.Await()
-	resPVRequests, _ := resChPVRequests.Await()
 	resNetZoneRequests, _ := resChNetZoneRequests.Await()
 	resNetRegionRequests, _ := resChNetRegionRequests.Await()
 	resNetInternetRequests, _ := resChNetInternetRequests.Await()
@@ -314,6 +308,14 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, cp costAnalyze
 		log.Warningf("ComputeCostData: continuing despite error parsing normalization values from %s: %s", queryNormalization, err.Error())
 	}
 
+	// Determine if there are vgpus configured and if so get the total allocatable number
+	// If there are no vgpus, the coefficient is set to 1.0
+	vgpuCount, err := getAllocatableVGPUs(cm.Cache)
+	vgpuCoeff := 10.0
+	if vgpuCount > 0.0 {
+		vgpuCoeff = vgpuCount
+	}
+
 	nodes, err := cm.GetNodeCost(cp)
 	if err != nil {
 		log.Warningf("GetNodeCost: no node cost model available: " + err.Error())
@@ -322,7 +324,7 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, cp costAnalyze
 
 	// Unmounted PVs represent the PVs that are not mounted or tied to a volume on a container
 	unmountedPVs := make(map[string][]*PersistentVolumeClaimData)
-	pvClaimMapping, err := GetPVInfo(resPVRequests, clusterID)
+	pvClaimMapping, err := GetPVInfoLocal(cm.Cache, clusterID)
 	if err != nil {
 		log.Warningf("GetPVInfo: unable to get PV data: %s", err.Error())
 	}
@@ -353,13 +355,6 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, cp costAnalyze
 	for key := range RAMUsedMap {
 		containers[key] = true
 	}
-	GPUReqMap, err := GetContainerMetricVector(resGPURequests, true, normalizationValue, clusterID)
-	if err != nil {
-		return nil, err
-	}
-	for key := range GPUReqMap {
-		containers[key] = true
-	}
 	CPUUsedMap, err := GetContainerMetricVector(resCPUUsage, false, 0, clusterID) // No need to normalize here, as this comes from a counter
 	if err != nil {
 		return nil, err
@@ -504,17 +499,30 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, cp costAnalyze
 					},
 				}
 
+				gpuReqCount := 0.0
+				if g, ok := container.Resources.Requests["nvidia.com/gpu"]; ok {
+					gpuReqCount = g.AsApproximateFloat64()
+				} else if g, ok := container.Resources.Limits["nvidia.com/gpu"]; ok {
+					gpuReqCount = g.AsApproximateFloat64()
+				} else if g, ok := container.Resources.Requests["k8s.amazonaws.com/vgpu"]; ok {
+					// divide vgpu request/limits by total vgpus to get the portion of physical gpus requested
+					gpuReqCount = g.AsApproximateFloat64() / vgpuCoeff
+				} else if g, ok := container.Resources.Limits["k8s.amazonaws.com/vgpu"]; ok {
+					gpuReqCount = g.AsApproximateFloat64() / vgpuCoeff
+				}
+				GPUReqV := []*util.Vector{
+					{
+						Value:     float64(gpuReqCount),
+						Timestamp: float64(time.Now().UTC().Unix()),
+					},
+				}
+
 				RAMUsedV, ok := RAMUsedMap[newKey]
 				if !ok {
 					klog.V(4).Info("no RAM usage for " + newKey)
 					RAMUsedV = []*util.Vector{{}}
 				}
 
-				GPUReqV, ok := GPUReqMap[newKey]
-				if !ok {
-					klog.V(4).Info("no GPU requests for " + newKey)
-					GPUReqV = []*util.Vector{{}}
-				}
 				CPUUsedV, ok := CPUUsedMap[newKey]
 				if !ok {
 					klog.V(4).Info("no CPU usage for " + newKey)
@@ -576,6 +584,7 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, cp costAnalyze
 			// with very short-lived pods that over-request resources.
 			RAMReqV := []*util.Vector{{}}
 			CPUReqV := []*util.Vector{{}}
+			GPUReqV := []*util.Vector{{}}
 
 			RAMUsedV, ok := RAMUsedMap[key]
 			if !ok {
@@ -583,11 +592,6 @@ func (cm *CostModel) ComputeCostData(cli prometheusClient.Client, cp costAnalyze
 				RAMUsedV = []*util.Vector{{}}
 			}
 
-			GPUReqV, ok := GPUReqMap[key]
-			if !ok {
-				klog.V(4).Info("no GPU requests for " + key)
-				GPUReqV = []*util.Vector{{}}
-			}
 			CPUUsedV, ok := CPUUsedMap[key]
 			if !ok {
 				klog.V(4).Info("no CPU usage for " + key)
@@ -710,7 +714,7 @@ func findDeletedPodInfo(cli prometheusClient.Client, missingContainers map[strin
 	if len(missingContainers) > 0 {
 		queryHistoricalPodLabels := fmt.Sprintf(`kube_pod_labels{}[%s]`, window)
 
-		podLabelsResult, _, err := prom.NewContext(cli).QuerySync(queryHistoricalPodLabels)
+		podLabelsResult, _, err := prom.NewNamedContext(cli, prom.ComputeCostDataContextName).QuerySync(queryHistoricalPodLabels)
 		if err != nil {
 			log.Errorf("failed to parse historical pod labels: %s", err.Error())
 		}
@@ -751,7 +755,7 @@ func findDeletedNodeInfo(cli prometheusClient.Client, missingNodes map[string]*c
 		queryHistoricalRAMCost := fmt.Sprintf(`avg(avg_over_time(node_ram_hourly_cost[%s] %s)) by (node, instance, %s)`, window, offsetStr, env.GetPromClusterLabel())
 		queryHistoricalGPUCost := fmt.Sprintf(`avg(avg_over_time(node_gpu_hourly_cost[%s] %s)) by (node, instance, %s)`, window, offsetStr, env.GetPromClusterLabel())
 
-		ctx := prom.NewContext(cli)
+		ctx := prom.NewNamedContext(cli, prom.ComputeCostDataContextName)
 		cpuCostResCh := ctx.Query(queryHistoricalCPUCost)
 		ramCostResCh := ctx.Query(queryHistoricalRAMCost)
 		gpuCostResCh := ctx.Query(queryHistoricalGPUCost)
@@ -930,6 +934,12 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 	nodeList := cm.Cache.GetAllNodes()
 	nodes := make(map[string]*costAnalyzerCloud.Node)
 
+	vgpuCount, err := getAllocatableVGPUs(cm.Cache)
+	vgpuCoeff := 10.0
+	if vgpuCount > 0.0 {
+		vgpuCoeff = vgpuCount
+	}
+
 	pmd := &costAnalyzerCloud.PricingMatchMetadata{
 		TotalNodes:        0,
 		PricingTypeCounts: make(map[costAnalyzerCloud.PricingType]int),
@@ -1010,6 +1020,12 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 				newCnode.GPU = fmt.Sprintf("%d", q.Value())
 				gpuc = float64(gpuCount)
 			}
+		} else if g, ok := n.Status.Capacity["k8s.amazonaws.com/vgpu"]; ok {
+			gpuCount := g.Value()
+			if gpuCount != 0 {
+				newCnode.GPU = fmt.Sprintf("%d", int(float64(q.Value())/vgpuCoeff))
+				gpuc = float64(gpuCount) / vgpuCoeff
+			}
 		} else {
 			gpuc, err = strconv.ParseFloat(newCnode.GPU, 64)
 			if err != nil {
@@ -1087,7 +1103,7 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 					return nil, err
 				}
 			} else {
-				nodePrice, err = strconv.ParseFloat(newCnode.VCPUCost, 64) // all the price was allocated the the CPU
+				nodePrice, err = strconv.ParseFloat(newCnode.VCPUCost, 64) // all the price was allocated to the CPU
 				if err != nil {
 					klog.V(3).Infof("Could not parse node vcpu price")
 					return nil, err
@@ -1161,7 +1177,7 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 					return nil, err
 				}
 			} else {
-				nodePrice, err = strconv.ParseFloat(newCnode.VCPUCost, 64) // all the price was allocated the the CPU
+				nodePrice, err = strconv.ParseFloat(newCnode.VCPUCost, 64) // all the price was allocated to the CPU
 				if err != nil {
 					klog.V(3).Infof("Could not parse node vcpu price")
 					return nil, err
@@ -1567,7 +1583,7 @@ func (cm *CostModel) costDataRange(cli prometheusClient.Client, cp costAnalyzerC
 
 	scrapeIntervalSeconds := cm.ScrapeInterval.Seconds()
 
-	ctx := prom.NewContext(cli)
+	ctx := prom.NewNamedContext(cli, prom.ComputeCostDataRangeContextName)
 
 	queryRAMAlloc := fmt.Sprintf(queryRAMAllocationByteHours, resStr, env.GetPromClusterLabel(), scrapeIntervalSeconds)
 	queryCPUAlloc := fmt.Sprintf(queryCPUAllocationVCPUHours, resStr, env.GetPromClusterLabel(), scrapeIntervalSeconds)
@@ -2189,6 +2205,31 @@ func getStatefulSetsOfPod(pod v1.Pod) []string {
 	return []string{}
 }
 
+func getAllocatableVGPUs(cache clustercache.ClusterCache) (float64, error) {
+	daemonsets := cache.GetAllDaemonSets()
+	vgpuCount := 0.0
+	for _, ds := range daemonsets {
+		dsContainerList := &ds.Spec.Template.Spec.Containers
+		for _, ctnr := range *dsContainerList {
+			if ctnr.Args != nil {
+				for _, arg := range ctnr.Args {
+					if strings.Contains(arg, "--vgpu=") {
+						vgpus, err := strconv.ParseFloat(arg[strings.IndexByte(arg, '=')+1:], 64)
+						if err != nil {
+							klog.V(1).Infof("failed to parse vgpu allocation string %s: %v", arg, err)
+							continue
+						}
+						vgpuCount = vgpus
+						return vgpuCount, nil
+					}
+
+				}
+			}
+		}
+	}
+	return vgpuCount, nil
+}
+
 type PersistentVolumeClaimData struct {
 	Class        string                `json:"class"`
 	Claim        string                `json:"claim"`

pkg/costmodel/key.go

@@ -64,9 +64,8 @@ func resultContainerKey(res *prom.QueryResult, clusterLabel, namespaceLabel, pod
 }
 
 type podKey struct {
-	Cluster   string
-	Namespace string
-	Pod       string
+	namespaceKey
+	Pod string
 }
 
 func (k podKey) String() string {
@@ -75,9 +74,11 @@ func (k podKey) String() string {
 
 func newPodKey(cluster, namespace, pod string) podKey {
 	return podKey{
-		Cluster:   cluster,
-		Namespace: namespace,
-		Pod:       pod,
+		namespaceKey: namespaceKey{
+			Cluster:   cluster,
+			Namespace: namespace,
+		},
+		Pod: pod,
 	}
 }
 
@@ -87,7 +88,7 @@ func newPodKey(cluster, namespace, pod string) podKey {
 // as the podKey's Cluster field. If a given field does not exist on the
 // result, an error is returned. (The only exception to that is clusterLabel,
 // which we expect may not exist, but has a default value.)
-func resultPodKey(res *prom.QueryResult, clusterLabel, namespaceLabel, podLabel string) (podKey, error) {
+func resultPodKey(res *prom.QueryResult, clusterLabel, namespaceLabel string) (podKey, error) {
 	key := podKey{}
 
 	cluster, err := res.GetString(clusterLabel)
@@ -102,9 +103,12 @@ func resultPodKey(res *prom.QueryResult, clusterLabel, namespaceLabel, podLabel
 	}
 	key.Namespace = namespace
 
-	pod, err := res.GetString(podLabel)
-	if err != nil {
-		return key, err
+	pod, err := res.GetString("pod")
+	if pod == "" || err != nil {
+		pod, err = res.GetString("pod_name")
+		if err != nil {
+			return key, err
+		}
 	}
 	key.Pod = pod
 
@@ -209,24 +213,30 @@ func resultDeploymentKey(res *prom.QueryResult, clusterLabel, namespaceLabel, co
 	return resultControllerKey("deployment", res, clusterLabel, namespaceLabel, controllerLabel)
 }
 
-// resultDeploymentKey creates a controllerKey for a StatefulSet.
+// resultStatefulSetKey creates a controllerKey for a StatefulSet.
 // (See resultControllerKey for more.)
 func resultStatefulSetKey(res *prom.QueryResult, clusterLabel, namespaceLabel, controllerLabel string) (controllerKey, error) {
 	return resultControllerKey("statefulset", res, clusterLabel, namespaceLabel, controllerLabel)
 }
 
-// resultDeploymentKey creates a controllerKey for a DaemonSet.
+// resultDaemonSetKey creates a controllerKey for a DaemonSet.
 // (See resultControllerKey for more.)
 func resultDaemonSetKey(res *prom.QueryResult, clusterLabel, namespaceLabel, controllerLabel string) (controllerKey, error) {
 	return resultControllerKey("daemonset", res, clusterLabel, namespaceLabel, controllerLabel)
 }
 
-// resultDeploymentKey creates a controllerKey for a Job.
+// resultJobKey creates a controllerKey for a Job.
 // (See resultControllerKey for more.)
 func resultJobKey(res *prom.QueryResult, clusterLabel, namespaceLabel, controllerLabel string) (controllerKey, error) {
 	return resultControllerKey("job", res, clusterLabel, namespaceLabel, controllerLabel)
 }
 
+// resultReplicaSetKey creates a controllerKey for a Job.
+// (See resultControllerKey for more.)
+func resultReplicaSetKey(res *prom.QueryResult, clusterLabel, namespaceLabel, controllerLabel string) (controllerKey, error) {
+	return resultControllerKey("replicaset", res, clusterLabel, namespaceLabel, controllerLabel)
+}
+
 type serviceKey struct {
 	Cluster   string
 	Namespace string

pkg/costmodel/promparsers.go

@@ -3,34 +3,45 @@ package costmodel
 import (
 	"errors"
 	"fmt"
+	"time"
 
 	costAnalyzerCloud "github.com/kubecost/cost-model/pkg/cloud"
+	"github.com/kubecost/cost-model/pkg/clustercache"
 	"github.com/kubecost/cost-model/pkg/env"
 	"github.com/kubecost/cost-model/pkg/log"
 	"github.com/kubecost/cost-model/pkg/prom"
 	"github.com/kubecost/cost-model/pkg/util"
-	"gopkg.in/yaml.v2"
 )
 
-const DEFAULT_KUBECOST_JOB_NAME = "kubecost"
-
-type ScrapeConfig struct {
-	JobName        string `yaml:"job_name,omitempty"`
-	ScrapeInterval string `yaml:"scrape_interval,omitempty"`
-}
-
-type PromCfg struct {
-	ScrapeConfigs []ScrapeConfig `yaml:"scrape_configs,omitempty"`
-}
-
-func GetPrometheusConfig(pcfg string) (PromCfg, error) {
-	var promCfg PromCfg
-	err := yaml.Unmarshal([]byte(pcfg), &promCfg)
-	return promCfg, err
-}
+func GetPVInfoLocal(cache clustercache.ClusterCache, defaultClusterID string) (map[string]*PersistentVolumeClaimData, error) {
+	toReturn := make(map[string]*PersistentVolumeClaimData)
 
-func GetKubecostJobName() string {
-	return DEFAULT_KUBECOST_JOB_NAME // TODO: look this up from a prometheus variable?
+	pvcs := cache.GetAllPersistentVolumeClaims()
+	for _, pvc := range pvcs {
+		var vals []*util.Vector
+		vals = append(vals, &util.Vector{
+			Timestamp: float64(time.Now().Unix()),
+			Value:     float64(pvc.Spec.Resources.Requests.Storage().Value()),
+		})
+		ns := pvc.Namespace
+		pvcName := pvc.Name
+		volumeName := pvc.Spec.VolumeName
+		pvClass := ""
+		if pvc.Spec.StorageClassName != nil {
+			pvClass = *pvc.Spec.StorageClassName
+		}
+		clusterID := defaultClusterID
+		key := fmt.Sprintf("%s,%s,%s", ns, pvcName, clusterID)
+		toReturn[key] = &PersistentVolumeClaimData{
+			Class:      pvClass,
+			Claim:      pvcName,
+			Namespace:  ns,
+			ClusterID:  clusterID,
+			VolumeName: volumeName,
+			Values:     vals,
+		}
+	}
+	return toReturn, nil
 }
 
 // TODO niko/prom move parsing functions from costmodel.go

pkg/costmodel/router.go

@@ -11,6 +11,11 @@ import (
 	"sync"
 	"time"
 
+	"github.com/kubecost/cost-model/pkg/services"
+	"github.com/kubecost/cost-model/pkg/util/httputil"
+	"github.com/kubecost/cost-model/pkg/util/timeutil"
+	"github.com/kubecost/cost-model/pkg/util/watcher"
+
 	"k8s.io/klog"
 
 	"github.com/julienschmidt/httprouter"
@@ -19,7 +24,6 @@ import (
 
 	"github.com/kubecost/cost-model/pkg/cloud"
 	"github.com/kubecost/cost-model/pkg/clustercache"
-	cm "github.com/kubecost/cost-model/pkg/clustermanager"
 	"github.com/kubecost/cost-model/pkg/costmodel/clusters"
 	"github.com/kubecost/cost-model/pkg/env"
 	"github.com/kubecost/cost-model/pkg/errors"
@@ -29,9 +33,7 @@ import (
 	"github.com/kubecost/cost-model/pkg/thanos"
 	"github.com/kubecost/cost-model/pkg/util/json"
 	prometheus "github.com/prometheus/client_golang/api"
-	prometheusClient "github.com/prometheus/client_golang/api"
 	prometheusAPI "github.com/prometheus/client_golang/api/prometheus/v1"
-	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/patrickmn/go-cache"
@@ -42,14 +44,13 @@ import (
 )
 
 const (
-	prometheusTroubleshootingEp = "http://docs.kubecost.com/custom-prom#troubleshoot"
-	RFC3339Milli                = "2006-01-02T15:04:05.000Z"
-	maxCacheMinutes1d           = 11
-	maxCacheMinutes2d           = 17
-	maxCacheMinutes7d           = 37
-	maxCacheMinutes30d          = 137
-	CustomPricingSetting        = "CustomPricing"
-	DiscountSetting             = "Discount"
+	RFC3339Milli         = "2006-01-02T15:04:05.000Z"
+	maxCacheMinutes1d    = 11
+	maxCacheMinutes2d    = 17
+	maxCacheMinutes7d    = 37
+	maxCacheMinutes30d   = 137
+	CustomPricingSetting = "CustomPricing"
+	DiscountSetting      = "Discount"
 )
 
 var (
@@ -61,10 +62,9 @@ var (
 // Prometheus, Kubernetes, the cloud provider, and caches.
 type Accesses struct {
 	Router            *httprouter.Router
-	PrometheusClient  prometheusClient.Client
-	ThanosClient      prometheusClient.Client
+	PrometheusClient  prometheus.Client
+	ThanosClient      prometheus.Client
 	KubeClientSet     kubernetes.Interface
-	ClusterManager    *cm.ClusterManager
 	ClusterMap        clusters.ClusterMap
 	CloudProvider     cloud.Provider
 	Model             *CostModel
@@ -81,13 +81,15 @@ type Accesses struct {
 	// settings will be published in a pub/sub model
 	settingsSubscribers map[string][]chan string
 	settingsMutex       sync.Mutex
+	// registered http service instances
+	httpServices services.HTTPServices
 }
 
 // GetPrometheusClient decides whether the default Prometheus client or the Thanos client
 // should be used.
-func (a *Accesses) GetPrometheusClient(remote bool) prometheusClient.Client {
+func (a *Accesses) GetPrometheusClient(remote bool) prometheus.Client {
 	// Use Thanos Client if it exists (enabled) and remote flag set
-	var pc prometheusClient.Client
+	var pc prometheus.Client
 
 	if remote && a.ThanosClient != nil {
 		pc = a.ThanosClient
@@ -123,16 +125,18 @@ func (a *Accesses) GetCacheRefresh(dur time.Duration) time.Duration {
 func (a *Accesses) ClusterCostsFromCacheHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")
 
+	duration := 24 * time.Hour
+	offset := time.Minute
 	durationHrs := "24h"
-	offset := "1m"
+	fmtOffset := "1m"
 	pClient := a.GetPrometheusClient(true)
 
-	key := fmt.Sprintf("%s:%s", durationHrs, offset)
+	key := fmt.Sprintf("%s:%s", durationHrs, fmtOffset)
 	if data, valid := a.ClusterCostsCache.Get(key); valid {
 		clusterCosts := data.(map[string]*ClusterCosts)
 		w.Write(WrapDataWithMessage(clusterCosts, nil, "clusterCosts cache hit"))
 	} else {
-		data, err := a.ComputeClusterCosts(pClient, a.CloudProvider, durationHrs, offset, true)
+		data, err := a.ComputeClusterCosts(pClient, a.CloudProvider, duration, offset, true)
 		w.Write(WrapDataWithMessage(data, err, fmt.Sprintf("clusterCosts cache miss: %s", key)))
 	}
 }
@@ -224,7 +228,7 @@ func normalizeTimeParam(param string) (string, error) {
 	return param, nil
 }
 
-// parsePercentString takes a string of expected format "N%" and returns a floating point 0.0N.
+// ParsePercentString takes a string of expected format "N%" and returns a floating point 0.0N.
 // If the "%" symbol is missing, it just returns 0.0N. Empty string is interpreted as "0%" and
 // return 0.0.
 func ParsePercentString(percentStr string) (float64, error) {
@@ -243,66 +247,6 @@ func ParsePercentString(percentStr string) (float64, error) {
 	return discount, nil
 }
 
-// parseDuration converts a Prometheus-style duration string into a Duration
-// TODO:CLEANUP delete this. do it now.
-func ParseDuration(duration string) (*time.Duration, error) {
-	unitStr := duration[len(duration)-1:]
-	var unit time.Duration
-	switch unitStr {
-	case "s":
-		unit = time.Second
-	case "m":
-		unit = time.Minute
-	case "h":
-		unit = time.Hour
-	case "d":
-		unit = 24.0 * time.Hour
-	default:
-		return nil, fmt.Errorf("error parsing duration: %s did not match expected format [0-9+](s|m|d|h)", duration)
-	}
-
-	amountStr := duration[:len(duration)-1]
-	amount, err := strconv.ParseInt(amountStr, 10, 64)
-	if err != nil {
-		return nil, fmt.Errorf("error parsing duration: %s did not match expected format [0-9+](s|m|d|h)", duration)
-	}
-
-	dur := time.Duration(amount) * unit
-	return &dur, nil
-}
-
-// ParseTimeRange returns a start and end time, respectively, which are converted from
-// a duration and offset, defined as strings with Prometheus-style syntax.
-func ParseTimeRange(duration, offset string) (*time.Time, *time.Time, error) {
-	// endTime defaults to the current time, unless an offset is explicity declared,
-	// in which case it shifts endTime back by given duration
-	endTime := time.Now()
-	if offset != "" {
-		o, err := ParseDuration(offset)
-		if err != nil {
-			return nil, nil, fmt.Errorf("error parsing offset (%s): %s", offset, err)
-		}
-		endTime = endTime.Add(-1 * *o)
-	}
-
-	// if duration is defined in terms of days, convert to hours
-	// e.g. convert "2d" to "48h"
-	durationNorm, err := normalizeTimeParam(duration)
-	if err != nil {
-		return nil, nil, fmt.Errorf("error parsing duration (%s): %s", duration, err)
-	}
-
-	// convert time duration into start and end times, formatted
-	// as ISO datetime strings
-	dur, err := time.ParseDuration(durationNorm)
-	if err != nil {
-		return nil, nil, fmt.Errorf("errorf parsing duration (%s): %s", durationNorm, err)
-	}
-	startTime := endTime.Add(-1 * dur)
-
-	return &startTime, &endTime, nil
-}
-
 func WrapData(data interface{}, err error) []byte {
 	var resp []byte
 
@@ -438,6 +382,26 @@ func (a *Accesses) ClusterCosts(w http.ResponseWriter, r *http.Request, ps httpr
 	window := r.URL.Query().Get("window")
 	offset := r.URL.Query().Get("offset")
 
+	if window == "" {
+		w.Write(WrapData(nil, fmt.Errorf("missing window arguement")))
+		return
+	}
+	windowDur, err := timeutil.ParseDuration(window)
+	if err != nil {
+		w.Write(WrapData(nil, fmt.Errorf("error parsing window (%s): %s", window, err)))
+		return
+	}
+
+	// offset is not a required parameter
+	var offsetDur time.Duration
+	if offset != "" {
+		offsetDur, err = timeutil.ParseDuration(offset)
+		if err != nil {
+			w.Write(WrapData(nil, fmt.Errorf("error parsing offset (%s): %s", offset, err)))
+			return
+		}
+	}
+
 	useThanos, _ := strconv.ParseBool(r.URL.Query().Get("multi"))
 
 	if useThanos && !thanos.IsEnabled() {
@@ -445,15 +409,16 @@ func (a *Accesses) ClusterCosts(w http.ResponseWriter, r *http.Request, ps httpr
 		return
 	}
 
-	var client prometheusClient.Client
+	var client prometheus.Client
 	if useThanos {
 		client = a.ThanosClient
-		offset = thanos.Offset()
+		offsetDur = thanos.OffsetDuration()
+
 	} else {
 		client = a.PrometheusClient
 	}
 
-	data, err := a.ComputeClusterCosts(client, a.CloudProvider, window, offset, true)
+	data, err := a.ComputeClusterCosts(client, a.CloudProvider, windowDur, offsetDur, true)
 	w.Write(WrapData(data, err))
 }
 
@@ -466,7 +431,27 @@ func (a *Accesses) ClusterCostsOverTime(w http.ResponseWriter, r *http.Request,
 	window := r.URL.Query().Get("window")
 	offset := r.URL.Query().Get("offset")
 
-	data, err := ClusterCostsOverTime(a.PrometheusClient, a.CloudProvider, start, end, window, offset)
+	if window == "" {
+		w.Write(WrapData(nil, fmt.Errorf("missing window arguement")))
+		return
+	}
+	windowDur, err := timeutil.ParseDuration(window)
+	if err != nil {
+		w.Write(WrapData(nil, fmt.Errorf("error parsing window (%s): %s", window, err)))
+		return
+	}
+
+	// offset is not a required parameter
+	var offsetDur time.Duration
+	if offset != "" {
+		offsetDur, err = timeutil.ParseDuration(offset)
+		if err != nil {
+			w.Write(WrapData(nil, fmt.Errorf("error parsing offset (%s): %s", offset, err)))
+			return
+		}
+	}
+
+	data, err := ClusterCostsOverTime(a.PrometheusClient, a.CloudProvider, start, end, windowDur, offsetDur)
 	w.Write(WrapData(data, err))
 }
 
@@ -507,7 +492,7 @@ func (a *Accesses) CostDataModelRange(w http.ResponseWriter, r *http.Request, ps
 	}
 
 	// Use Thanos Client if it exists (enabled) and remote flag set
-	var pClient prometheusClient.Client
+	var pClient prometheus.Client
 	if remote != "false" && a.ThanosClient != nil {
 		pClient = a.ThanosClient
 	} else {
@@ -737,38 +722,192 @@ func (a *Accesses) GetPrometheusMetadata(w http.ResponseWriter, _ *http.Request,
 	w.Write(WrapData(prom.Validate(a.PrometheusClient)))
 }
 
-// Creates a new ClusterManager instance using a boltdb storage. If that fails,
-// then we fall back to a memory-only storage.
-func newClusterManager() *cm.ClusterManager {
-	clustersConfigFile := "/var/configs/clusters/default-clusters.yaml"
+func (a *Accesses) PrometheusQuery(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
 
-	// Return a memory-backed cluster manager populated by configmap
-	return cm.NewConfiguredClusterManager(cm.NewMapDBClusterStorage(), clustersConfigFile)
+	qp := httputil.NewQueryParams(r.URL.Query())
+	query := qp.Get("query", "")
+	if query == "" {
+		w.Write(WrapData(nil, fmt.Errorf("Query Parameter 'query' is unset'")))
+		return
+	}
 
-	// NOTE: The following should be used with a persistent disk store. Since the
-	// NOTE: configmap approach is currently the "persistent" source (entries are read-only
-	// NOTE: on the backend), we don't currently need to store on disk.
-	/*
-		path := env.GetConfigPath()
-		db, err := bolt.Open(path+"costmodel.db", 0600, nil)
-		if err != nil {
-			klog.V(1).Infof("[Error] Failed to create costmodel.db: %s", err.Error())
-			return cm.NewConfiguredClusterManager(cm.NewMapDBClusterStorage(), clustersConfigFile)
-		}
+	ctx := prom.NewNamedContext(a.PrometheusClient, prom.FrontendContextName)
+	body, err := ctx.RawQuery(query)
+	if err != nil {
+		w.Write(WrapData(nil, fmt.Errorf("Error running query %s. Error: %s", query, err)))
+		return
+	}
+
+	w.Write(body)
+}
+
+func (a *Accesses) PrometheusQueryRange(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+
+	qp := httputil.NewQueryParams(r.URL.Query())
+	query := qp.Get("query", "")
+	if query == "" {
+		fmt.Fprintf(w, "Error parsing query from request parameters.")
+		return
+	}
+
+	start, end, duration, err := toStartEndStep(qp)
+	if err != nil {
+		fmt.Fprintf(w, err.Error())
+		return
+	}
+
+	ctx := prom.NewNamedContext(a.PrometheusClient, prom.FrontendContextName)
+	body, err := ctx.RawQueryRange(query, start, end, duration)
+	if err != nil {
+		fmt.Fprintf(w, "Error running query %s. Error: %s", query, err)
+		return
+	}
+
+	w.Write(body)
+}
 
-		store, err := cm.NewBoltDBClusterStorage("clusters", db)
+func (a *Accesses) ThanosQuery(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+
+	if !thanos.IsEnabled() {
+		w.Write(WrapData(nil, fmt.Errorf("ThanosDisabled")))
+		return
+	}
+
+	qp := httputil.NewQueryParams(r.URL.Query())
+	query := qp.Get("query", "")
+	if query == "" {
+		w.Write(WrapData(nil, fmt.Errorf("Query Parameter 'query' is unset'")))
+		return
+	}
+
+	ctx := prom.NewNamedContext(a.ThanosClient, prom.FrontendContextName)
+	body, err := ctx.RawQuery(query)
+	if err != nil {
+		w.Write(WrapData(nil, fmt.Errorf("Error running query %s. Error: %s", query, err)))
+		return
+	}
+
+	w.Write(body)
+}
+
+func (a *Accesses) ThanosQueryRange(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+
+	if !thanos.IsEnabled() {
+		w.Write(WrapData(nil, fmt.Errorf("ThanosDisabled")))
+		return
+	}
+
+	qp := httputil.NewQueryParams(r.URL.Query())
+	query := qp.Get("query", "")
+	if query == "" {
+		fmt.Fprintf(w, "Error parsing query from request parameters.")
+		return
+	}
+
+	start, end, duration, err := toStartEndStep(qp)
+	if err != nil {
+		fmt.Fprintf(w, err.Error())
+		return
+	}
+
+	ctx := prom.NewNamedContext(a.ThanosClient, prom.FrontendContextName)
+	body, err := ctx.RawQueryRange(query, start, end, duration)
+	if err != nil {
+		fmt.Fprintf(w, "Error running query %s. Error: %s", query, err)
+		return
+	}
+
+	w.Write(body)
+}
+
+// helper for query range proxy requests
+func toStartEndStep(qp httputil.QueryParams) (start, end time.Time, step time.Duration, err error) {
+	var e error
+
+	ss := qp.Get("start", "")
+	es := qp.Get("end", "")
+	ds := qp.Get("duration", "")
+	layout := "2006-01-02T15:04:05.000Z"
+
+	start, e = time.Parse(layout, ss)
+	if e != nil {
+		err = fmt.Errorf("Error parsing time %s. Error: %s", ss, err)
+		return
+	}
+	end, e = time.Parse(layout, es)
+	if e != nil {
+		err = fmt.Errorf("Error parsing time %s. Error: %s", es, err)
+		return
+	}
+	step, e = time.ParseDuration(ds)
+	if e != nil {
+		err = fmt.Errorf("Error parsing duration %s. Error: %s", ds, err)
+		return
+	}
+	err = nil
+
+	return
+}
+
+func (a *Accesses) GetPrometheusQueueState(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+
+	promQueueState, err := prom.GetPrometheusQueueState(a.PrometheusClient)
+	if err != nil {
+		w.Write(WrapData(nil, err))
+		return
+	}
+
+	result := map[string]*prom.PrometheusQueueState{
+		"prometheus": promQueueState,
+	}
+
+	if thanos.IsEnabled() {
+		thanosQueueState, err := prom.GetPrometheusQueueState(a.ThanosClient)
 		if err != nil {
-			klog.V(1).Infof("[Error] Failed to Create Cluster Storage: %s", err.Error())
-			return cm.NewConfiguredClusterManager(cm.NewMapDBClusterStorage(), clustersConfigFile)
+			log.Warningf("Error getting Thanos queue state: %s", err)
+		} else {
+			result["thanos"] = thanosQueueState
 		}
+	}
 
-		return cm.NewConfiguredClusterManager(store, clustersConfigFile)
-	*/
+	w.Write(WrapData(result, nil))
 }
 
-type ConfigWatchers struct {
-	ConfigmapName string
-	WatchFunc     func(string, map[string]string) error
+// GetPrometheusMetrics retrieves availability of Prometheus and Thanos metrics
+func (a *Accesses) GetPrometheusMetrics(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+
+	promMetrics, err := prom.GetPrometheusMetrics(a.PrometheusClient, "")
+	if err != nil {
+		w.Write(WrapData(nil, err))
+		return
+	}
+
+	result := map[string][]*prom.PrometheusDiagnostic{
+		"prometheus": promMetrics,
+	}
+
+	if thanos.IsEnabled() {
+		thanosMetrics, err := prom.GetPrometheusMetrics(a.ThanosClient, thanos.QueryOffset())
+		if err != nil {
+			log.Warningf("Error getting Thanos queue state: %s", err)
+		} else {
+			result["thanos"] = thanosMetrics
+		}
+	}
+
+	w.Write(WrapData(result, nil))
 }
 
 // captures the panic event in sentry
@@ -801,12 +940,14 @@ func handlePanic(p errors.Panic) bool {
 	return p.Type == errors.PanicTypeHTTP
 }
 
-func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
+func Initialize(additionalConfigWatchers ...*watcher.ConfigMapWatcher) *Accesses {
 	klog.InitFlags(nil)
 	flag.Set("v", "3")
 	flag.Parse()
 	klog.V(1).Infof("Starting cost-model (git commit \"%s\")", env.GetAppVersion())
 
+	configWatchers := watcher.NewConfigMapWatchers(additionalConfigWatchers...)
+
 	var err error
 	if errorReportingEnabled {
 		err = sentry.Init(sentry.ClientOptions{Release: env.GetAppVersion()})
@@ -830,56 +971,40 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 
 	timeout := 120 * time.Second
 	keepAlive := 120 * time.Second
-	scrapeInterval, _ := time.ParseDuration("1m")
-
-	promCli, _ := prom.NewPrometheusClient(address, timeout, keepAlive, queryConcurrency, "")
+	scrapeInterval := time.Minute
 
-	api := prometheusAPI.NewAPI(promCli)
-	pcfg, err := api.Config(context.Background())
+	promCli, err := prom.NewPrometheusClient(address, timeout, keepAlive, queryConcurrency, "")
 	if err != nil {
-		klog.Infof("No valid prometheus config file at %s. Error: %s . Troubleshooting help available at: %s. Ignore if using cortex/thanos here.", address, err.Error(), prometheusTroubleshootingEp)
-	} else {
-		klog.V(1).Info("Retrieved a prometheus config file from: " + address)
-		sc, err := GetPrometheusConfig(pcfg.YAML)
-		if err != nil {
-			klog.Infof("Fix YAML error %s", err)
-		}
-		for _, scrapeconfig := range sc.ScrapeConfigs {
-			if scrapeconfig.JobName == GetKubecostJobName() {
-				if scrapeconfig.ScrapeInterval != "" {
-					si := scrapeconfig.ScrapeInterval
-					sid, err := time.ParseDuration(si)
-					if err != nil {
-						klog.Infof("error parseing scrapeConfig for %s", scrapeconfig.JobName)
-					} else {
-						klog.Infof("Found Kubecost job scrape interval of: %s", si)
-						scrapeInterval = sid
-					}
-				}
-			}
-		}
+		klog.Fatalf("Failed to create prometheus client, Error: %v", err)
 	}
-	klog.Infof("Using scrape interval of %f", scrapeInterval.Seconds())
 
 	m, err := prom.Validate(promCli)
-	if err != nil || m.Running == false {
-		if err != nil {
-			klog.Errorf("Failed to query prometheus at %s. Error: %s . Troubleshooting help available at: %s", address, err.Error(), prometheusTroubleshootingEp)
-		} else if m.Running == false {
-			klog.Errorf("Prometheus at %s is not running. Troubleshooting help available at: %s", address, prometheusTroubleshootingEp)
-		}
-
-		api := prometheusAPI.NewAPI(promCli)
-		_, err = api.Config(context.Background())
+	if err != nil || !m.Running {
 		if err != nil {
-			klog.Infof("No valid prometheus config file at %s. Error: %s . Troubleshooting help available at: %s. Ignore if using cortex/thanos here.", address, err.Error(), prometheusTroubleshootingEp)
-		} else {
-			klog.V(1).Info("Retrieved a prometheus config file from: " + address)
+			klog.Errorf("Failed to query prometheus at %s. Error: %s . Troubleshooting help available at: %s", address, err.Error(), prom.PrometheusTroubleshootingURL)
+		} else if !m.Running {
+			klog.Errorf("Prometheus at %s is not running. Troubleshooting help available at: %s", address, prom.PrometheusTroubleshootingURL)
 		}
 	} else {
 		klog.V(1).Info("Success: retrieved the 'up' query against prometheus at: " + address)
 	}
 
+	api := prometheusAPI.NewAPI(promCli)
+	_, err = api.Config(context.Background())
+	if err != nil {
+		klog.Infof("No valid prometheus config file at %s. Error: %s . Troubleshooting help available at: %s. Ignore if using cortex/thanos here.", address, err.Error(), prom.PrometheusTroubleshootingURL)
+	} else {
+		klog.Infof("Retrieved a prometheus config file from: %s", address)
+	}
+
+	// Lookup scrape interval for kubecost job, update if found
+	si, err := prom.ScrapeIntervalFor(promCli, env.GetKubecostJobName())
+	if err == nil {
+		scrapeInterval = si
+	}
+
+	klog.Infof("Using scrape interval of %f", scrapeInterval.Seconds())
+
 	// Kubernetes API setup
 	var kc *rest.Config
 	if kubeconfig := env.GetKubeConfigPath(); kubeconfig != "" {
@@ -906,52 +1031,25 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 		panic(err.Error())
 	}
 
-	watchConfigFunc := func(c interface{}) {
-		conf := c.(*v1.ConfigMap)
-		if conf.GetName() == "pricing-configs" {
-			_, err := cloudProvider.UpdateConfigFromConfigMap(conf.Data)
-			if err != nil {
-				klog.Infof("ERROR UPDATING %s CONFIG: %s", "pricing-configs", err.Error())
-			}
-		}
-		for _, cw := range additionalConfigWatchers {
-			if conf.GetName() == cw.ConfigmapName {
-				err := cw.WatchFunc(conf.GetName(), conf.Data)
-				if err != nil {
-					klog.Infof("ERROR UPDATING %s CONFIG: %s", cw.ConfigmapName, err.Error())
-				}
-			}
-		}
-	}
+	// Append the pricing config watcher
+	configWatchers.AddWatcher(cloud.ConfigWatcherFor(cloudProvider))
+	watchConfigFunc := configWatchers.ToWatchFunc()
+	watchedConfigs := configWatchers.GetWatchedConfigs()
 
 	kubecostNamespace := env.GetKubecostNamespace()
 	// We need an initial invocation because the init of the cache has happened before we had access to the provider.
-	configs, err := kubeClientset.CoreV1().ConfigMaps(kubecostNamespace).Get(context.Background(), "pricing-configs", metav1.GetOptions{})
-	if err != nil {
-		klog.Infof("No %s configmap found at installtime, using existing configs: %s", "pricing-configs", err.Error())
-	} else {
-		watchConfigFunc(configs)
-	}
-
-	for _, cw := range additionalConfigWatchers {
-		configs, err := kubeClientset.CoreV1().ConfigMaps(kubecostNamespace).Get(context.Background(), cw.ConfigmapName, metav1.GetOptions{})
+	for _, cw := range watchedConfigs {
+		configs, err := kubeClientset.CoreV1().ConfigMaps(kubecostNamespace).Get(context.Background(), cw, metav1.GetOptions{})
 		if err != nil {
-			klog.Infof("No %s configmap found at installtime, using existing configs: %s", cw.ConfigmapName, err.Error())
+			klog.Infof("No %s configmap found at install time, using existing configs: %s", cw, err.Error())
 		} else {
+			klog.Infof("Found configmap %s, watching...", configs.Name)
 			watchConfigFunc(configs)
 		}
 	}
 
 	k8sCache.SetConfigMapUpdateFunc(watchConfigFunc)
 
-	// TODO: General Architecture Note: Several passes have been made to modularize a lot of
-	// TODO: our code, but the router still continues to be the obvious entry point for new \
-	// TODO: features. We should look to split out the actual "router" functionality and
-	// TODO: implement a builder -> controller for stitching new features and other dependencies.
-	clusterManager := newClusterManager()
-
-	// Initialize metrics here
-
 	remoteEnabled := env.IsRemoteEnabled()
 	if remoteEnabled {
 		info, err := cloudProvider.ClusterInfo()
@@ -966,7 +1064,7 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 	}
 
 	// Thanos Client
-	var thanosClient prometheusClient.Client
+	var thanosClient prometheus.Client
 	if thanos.IsEnabled() {
 		thanosAddress := thanos.QueryURL()
 
@@ -990,10 +1088,11 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 
 	// Initialize ClusterMap for maintaining ClusterInfo by ClusterID
 	var clusterMap clusters.ClusterMap
+	localCIProvider := NewLocalClusterInfoProvider(kubeClientset, cloudProvider)
 	if thanosClient != nil {
-		clusterMap = clusters.NewClusterMap(thanosClient, 10*time.Minute)
+		clusterMap = clusters.NewClusterMap(thanosClient, localCIProvider, 10*time.Minute)
 	} else {
-		clusterMap = clusters.NewClusterMap(promCli, 5*time.Minute)
+		clusterMap = clusters.NewClusterMap(promCli, localCIProvider, 5*time.Minute)
 	}
 
 	// cache responses from model and aggregation for a default of 10 minutes;
@@ -1029,7 +1128,6 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 		PrometheusClient:  promCli,
 		ThanosClient:      thanosClient,
 		KubeClientSet:     kubeClientset,
-		ClusterManager:    clusterManager,
 		ClusterMap:        clusterMap,
 		CloudProvider:     cloudProvider,
 		Model:             costModel,
@@ -1040,6 +1138,7 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 		OutOfClusterCache: outOfClusterCache,
 		SettingsCache:     settingsCache,
 		CacheExpiration:   cacheExpiration,
+		httpServices:      services.NewCostModelServices(),
 	}
 	// Use the Accesses instance, itself, as the CostModelAggregator. This is
 	// confusing and unconventional, but necessary so that we can swap it
@@ -1063,9 +1162,9 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 		log.Infof("Init: AggregateCostModel cache warming disabled")
 	}
 
-	a.MetricsEmitter.Start()
-
-	managerEndpoints := cm.NewClusterManagerEndpoints(a.ClusterManager)
+	if !env.IsKubecostMetricsPodEnabled() {
+		a.MetricsEmitter.Start()
+	}
 
 	a.Router.GET("/costDataModel", a.CostDataModel)
 	a.Router.GET("/costDataModelRange", a.CostDataModelRange)
@@ -1085,10 +1184,17 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) *Accesses {
 	a.Router.GET("/pricingSourceStatus", a.GetPricingSourceStatus)
 	a.Router.GET("/pricingSourceCounts", a.GetPricingSourceCounts)
 
-	// cluster manager endpoints
-	a.Router.GET("/clusters", managerEndpoints.GetAllClusters)
-	a.Router.PUT("/clusters", managerEndpoints.PutCluster)
-	a.Router.DELETE("/clusters/:id", managerEndpoints.DeleteCluster)
+	// prom query proxies
+	a.Router.GET("/prometheusQuery", a.PrometheusQuery)
+	a.Router.GET("/prometheusQueryRange", a.PrometheusQueryRange)
+	a.Router.GET("/thanosQuery", a.ThanosQuery)
+	a.Router.GET("/thanosQueryRange", a.ThanosQueryRange)
+
+	// diagnostics
+	a.Router.GET("/diagnostics/requestQueue", a.GetPrometheusQueueState)
+	a.Router.GET("/diagnostics/prometheusMetrics", a.GetPrometheusMetrics)
+
+	a.httpServices.RegisterAll(a.Router)
 
 	return a
 }

pkg/costmodel/settings.go

@@ -89,8 +89,8 @@ func (a *Accesses) customPricingHasChanged() bool {
 	// describe parameters by which we determine whether or not custom
 	// pricing settings have changed
 	encodeCustomPricing := func(cp *cloud.CustomPricing) string {
-		return fmt.Sprintf("%s:%s:%s:%s:%s:%s:%s:%s:%+v", cp.CustomPricesEnabled, cp.CPU, cp.SpotCPU,
-			cp.RAM, cp.SpotRAM, cp.GPU, cp.Storage, cp.CurrencyCode, cp.SharedCosts)
+		return fmt.Sprintf("%s:%s:%s:%s:%s:%s:%s:%s:%s", cp.CustomPricesEnabled, cp.CPU, cp.SpotCPU,
+			cp.RAM, cp.SpotRAM, cp.GPU, cp.Storage, cp.CurrencyCode, cp.SharedOverhead)
 	}
 
 	// compare cached custom pricing parameters with current values

pkg/env/costmodelenv.go

@@ -71,12 +71,19 @@ const (
 	LegacyExternalAPIDisabledVar = "LEGACY_EXTERNAL_API_DISABLED"
 
 	PromClusterIDLabelEnvVar = "PROM_CLUSTER_ID_LABEL"
+
+	PricingConfigmapName  = "PRICING_CONFIGMAP_NAME"
+	KubecostJobNameEnvVar = "KUBECOST_JOB_NAME"
 )
 
+func GetPricingConfigmapName() string {
+	return Get(PricingConfigmapName, "pricing-configs")
+}
+
 // GetAWSAccessKeyID returns the environment variable value for AWSAccessKeyIDEnvVar which represents
 // the AWS access key for authentication
 func GetAppVersion() string {
-	return Get(AppVersionEnvVar, "1.82.0")
+	return Get(AppVersionEnvVar, "1.87.0")
 }
 
 // IsEmitNamespaceAnnotationsMetric returns true if cost-model is configured to emit the kube_namespace_annotations metric
@@ -349,6 +356,11 @@ func GetParsedUTCOffset() time.Duration {
 	return offset
 }
 
+// GetKubecostJobName returns the environment variable value for KubecostJobNameEnvVar
+func GetKubecostJobName() string {
+	return Get(KubecostJobNameEnvVar, "kubecost")
+}
+
 func IsCacheWarmingEnabled() bool {
 	return GetBool(CacheWarmingEnabledEnvVar, true)
 }

pkg/env/kubemetricsenv.go

@@ -0,0 +1,15 @@
+package env
+
+const (
+	KubecostMetricsPodEnabledEnvVar = "KUBECOST_METRICS_POD_ENABLED"
+	KubecostMetricsPodPortEnvVar    = "KUBECOST_METRICS_PORT"
+)
+
+func GetKubecostMetricsPort() int {
+	return GetInt(KubecostMetricsPodPortEnvVar, 9005)
+}
+
+// IsKubecostMetricsPodEnabled returns true if the kubecost metrics pod is deployed
+func IsKubecostMetricsPodEnabled() bool {
+	return GetBool(KubecostMetricsPodEnabledEnvVar, false)
+}

pkg/kubecost/allocation.go

@@ -63,6 +63,8 @@ type Allocation struct {
 	GPUHours                   float64               `json:"gpuHours"`
 	GPUCost                    float64               `json:"gpuCost"`
 	GPUCostAdjustment          float64               `json:"gpuCostAdjustment"`
+	NetworkTransferBytes       float64               `json:"networkTransferBytes"`
+	NetworkReceiveBytes        float64               `json:"networkReceiveBytes"`
 	NetworkCost                float64               `json:"networkCost"`
 	NetworkCostAdjustment      float64               `json:"networkCostAdjustment"`
 	LoadBalancerCost           float64               `json:"loadBalancerCost"`
@@ -205,6 +207,8 @@ func (a *Allocation) Clone() *Allocation {
 		GPUHours:                   a.GPUHours,
 		GPUCost:                    a.GPUCost,
 		GPUCostAdjustment:          a.GPUCostAdjustment,
+		NetworkTransferBytes:       a.NetworkTransferBytes,
+		NetworkReceiveBytes:        a.NetworkReceiveBytes,
 		NetworkCost:                a.NetworkCost,
 		NetworkCostAdjustment:      a.NetworkCostAdjustment,
 		LoadBalancerCost:           a.LoadBalancerCost,
@@ -276,6 +280,12 @@ func (a *Allocation) Equal(that *Allocation) bool {
 	if !util.IsApproximately(a.GPUCostAdjustment, that.GPUCostAdjustment) {
 		return false
 	}
+	if !util.IsApproximately(a.NetworkTransferBytes, that.NetworkTransferBytes) {
+		return false
+	}
+	if !util.IsApproximately(a.NetworkReceiveBytes, that.NetworkReceiveBytes) {
+		return false
+	}
 	if !util.IsApproximately(a.NetworkCost, that.NetworkCost) {
 		return false
 	}
@@ -500,6 +510,8 @@ func (a *Allocation) MarshalJSON() ([]byte, error) {
 	jsonEncodeFloat64(buffer, "gpuHours", a.GPUHours, ",")
 	jsonEncodeFloat64(buffer, "gpuCost", a.GPUCost, ",")
 	jsonEncodeFloat64(buffer, "gpuCostAdjustment", a.GPUCostAdjustment, ",")
+	jsonEncodeFloat64(buffer, "networkTransferBytes", a.NetworkTransferBytes, ",")
+	jsonEncodeFloat64(buffer, "networkReceiveBytes", a.NetworkReceiveBytes, ",")
 	jsonEncodeFloat64(buffer, "networkCost", a.NetworkCost, ",")
 	jsonEncodeFloat64(buffer, "networkCostAdjustment", a.NetworkCostAdjustment, ",")
 	jsonEncodeFloat64(buffer, "loadBalancerCost", a.LoadBalancerCost, ",")
@@ -551,6 +563,11 @@ func (a *Allocation) IsUnallocated() bool {
 	return strings.Contains(a.Name, UnallocatedSuffix)
 }
 
+// IsUnmounted is true if the given Allocation represents unmounted volume costs.
+func (a *Allocation) IsUnmounted() bool {
+	return strings.Contains(a.Name, UnmountedSuffix)
+}
+
 // Minutes returns the number of minutes the Allocation represents, as defined
 // by the difference between the end and start times.
 func (a *Allocation) Minutes() float64 {
@@ -585,9 +602,38 @@ func (a *Allocation) add(that *Allocation) {
 		log.Warningf("Allocation.AggregateBy: trying to add a nil receiver")
 		return
 	}
+
+	// Generate keys for each allocation to allow for special logic to set the controller
+	// in the case of keys matching but controllers not matching.
+	aggByForKey := []string{"cluster", "node", "namespace", "pod", "container"}
+	leftKey := a.generateKey(aggByForKey, nil)
+	rightKey := a.generateKey(aggByForKey, nil)
+	leftProperties := a.Properties
+	rightProperties := that.Properties
+
 	// Preserve string properties that are matching between the two allocations
 	a.Properties = a.Properties.Intersection(that.Properties)
 
+	// Overwrite regular intersection logic for the controller name property in the
+	// case that the Allocation keys are the same but the controllers are not.
+	if leftKey == rightKey &&
+		leftProperties != nil &&
+		rightProperties != nil &&
+		leftProperties.Controller != rightProperties.Controller {
+		if leftProperties.Controller == "" {
+			a.Properties.Controller = rightProperties.Controller
+		} else if rightProperties.Controller == "" {
+			a.Properties.Controller = leftProperties.Controller
+		} else {
+			controllers := []string{
+				leftProperties.Controller,
+				rightProperties.Controller,
+			}
+			sort.Strings(controllers)
+			a.Properties.Controller = controllers[0]
+		}
+	}
+
 	// Expand the window to encompass both Allocations
 	a.Window = a.Window.Expand(that.Window)
 
@@ -632,6 +678,8 @@ func (a *Allocation) add(that *Allocation) {
 	a.CPUCoreHours += that.CPUCoreHours
 	a.GPUHours += that.GPUHours
 	a.RAMByteHours += that.RAMByteHours
+	a.NetworkTransferBytes += that.NetworkTransferBytes
+	a.NetworkReceiveBytes += that.NetworkReceiveBytes
 
 	// Sum all cumulative cost fields
 	a.CPUCost += that.CPUCost
@@ -696,13 +744,14 @@ func NewAllocationSet(start, end time.Time, allocs ...*Allocation) *AllocationSe
 // simple flag for sharing idle resources.
 type AllocationAggregationOptions struct {
 	FilterFuncs       []AllocationMatchFunc
-	SplitIdle         bool
 	IdleByNode        bool
+	LabelConfig       *LabelConfig
 	MergeUnallocated  bool
+	SharedHourlyCosts map[string]float64
 	ShareFuncs        []AllocationMatchFunc
 	ShareIdle         string
 	ShareSplit        string
-	SharedHourlyCosts map[string]float64
+	SplitIdle         bool
 }
 
 // AggregateBy aggregates the Allocations in the given AllocationSet by the given
@@ -733,13 +782,32 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 	//     the output (i.e. they can be used to generate a valid key for
 	//     the given properties) then aggregate; otherwise... ignore them?
 	// 10. If the merge idle option is enabled, merge any remaining idle
-	//     allocations into a single idle allocation
+	//     allocations into a single idle allocation. If there was any idle
+	//	   whose costs were not distributed because there was no usage of a
+	//     specific resource type, re-add the idle to the aggregation with
+	//     only that type.
+	if as.IsEmpty() {
+		return nil
+	}
 
 	if options == nil {
 		options = &AllocationAggregationOptions{}
 	}
 
-	if as.IsEmpty() {
+	if options.LabelConfig == nil {
+		options.LabelConfig = NewLabelConfig()
+	}
+
+	var undistributedIdleMap map[string]bool
+
+	// If aggregateBy is nil, we don't aggregate anything. On the other hand,
+	// an empty slice implies that we should aggregate everything. See
+	// generateKey for why that makes sense.
+	shouldAggregate := aggregateBy != nil
+	shouldFilter := len(options.FilterFuncs) > 0
+	shouldShare := len(options.SharedHourlyCosts) > 0 || len(options.ShareFuncs) > 0
+	if !shouldAggregate && !shouldFilter && !shouldShare {
+		// There is nothing for AggregateBy to do, so simply return nil
 		return nil
 	}
 
@@ -855,7 +923,7 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 	// the shared allocations).
 	var idleCoefficients map[string]map[string]map[string]float64
 	if idleSet.Length() > 0 && options.ShareIdle != ShareNone {
-		idleCoefficients, err = computeIdleCoeffs(options, as, shareSet)
+		idleCoefficients, undistributedIdleMap, err = computeIdleCoeffs(options, as, shareSet)
 		if err != nil {
 			log.Warningf("AllocationSet.AggregateBy: compute idle coeff: %s", err)
 			return fmt.Errorf("error computing idle coefficients: %s", err)
@@ -888,7 +956,7 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 	// need to track this on a per-cluster or per-node, per-allocation, per-resource basis.
 	var idleFiltrationCoefficients map[string]map[string]map[string]float64
 	if len(options.FilterFuncs) > 0 && options.ShareIdle == ShareNone {
-		idleFiltrationCoefficients, err = computeIdleCoeffs(options, as, shareSet)
+		idleFiltrationCoefficients, _, err = computeIdleCoeffs(options, as, shareSet)
 		if err != nil {
 			return fmt.Errorf("error computing idle filtration coefficients: %s", err)
 		}
@@ -902,7 +970,7 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 			hours := as.Resolution().Hours()
 
 			// If set ends in the future, adjust hours accordingly
-			diff := time.Now().Sub(as.End())
+			diff := time.Since(as.End())
 			if diff < 0.0 {
 				hours += diff.Hours()
 			}
@@ -934,7 +1002,7 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 	for _, alloc := range as.allocations {
 		idleId, err := alloc.getIdleId(options)
 		if err != nil {
-			log.DedupedWarningf(3,"AllocationSet.AggregateBy: missing idleId for allocation: %s", alloc.Name)
+			log.DedupedWarningf(3, "AllocationSet.AggregateBy: missing idleId for allocation: %s", alloc.Name)
 		}
 
 		skip := false
@@ -1003,7 +1071,7 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 		}
 
 		// (5) generate key to use for aggregation-by-key and allocation name
-		key := alloc.generateKey(aggregateBy)
+		key := alloc.generateKey(aggregateBy, options.LabelConfig)
 
 		alloc.Name = key
 		if options.MergeUnallocated && alloc.IsUnallocated() {
@@ -1110,7 +1178,9 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 		for _, alloc := range aggSet.allocations {
 			for _, sharedAlloc := range shareSet.allocations {
 				if _, ok := shareCoefficients[alloc.Name]; !ok {
-					log.Warningf("AllocationSet.AggregateBy: error getting share coefficienct for '%s'", alloc.Name)
+					if !alloc.IsIdle() {
+						log.Warningf("AllocationSet.AggregateBy: error getting share coefficienct for '%s'", alloc.Name)
+					}
 					continue
 				}
 
@@ -1132,7 +1202,7 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 			}
 		}
 		if !skip {
-			key := alloc.generateKey(aggregateBy)
+			key := alloc.generateKey(aggregateBy, options.LabelConfig)
 
 			alloc.Name = key
 			aggSet.Insert(alloc)
@@ -1148,6 +1218,66 @@ func (as *AllocationSet) AggregateBy(aggregateBy []string, options *AllocationAg
 		}
 	}
 
+	// In the edge case that some idle has not been distributed because
+	// there is no usage of that resource type, add idle back to
+	// aggregations with only that cost applied.
+
+	// E.g. in the case where we have a result that looks like this on the
+	// frontend:
+
+	// Name		CPU		GPU		RAM
+	// __idle__ $10     $12     $6
+	// kubecost $2      $0      $1
+
+	// Sharing idle weighted would result in no idle GPU cost being
+	// distributed, because the coefficient for the kubecost GPU cost would
+	// be zero. Thus, instead we re-add idle to the aggSet with distributed
+	// costs zeroed out but the undistributed costs left in.
+
+	// Name		CPU		GPU		RAM
+	// __idle__ $0      $12     $0
+	// kubecost $12     $0      $7
+
+	if idleSet.Length() > 0 && !options.SplitIdle {
+		if undistributedIdleMap["cpu"] || undistributedIdleMap["gpu"] || undistributedIdleMap["ram"] {
+
+			for _, idleAlloc := range idleSet.allocations {
+
+				skip := false
+
+				// if the idle does not apply to the non-filtered values, skip it
+				for _, ff := range options.FilterFuncs {
+					if !ff(idleAlloc) {
+						skip = true
+						break
+					}
+				}
+
+				if skip {
+					continue
+				}
+
+				// if the idle doesn't have a cost to be shared, also skip it
+				if idleAlloc.CPUCost != 0 && idleAlloc.GPUCost != 0 && idleAlloc.RAMCost != 0 {
+
+					// artificially set the already shared costs to zero
+					if !undistributedIdleMap["cpu"] {
+						idleAlloc.CPUCost = 0
+					}
+					if !undistributedIdleMap["gpu"] {
+						idleAlloc.GPUCost = 0
+					}
+					if !undistributedIdleMap["ram"] {
+						idleAlloc.RAMCost = 0
+					}
+
+					idleAlloc.Name = IdleSuffix
+					aggSet.Insert(idleAlloc)
+				}
+			}
+		}
+	}
+
 	as.allocations = aggSet.allocations
 
 	return nil
@@ -1170,10 +1300,14 @@ func computeShareCoeffs(aggregateBy []string, options *AllocationAggregationOpti
 			// Skip idle allocations in coefficient calculation
 			continue
 		}
+		if alloc.IsUnmounted() {
+			// Skip unmounted allocations in coefficient calculation
+			continue
+		}
 
 		// Determine the post-aggregation key under which the allocation will
 		// be shared.
-		name := alloc.generateKey(aggregateBy)
+		name := alloc.generateKey(aggregateBy, options.LabelConfig)
 
 		// If the current allocation will be filtered out in step 3, contribute
 		// its share of the shared coefficient to a "__filtered__" bin, which
@@ -1218,8 +1352,13 @@ func computeShareCoeffs(aggregateBy []string, options *AllocationAggregationOpti
 	return coeffs, nil
 }
 
-func computeIdleCoeffs(options *AllocationAggregationOptions, as *AllocationSet, shareSet *AllocationSet) (map[string]map[string]map[string]float64, error) {
+func computeIdleCoeffs(options *AllocationAggregationOptions, as *AllocationSet, shareSet *AllocationSet) (map[string]map[string]map[string]float64, map[string]bool, error) {
 	types := []string{"cpu", "gpu", "ram"}
+	undistributedIdleMap := map[string]bool{
+		"cpu": true,
+		"gpu": true,
+		"ram": true,
+	}
 
 	// Compute idle coefficients, then save them in AllocationAggregationOptions
 	coeffs := map[string]map[string]map[string]float64{}
@@ -1275,6 +1414,7 @@ func computeIdleCoeffs(options *AllocationAggregationOptions, as *AllocationSet,
 			totals[idleId]["gpu"] += alloc.GPUTotalCost()
 			totals[idleId]["ram"] += alloc.RAMTotalCost()
 		}
+
 	}
 
 	// Do the same for shared allocations
@@ -1330,12 +1470,13 @@ func computeIdleCoeffs(options *AllocationAggregationOptions, as *AllocationSet,
 			for _, r := range types {
 				if coeffs[c][a][r] > 0 && totals[c][r] > 0 {
 					coeffs[c][a][r] /= totals[c][r]
+					undistributedIdleMap[r] = false
 				}
 			}
 		}
 	}
 
-	return coeffs, nil
+	return coeffs, undistributedIdleMap, nil
 }
 
 // getIdleId returns the providerId or cluster of an Allocation depending on the IdleByNode
@@ -1358,11 +1499,15 @@ func (a *Allocation) getIdleId(options *AllocationAggregationOptions) (string, e
 	return idleId, nil
 }
 
-func (a *Allocation) generateKey(aggregateBy []string) string {
+func (a *Allocation) generateKey(aggregateBy []string, labelConfig *LabelConfig) string {
 	if a == nil {
 		return ""
 	}
 
+	if labelConfig == nil {
+		labelConfig = NewLabelConfig()
+	}
+
 	// Names will ultimately be joined into a single name, which uniquely
 	// identifies allocations.
 	names := []string{}
@@ -1404,7 +1549,7 @@ func (a *Allocation) generateKey(aggregateBy []string) string {
 			names = append(names, a.Properties.Container)
 		case agg == AllocationServiceProp:
 			services := a.Properties.Services
-			if services == nil || len(services) == 0 {
+			if len(services) == 0 {
 				// Indicate that allocation has no services
 				names = append(names, UnallocatedSuffix)
 			} else {
@@ -1417,78 +1562,113 @@ func (a *Allocation) generateKey(aggregateBy []string) string {
 		case strings.HasPrefix(agg, "label:"):
 			labels := a.Properties.Labels
 			if labels == nil {
-				// Indicate that allocation has no labels
 				names = append(names, UnallocatedSuffix)
 			} else {
-				labelNames := []string{}
-				aggLabels := strings.Split(strings.TrimPrefix(agg, "label:"), ";")
-				for _, labelName := range aggLabels {
-					if val, ok := labels[labelName]; ok {
-						labelNames = append(labelNames, fmt.Sprintf("%s=%s", labelName, val))
-					} else if indexOf(UnallocatedSuffix, labelNames) == -1 { // if UnallocatedSuffix not already in names
-						labelNames = append(labelNames, UnallocatedSuffix)
-					}
-				}
-				// resolve arbitrary ordering. e.g., app=app0/env=env0 is the same agg as env=env0/app=app0
-				if len(labelNames) > 1 {
-					sort.Strings(labelNames)
+				labelName := labelConfig.Sanitize(strings.TrimPrefix(agg, "label:"))
+				if labelValue, ok := labels[labelName]; ok {
+					names = append(names, fmt.Sprintf("%s=%s", labelName, labelValue))
+				} else {
+					names = append(names, UnallocatedSuffix)
 				}
-				unallocatedSuffixIndex := indexOf(UnallocatedSuffix, labelNames)
-				// suffix should be at index 0 if it exists b/c of underscores
-				if unallocatedSuffixIndex != -1 {
-					labelNames = append(labelNames[:unallocatedSuffixIndex], labelNames[unallocatedSuffixIndex+1:]...)
-					labelNames = append(labelNames, UnallocatedSuffix) // append to end
-				}
-
-				names = append(names, labelNames...)
 			}
 		case strings.HasPrefix(agg, "annotation:"):
 			annotations := a.Properties.Annotations
 			if annotations == nil {
-				// Indicate that allocation has no annotations
 				names = append(names, UnallocatedSuffix)
 			} else {
-				annotationNames := []string{}
-				aggAnnotations := strings.Split(strings.TrimPrefix(agg, "annotation:"), ";")
-				for _, annotationName := range aggAnnotations {
-					if val, ok := annotations[annotationName]; ok {
-						annotationNames = append(annotationNames, fmt.Sprintf("%s=%s", annotationName, val))
-					} else if indexOf(UnallocatedSuffix, annotationNames) == -1 { // if UnallocatedSuffix not already in names
-						annotationNames = append(annotationNames, UnallocatedSuffix)
+				annotationName := labelConfig.Sanitize(strings.TrimPrefix(agg, "annotation:"))
+				if annotationValue, ok := annotations[annotationName]; ok {
+					names = append(names, fmt.Sprintf("%s=%s", annotationName, annotationValue))
+				} else {
+					names = append(names, UnallocatedSuffix)
+				}
+			}
+		case agg == AllocationDepartmentProp:
+			labels := a.Properties.Labels
+			if labels == nil {
+				names = append(names, UnallocatedSuffix)
+			} else {
+				labelNames := strings.Split(labelConfig.DepartmentLabel, ",")
+				for _, labelName := range labelNames {
+					labelName = labelConfig.Sanitize(labelName)
+					if labelValue, ok := labels[labelName]; ok {
+						names = append(names, labelValue)
+					} else {
+						names = append(names, UnallocatedSuffix)
 					}
 				}
-				// resolve arbitrary ordering. e.g., app=app0/env=env0 is the same agg as env=env0/app=app0
-				if len(annotationNames) > 1 {
-					sort.Strings(annotationNames)
+			}
+		case agg == AllocationEnvironmentProp:
+			labels := a.Properties.Labels
+			if labels == nil {
+				names = append(names, UnallocatedSuffix)
+			} else {
+				labelNames := strings.Split(labelConfig.EnvironmentLabel, ",")
+				for _, labelName := range labelNames {
+					labelName = labelConfig.Sanitize(labelName)
+					if labelValue, ok := labels[labelName]; ok {
+						names = append(names, labelValue)
+					} else {
+						names = append(names, UnallocatedSuffix)
+					}
 				}
-				unallocatedSuffixIndex := indexOf(UnallocatedSuffix, annotationNames)
-				// suffix should be at index 0 if it exists b/c of underscores
-				if unallocatedSuffixIndex != -1 {
-					annotationNames = append(annotationNames[:unallocatedSuffixIndex], annotationNames[unallocatedSuffixIndex+1:]...)
-					annotationNames = append(annotationNames, UnallocatedSuffix) // append to end
+			}
+		case agg == AllocationOwnerProp:
+			labels := a.Properties.Labels
+			if labels == nil {
+				names = append(names, UnallocatedSuffix)
+			} else {
+				labelNames := strings.Split(labelConfig.OwnerLabel, ",")
+				for _, labelName := range labelNames {
+					labelName = labelConfig.Sanitize(labelName)
+					if labelValue, ok := labels[labelName]; ok {
+						names = append(names, labelValue)
+					} else {
+						names = append(names, UnallocatedSuffix)
+					}
+				}
+			}
+		case agg == AllocationProductProp:
+			labels := a.Properties.Labels
+			if labels == nil {
+				names = append(names, UnallocatedSuffix)
+			} else {
+				labelNames := strings.Split(labelConfig.ProductLabel, ",")
+				for _, labelName := range labelNames {
+					labelName = labelConfig.Sanitize(labelName)
+					if labelValue, ok := labels[labelName]; ok {
+						names = append(names, labelValue)
+					} else {
+						names = append(names, UnallocatedSuffix)
+					}
+				}
+			}
+		case agg == AllocationTeamProp:
+			labels := a.Properties.Labels
+			if labels == nil {
+				names = append(names, UnallocatedSuffix)
+			} else {
+				labelNames := strings.Split(labelConfig.TeamLabel, ",")
+				for _, labelName := range labelNames {
+					labelName = labelConfig.Sanitize(labelName)
+					if labelValue, ok := labels[labelName]; ok {
+						names = append(names, labelValue)
+					} else {
+						names = append(names, UnallocatedSuffix)
+					}
 				}
-
-				names = append(names, annotationNames...)
 			}
+		default:
+			// This case should never be reached, as input up until this point
+			// should be checked and rejected if invalid. But if we do get a
+			// value we don't recognize, log a warning.
+			log.Warningf("AggregateBy: illegal aggregation parameter: %s", agg)
 		}
-
 	}
 
 	return strings.Join(names, "/")
 }
 
-// TODO:CLEANUP get rid of this
-// Helper function to check for slice membership. Not sure if repeated elsewhere in our codebase.
-func indexOf(v string, arr []string) int {
-	for i, s := range arr {
-		// This is caseless equivalence
-		if strings.EqualFold(v, s) {
-			return i
-		}
-	}
-	return -1
-}
-
 // Clone returns a new AllocationSet with a deep copy of the given
 // AllocationSet's allocations.
 func (as *AllocationSet) Clone() *AllocationSet {
@@ -2274,7 +2454,7 @@ func (asr *AllocationSetRange) Length() int {
 // MarshalJSON JSON-encodes the range
 func (asr *AllocationSetRange) MarshalJSON() ([]byte, error) {
 	asr.RLock()
-	asr.RUnlock()
+	defer asr.RUnlock()
 	return json.Marshal(asr.allocations)
 }
 
@@ -2329,3 +2509,67 @@ func (asr *AllocationSetRange) Window() Window {
 
 	return NewWindow(&start, &end)
 }
+
+// Start returns the earliest start of all Allocations in the AllocationSetRange.
+// It returns an error if there are no allocations.
+func (asr *AllocationSetRange) Start() (time.Time, error) {
+	start := time.Time{}
+	firstStartNotSet := true
+	asr.Each(func(i int, as *AllocationSet) {
+		as.Each(func(s string, a *Allocation) {
+			if firstStartNotSet {
+				start = a.Start
+				firstStartNotSet = false
+			}
+			if a.Start.Before(start) {
+				start = a.Start
+			}
+		})
+	})
+
+	if firstStartNotSet {
+		return start, fmt.Errorf("had no data to compute a start from")
+	}
+
+	return start, nil
+}
+
+// End returns the latest end of all Allocations in the AllocationSetRange.
+// It returns an error if there are no allocations.
+func (asr *AllocationSetRange) End() (time.Time, error) {
+	end := time.Time{}
+	firstEndNotSet := true
+	asr.Each(func(i int, as *AllocationSet) {
+		as.Each(func(s string, a *Allocation) {
+			if firstEndNotSet {
+				end = a.End
+				firstEndNotSet = false
+			}
+			if a.End.After(end) {
+				end = a.End
+			}
+		})
+	})
+
+	if firstEndNotSet {
+		return end, fmt.Errorf("had no data to compute an end from")
+	}
+
+	return end, nil
+}
+
+// Minutes returns the duration, in minutes, between the earliest start
+// and the latest end of all allocations in the AllocationSetRange.
+func (asr *AllocationSetRange) Minutes() float64 {
+	start, err := asr.Start()
+	if err != nil {
+		return 0
+	}
+	end, err := asr.End()
+	if err != nil {
+		return 0
+	}
+	duration := end.Sub(start)
+
+	return duration.Minutes()
+}

pkg/kubecost/allocation_test.go

@@ -355,6 +355,30 @@ func TestAllocation_Share(t *testing.T) {
 	}
 }
 
+func TestAllocation_AddDifferentController(t *testing.T) {
+	a1 := &Allocation{
+		Properties: &AllocationProperties{
+			Container:  "container",
+			Pod:        "pod",
+			Namespace:  "ns",
+			Cluster:    "cluster",
+			Controller: "controller 1",
+		},
+	}
+	a2 := a1.Clone()
+	a2.Properties.Controller = "controller 2"
+
+	result, err := a1.Add(a2)
+	if err != nil {
+		t.Fatalf("Allocation.Add: unexpected error: %s", err)
+	}
+
+	if result.Properties.Controller == "" {
+		t.Errorf("Adding allocations whose properties only differ in controller name should not result in an empty string controller name.")
+	}
+
+}
+
 func TestAllocation_MarshalJSON(t *testing.T) {
 	start := time.Date(2021, time.January, 1, 0, 0, 0, 0, time.UTC)
 	end := time.Date(2021, time.January, 2, 0, 0, 0, 0, time.UTC)
@@ -435,7 +459,7 @@ func TestAllocationSet_generateKey(t *testing.T) {
 		AllocationClusterProp,
 	}
 
-	key = alloc.generateKey(props)
+	key = alloc.generateKey(props, nil)
 	if key != "" {
 		t.Fatalf("generateKey: expected \"\"; actual \"%s\"", key)
 	}
@@ -449,7 +473,7 @@ func TestAllocationSet_generateKey(t *testing.T) {
 		},
 	}
 
-	key = alloc.generateKey(props)
+	key = alloc.generateKey(props, nil)
 	if key != "cluster1" {
 		t.Fatalf("generateKey: expected \"cluster1\"; actual \"%s\"", key)
 	}
@@ -460,7 +484,7 @@ func TestAllocationSet_generateKey(t *testing.T) {
 		"label:app",
 	}
 
-	key = alloc.generateKey(props)
+	key = alloc.generateKey(props, nil)
 	if key != "cluster1//app=app1" {
 		t.Fatalf("generateKey: expected \"cluster1//app=app1\"; actual \"%s\"", key)
 	}
@@ -473,10 +497,72 @@ func TestAllocationSet_generateKey(t *testing.T) {
 			"env": "env1",
 		},
 	}
-	key = alloc.generateKey(props)
+	key = alloc.generateKey(props, nil)
 	if key != "cluster1/namespace1/app=app1" {
 		t.Fatalf("generateKey: expected \"cluster1/namespace1/app=app1\"; actual \"%s\"", key)
 	}
+
+	props = []string{
+		AllocationDepartmentProp,
+		AllocationEnvironmentProp,
+		AllocationOwnerProp,
+		AllocationProductProp,
+		AllocationTeamProp,
+	}
+
+	labelConfig := NewLabelConfig()
+
+	alloc.Properties = &AllocationProperties{
+		Cluster:   "cluster1",
+		Namespace: "namespace1",
+		Labels: map[string]string{
+			labelConfig.DepartmentLabel:  "dept1",
+			labelConfig.EnvironmentLabel: "envt1",
+			labelConfig.OwnerLabel:       "ownr1",
+			labelConfig.ProductLabel:     "prod1",
+			labelConfig.TeamLabel:        "team1",
+		},
+	}
+	key = alloc.generateKey(props, nil)
+	if key != "dept1/envt1/ownr1/prod1/team1" {
+		t.Fatalf("generateKey: expected \"dept1/envt1/ownr1/prod1/team1\"; actual \"%s\"", key)
+	}
+
+	// Ensure that labels with illegal Prometheus characters in LabelConfig
+	// still match their sanitized values. Ensure also that multiple comma-
+	// separated values work.
+
+	labelConfig.DepartmentLabel = "prom/illegal-department"
+	labelConfig.EnvironmentLabel = " env "
+	labelConfig.OwnerLabel = "$owner%"
+	labelConfig.ProductLabel = "app.kubernetes.io/app"
+	labelConfig.TeamLabel = "team,app.kubernetes.io/team,k8s-team"
+
+	alloc.Properties = &AllocationProperties{
+		Cluster:   "cluster1",
+		Namespace: "namespace1",
+		Labels: map[string]string{
+			"prom_illegal_department": "dept1",
+			"env":                     "envt1",
+			"_owner_":                 "ownr1",
+			"team":                    "team1",
+			"app_kubernetes_io_app":   "prod1",
+			"app_kubernetes_io_team":  "team2",
+		},
+	}
+
+	props = []string{
+		AllocationDepartmentProp,
+		AllocationEnvironmentProp,
+		AllocationOwnerProp,
+		AllocationProductProp,
+		AllocationTeamProp,
+	}
+
+	key = alloc.generateKey(props, labelConfig)
+	if key != "dept1/envt1/ownr1/prod1/team1/team2/__unallocated__" {
+		t.Fatalf("generateKey: expected \"dept1/envt1/ownr1/prod1/team1/team2/__unallocated__\"; actual \"%s\"", key)
+	}
 }
 
 func TestNewAllocationSet(t *testing.T) {
@@ -920,17 +1006,17 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 		// 2d AggregationProperties=(Label:app, Label:environment)
 		"2d": {
 			start:      start,
-			aggBy:      []string{"label:app;env"},
+			aggBy:      []string{"label:app", "label:env"},
 			aggOpts:    nil,
 			numResults: 3 + numIdle + numUnallocated,
 			totalCost:  activeTotalCost + idleTotalCost,
 			// sets should be {idle, unallocated, app1/env1, app2/env2, app2/unallocated}
 			results: map[string]float64{
-				"app=app1/env=env1":             16.00,
-				"app=app2/env=env2":             12.00,
-				"app=app2/" + UnallocatedSuffix: 12.00,
-				IdleSuffix:                      30.00,
-				UnallocatedSuffix:               42.00,
+				"app=app1/env=env1":                         16.00,
+				"app=app2/env=env2":                         12.00,
+				"app=app2/" + UnallocatedSuffix:             12.00,
+				IdleSuffix:                                  30.00,
+				UnallocatedSuffix + "/" + UnallocatedSuffix: 42.00,
 			},
 			windowStart: startYesterday,
 			windowEnd:   endYesterday,
@@ -939,17 +1025,17 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 		// 2e AggregationProperties=(Cluster, Label:app, Label:environment)
 		"2e": {
 			start:      start,
-			aggBy:      []string{AllocationClusterProp, "label:app;env"},
+			aggBy:      []string{AllocationClusterProp, "label:app", "label:env"},
 			aggOpts:    nil,
 			numResults: 6,
 			totalCost:  activeTotalCost + idleTotalCost,
 			results: map[string]float64{
-				"cluster1/app=app2/env=env2":             12.00,
-				"__idle__":                               30.00,
-				"cluster1/app=app1/env=env1":             16.00,
-				"cluster1/" + UnallocatedSuffix:          18.00,
-				"cluster2/app=app2/" + UnallocatedSuffix: 12.00,
-				"cluster2/" + UnallocatedSuffix:          24.00,
+				"cluster1/app=app2/env=env2": 12.00,
+				"__idle__":                   30.00,
+				"cluster1/app=app1/env=env1": 16.00,
+				"cluster1/" + UnallocatedSuffix + "/" + UnallocatedSuffix: 18.00,
+				"cluster2/app=app2/" + UnallocatedSuffix:                  12.00,
+				"cluster2/" + UnallocatedSuffix + "/" + UnallocatedSuffix: 24.00,
 			},
 			windowStart: startYesterday,
 			windowEnd:   endYesterday,
@@ -1498,8 +1584,8 @@ func TestAllocationSet_AggregateBy(t *testing.T) {
 			start: start,
 			aggBy: []string{AllocationNamespaceProp},
 			aggOpts: &AllocationAggregationOptions{
-				ShareIdle:   ShareEven,
-				IdleByNode:  true,
+				ShareIdle:  ShareEven,
+				IdleByNode: true,
 			},
 			numResults: 3,
 			totalCost:  112.00,
@@ -2135,3 +2221,265 @@ func TestAllocationSetRange_InsertRange(t *testing.T) {
 
 // TODO niko/etl
 // func TestAllocationSetRange_Window(t *testing.T) {}
+
+func TestAllocationSetRange_Start(t *testing.T) {
+	tests := []struct {
+		name string
+		arg  *AllocationSetRange
+
+		expectError bool
+		expected    time.Time
+	}{
+		{
+			name: "Empty ASR",
+			arg:  nil,
+
+			expectError: true,
+		},
+		{
+			name: "Single allocation",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								Start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "Two allocations",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								Start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+							"b": &Allocation{
+								Start: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "Two AllocationSets",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								Start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"b": &Allocation{
+								Start: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+	}
+
+	for _, test := range tests {
+		result, err := test.arg.Start()
+		if test.expectError && err != nil {
+			continue
+		}
+
+		if test.expectError && err == nil {
+			t.Errorf("%s: expected error and got none", test.name)
+		} else if result != test.expected {
+			t.Errorf("%s: expected %s but got %s", test.name, test.expected, result)
+		}
+	}
+}
+
+func TestAllocationSetRange_End(t *testing.T) {
+	tests := []struct {
+		name string
+		arg  *AllocationSetRange
+
+		expectError bool
+		expected    time.Time
+	}{
+		{
+			name: "Empty ASR",
+			arg:  nil,
+
+			expectError: true,
+		},
+		{
+			name: "Single allocation",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								End: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "Two allocations",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								End: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+							"b": &Allocation{
+								End: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "Two AllocationSets",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								End: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"b": &Allocation{
+								End: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+		},
+	}
+
+	for _, test := range tests {
+		result, err := test.arg.End()
+		if test.expectError && err != nil {
+			continue
+		}
+
+		if test.expectError && err == nil {
+			t.Errorf("%s: expected error and got none", test.name)
+		} else if result != test.expected {
+			t.Errorf("%s: expected %s but got %s", test.name, test.expected, result)
+		}
+	}
+}
+
+func TestAllocationSetRange_Minutes(t *testing.T) {
+	tests := []struct {
+		name string
+		arg  *AllocationSetRange
+
+		expected float64
+	}{
+		{
+			name: "Empty ASR",
+			arg:  nil,
+
+			expected: 0,
+		},
+		{
+			name: "Single allocation",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								Start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+								End:   time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: 24 * 60,
+		},
+		{
+			name: "Two allocations",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								Start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+								End:   time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+							"b": &Allocation{
+								Start: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+								End:   time.Date(1970, 1, 3, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: 2 * 24 * 60,
+		},
+		{
+			name: "Two AllocationSets",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								Start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+								End:   time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"b": &Allocation{
+								Start: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+								End:   time.Date(1970, 1, 3, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: 2 * 24 * 60,
+		},
+	}
+
+	for _, test := range tests {
+		result := test.arg.Minutes()
+		if result != test.expected {
+			t.Errorf("%s: expected %f but got %f", test.name, test.expected, result)
+		}
+	}
+}

pkg/kubecost/allocationprops.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"sort"
 	"strings"
+
+	"github.com/kubecost/cost-model/pkg/prom"
 )
 
 const (
@@ -23,6 +25,11 @@ const (
 	AllocationStatefulSetProp    string = "statefulset"
 	AllocationDaemonSetProp      string = "daemonset"
 	AllocationJobProp            string = "job"
+	AllocationDepartmentProp     string = "department"
+	AllocationEnvironmentProp    string = "environment"
+	AllocationOwnerProp          string = "owner"
+	AllocationProductProp        string = "product"
+	AllocationTeamProp           string = "team"
 )
 
 func ParseProperty(text string) (string, error) {
@@ -57,7 +64,28 @@ func ParseProperty(text string) (string, error) {
 		return AllocationStatefulSetProp, nil
 	case "job":
 		return AllocationJobProp, nil
+	case "department":
+		return AllocationDepartmentProp, nil
+	case "environment":
+		return AllocationEnvironmentProp, nil
+	case "owner":
+		return AllocationOwnerProp, nil
+	case "product":
+		return AllocationProductProp, nil
+	case "team":
+		return AllocationTeamProp, nil
+	}
+
+	if strings.HasPrefix(text, "label:") {
+		label := prom.SanitizeLabelName(strings.TrimSpace(strings.TrimPrefix(text, "label:")))
+		return fmt.Sprintf("label:%s", label), nil
 	}
+
+	if strings.HasPrefix(text, "annotation:") {
+		annotation := prom.SanitizeLabelName(strings.TrimSpace(strings.TrimPrefix(text, "annotation:")))
+		return fmt.Sprintf("annotation:%s", annotation), nil
+	}
+
 	return AllocationNilProp, fmt.Errorf("invalid allocation property: %s", text)
 }
 
@@ -72,8 +100,8 @@ type AllocationProperties struct {
 	Pod            string                `json:"pod,omitempty"`
 	Services       []string              `json:"services,omitempty"`
 	ProviderID     string                `json:"providerID,omitempty"`
-	Labels         AllocationLabels      `json:"allocationLabels,omitempty"`
-	Annotations    AllocationAnnotations `json:"allocationAnnotations,omitempty"`
+	Labels         AllocationLabels      `json:"labels,omitempty"`
+	Annotations    AllocationAnnotations `json:"annotations,omitempty"`
 }
 
 // AllocationLabels is a schema-free mapping of key/value pairs that can be

pkg/kubecost/asset.go

@@ -124,11 +124,16 @@ type Asset interface {
 //   => nil, err
 //
 // (See asset_test.go for assertions of these examples and more.)
-func AssetToExternalAllocation(asset Asset, aggregateBy []string, externalLabelsCfg map[string]string) (*Allocation, error) {
+func AssetToExternalAllocation(asset Asset, aggregateBy []string, labelConfig *LabelConfig) (*Allocation, error) {
 	if asset == nil {
 		return nil, fmt.Errorf("asset is nil")
 	}
 
+	// Use default label config if one is not provided.
+	if labelConfig == nil {
+		labelConfig = NewLabelConfig()
+	}
+
 	// names will collect the slash-separated names accrued by iterating over
 	// aggregateBy and checking the relevant labels.
 	names := []string{}
@@ -140,83 +145,89 @@ func AssetToExternalAllocation(asset Asset, aggregateBy []string, externalLabels
 	// props records the relevant Properties to set on the resultant Allocation
 	props := AllocationProperties{}
 
+	// For each aggregation parameter, try to find a match in the asset's
+	// labels, using the labelConfig to translate. For an aggregation parameter
+	// defined by a label (e.g. "label:app") this is simple: look for the label
+	// and use it (e.g. if "app" is a defined label on the asset, then use its
+	// value). For an aggregation parameter defined by a non-label property
+	// (e.g. "namespace") this requires using the labelConfig to look up the
+	// label name associated with that property and to use the value under that
+	// label, if set (e.g. if the aggregation property is "namespace" and the
+	// labelConfig is configured with "namespace_external_label" => "kubens"
+	// and the asset has label "kubens":"kubecost", then file the asset as an
+	// external cost under "kubecost").
 	for _, aggBy := range aggregateBy {
-		// labelName should be derived from the mapping of properties to
-		// label names, unless the aggBy is explicitly a label, in which
-		// case we should pull the label name from the aggBy string.
-		// Unless this matches a special aggregation, as we have that mapping already transformed...
-		labelName := aggBy
-		agglName := aggBy
-		if strings.HasPrefix(aggBy, "label:") {
-			labelName = strings.TrimPrefix(aggBy, "label:")
-			agglName = labelName
-			if v, ok := externalLabelsCfg[labelName]; ok {
-				agglName = v
-			}
-		}
+		name := labelConfig.GetExternalAllocationName(asset.Labels(), aggBy)
 
-		if labelName == "" {
+		if name == "" {
 			// No matching label has been defined in the cost-analyzer label config
 			// relating to the given aggregateBy property.
 			names = append(names, UnallocatedSuffix)
 			continue
-		}
+		} else {
+			names = append(names, name)
+			match = true
 
-		if value := asset.Labels()[agglName]; value != "" {
-			// Valid label value was found for one of the aggregation properties,
-			// so add it to the name.
-			if strings.HasPrefix(aggBy, "label:") {
-				// Use naming convention labelName=labelValue for labels
-				// e.g. aggBy="label:env", value="prod" => "env=prod"
-				names = append(names, fmt.Sprintf("%s=%s", strings.TrimPrefix(aggBy, "label:"), value))
-				match = true
-
-				// Set the corresponding label in props
-				labels := props.Labels
-				if labels == nil {
-					labels = map[string]string{}
-				}
+			// Default labels to an empty map, if necessary
+			if props.Labels == nil {
+				props.Labels = map[string]string{}
+			}
 
-				labels[labelName] = value
-				props.Labels = labels
-			} else {
-				names = append(names, value)
-				match = true
-
-				// Set the corresponding property on props
-				switch aggBy {
-				case AllocationClusterProp:
-					props.Cluster = value
-				case AllocationNodeProp:
-					props.Node = value
-				case AllocationNamespaceProp:
-					props.Namespace = value
-				case AllocationControllerKindProp:
-					props.ControllerKind = value
-				case AllocationControllerProp:
-					props.Controller = value
-				case AllocationPodProp:
-					props.Pod = value
-				case AllocationContainerProp:
-					props.Container = value
-				case AllocationServiceProp:
-					// TODO: external allocation: how to do this? multi-service?
-					props.Services = []string{value}
+			// Set the corresponding property on props
+			switch aggBy {
+			case AllocationClusterProp:
+				props.Cluster = name
+			case AllocationNodeProp:
+				props.Node = name
+			case AllocationNamespaceProp:
+				props.Namespace = name
+			case AllocationControllerKindProp:
+				props.ControllerKind = name
+			case AllocationControllerProp:
+				props.Controller = name
+			case AllocationPodProp:
+				props.Pod = name
+			case AllocationContainerProp:
+				props.Container = name
+			case AllocationServiceProp:
+				props.Services = []string{name}
+			case AllocationDeploymentProp:
+				props.Controller = name
+				props.ControllerKind = "deployment"
+			case AllocationStatefulSetProp:
+				props.Controller = name
+				props.ControllerKind = "statefulset"
+			case AllocationDaemonSetProp:
+				props.Controller = name
+				props.ControllerKind = "daemonset"
+			case AllocationDepartmentProp:
+				props.Labels[labelConfig.DepartmentLabel] = name
+			case AllocationEnvironmentProp:
+				props.Labels[labelConfig.EnvironmentLabel] = name
+			case AllocationOwnerProp:
+				props.Labels[labelConfig.OwnerLabel] = name
+			case AllocationProductProp:
+				props.Labels[labelConfig.ProductLabel] = name
+			case AllocationTeamProp:
+				props.Labels[labelConfig.TeamLabel] = name
+			default:
+				if strings.HasPrefix(aggBy, "label:") {
+					// Set the corresponding label in props
+					labelName := strings.TrimPrefix(aggBy, "label:")
+					labelValue := strings.TrimPrefix(name, labelName+"=")
+					props.Labels[labelName] = labelValue
 				}
 			}
-		} else {
-			// No value label value was found on the Asset; consider it
-			// unallocated. Note that this case is only truly relevant if at
-			// least one other property matches (e.g. case 3 in the examples)
-			// because if there are no matches, then an error is returned.
-			names = append(names, UnallocatedSuffix)
 		}
 	}
 
+	// If not a signle aggregation property generated a matching label property,
+	// then consider the asset ineligible to be treated as an external allocation.
 	if !match {
 		return nil, fmt.Errorf("asset does not qualify as an external allocation")
 	}
 
+	// Use naming to label as an external allocation. See IsExternal() for more.
 	names = append(names, ExternalSuffix)
 
 	// TODO: external allocation: efficiency?
@@ -3380,6 +3391,10 @@ func (as *AssetSet) ReconciliationMatch(query Asset) (Asset, bool, error) {
 
 	var providerIDMatch Asset
 	for _, asset := range as.assets {
+		// Ignore cloud assets when looking for reconciliation matches
+		if asset.Type() == CloudAssetType {
+			continue
+		}
 		if k, err := key(asset, fullMatchProps); err != nil {
 			return nil, false, err
 		} else if k == fullMatchKey {
@@ -3721,6 +3736,71 @@ func (asr *AssetSetRange) Window() Window {
 	return NewWindow(&start, &end)
 }
 
+// Start returns the earliest start of all Assets in the AssetSetRange.
+// It returns an error if there are no assets
+func (asr *AssetSetRange) Start() (time.Time, error) {
+	start := time.Time{}
+	firstStartNotSet := true
+	asr.Each(func(i int, as *AssetSet) {
+		as.Each(func(s string, a Asset) {
+			if firstStartNotSet {
+				start = a.Start()
+				firstStartNotSet = false
+			}
+			if a.Start().Before(start) {
+				start = a.Start()
+			}
+		})
+	})
+
+	if firstStartNotSet {
+		return start, fmt.Errorf("had no data to compute a start from")
+	}
+
+	return start, nil
+}
+
+// End returns the latest end of all Assets in the AssetSetRange.
+// It returns an error if there are no assets.
+func (asr *AssetSetRange) End() (time.Time, error) {
+	end := time.Time{}
+	firstEndNotSet := true
+	asr.Each(func(i int, as *AssetSet) {
+		as.Each(func(s string, a Asset) {
+			if firstEndNotSet {
+				end = a.End()
+				firstEndNotSet = false
+			}
+			if a.End().After(end) {
+				end = a.End()
+			}
+		})
+	})
+
+	if firstEndNotSet {
+		return end, fmt.Errorf("had no data to compute an end from")
+	}
+
+	return end, nil
+}
+
+// Minutes returns the duration, in minutes, between the earliest start
+// and the latest end of all assets in the AssetSetRange.
+func (asr *AssetSetRange) Minutes() float64 {
+	start, err := asr.Start()
+	if err != nil {
+		return 0
+	}
+	end, err := asr.End()
+	if err != nil {
+		return 0
+	}
+
+	duration := end.Sub(start)
+
+	return duration.Minutes()
+}
+
 // Returns true if string slices a and b contain all of the same strings, in any order.
 func sameContents(a, b []string) bool {
 	if len(a) != len(b) {

pkg/kubecost/asset_test.go

@@ -923,9 +923,11 @@ func TestAssetToExternalAllocation(t *testing.T) {
 	var alloc *Allocation
 	var err error
 
-	alloc, err = AssetToExternalAllocation(asset, []string{"namespace"}, map[string]string{})
+	labelConfig := NewLabelConfig()
+
+	_, err = AssetToExternalAllocation(asset, []string{"namespace"}, labelConfig)
 	if err == nil {
-		t.Fatalf("expected error due to nil asset")
+		t.Fatalf("expected error due to nil asset; no error returned")
 	}
 
 	// Consider this Asset:
@@ -938,20 +940,25 @@ func TestAssetToExternalAllocation(t *testing.T) {
 	//   }
 	cloud := NewCloud(ComputeCategory, "abc123", start1, start2, windows[0])
 	cloud.SetLabels(map[string]string{
-		"namespace": "monitoring",
-		"env":       "prod",
-		"product":   "cost-analyzer",
+		"kubernetes_namespace":        "monitoring",
+		"env":                         "prod",
+		"app":                         "cost-analyzer",
+		"kubernetes_label_app":        "app",
+		"kubernetes_label_department": "department",
+		"kubernetes_label_env":        "env",
+		"kubernetes_label_owner":      "owner",
+		"kubernetes_label_team":       "team",
 	})
 	cloud.Cost = 10.00
 	asset = cloud
 
-	alloc, err = AssetToExternalAllocation(asset, []string{"namespace"}, map[string]string{})
+	_, err = AssetToExternalAllocation(asset, []string{"namespace"}, nil)
 	if err != nil {
-		t.Fatalf("expected to not error")
+		t.Fatalf("unexpected error: %s", err)
 	}
-	alloc, err = AssetToExternalAllocation(asset, nil, map[string]string{})
+	_, err = AssetToExternalAllocation(asset, nil, nil)
 	if err == nil {
-		t.Fatalf("expected error due to nil aggregateBy")
+		t.Fatalf("expected error due to nil aggregateBy; no error returned")
 	}
 
 	// Given the following parameters, we expect to return:
@@ -971,13 +978,18 @@ func TestAssetToExternalAllocation(t *testing.T) {
 	//   allocationPropertyLabels = {"namespace":"kubernetes_namespace"}
 	//   => Allocation{Name: "monitoring/__unallocated__", ExternalCost: 10.00, TotalCost: 10.00}, nil
 	//
-	//   4) no match
+	//	 4) label alias match(es)
+	//	 aggregateBy = ["product", "deployment", "environment", "owner", "team"]
+	//   allocationPropertyLabels = {"namespace":"kubernetes_namespace"}
+	//   => Allocation{Name: "app/department/env/owner/team", ExternalCost: 10.00, TotalCost: 10.00}, nil
+	//
+	//   5) no match
 	//   aggregateBy = ["cluster"]
 	//   allocationPropertyLabels = {"namespace":"kubernetes_namespace"}
 	//   => nil, err
 
 	// 1) single-prop full match
-	alloc, err = AssetToExternalAllocation(asset, []string{"namespace"}, map[string]string{})
+	alloc, err = AssetToExternalAllocation(asset, []string{"namespace"}, nil)
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
@@ -985,7 +997,7 @@ func TestAssetToExternalAllocation(t *testing.T) {
 		t.Fatalf("expected external allocation with name '%s'; got '%s'", "monitoring/__external__", alloc.Name)
 	}
 	if ns := alloc.Properties.Namespace; ns != "monitoring" {
-		t.Fatalf("expected external allocation with AllocationProperties.Namespace '%s'; got '%s' (%s)", "monitoring", ns, err)
+		t.Fatalf("expected external allocation with AllocationProperties.Namespace '%s'; got '%s'", "monitoring", ns)
 	}
 	if alloc.ExternalCost != 10.00 {
 		t.Fatalf("expected external allocation with ExternalCost %f; got %f", 10.00, alloc.ExternalCost)
@@ -995,7 +1007,7 @@ func TestAssetToExternalAllocation(t *testing.T) {
 	}
 
 	// 2) multi-prop full match
-	alloc, err = AssetToExternalAllocation(asset, []string{"namespace", "label:env"}, map[string]string{})
+	alloc, err = AssetToExternalAllocation(asset, []string{"namespace", "label:env"}, nil)
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
@@ -1016,7 +1028,7 @@ func TestAssetToExternalAllocation(t *testing.T) {
 	}
 
 	// 3) multi-prop partial match
-	alloc, err = AssetToExternalAllocation(asset, []string{"namespace", "label:foo"}, map[string]string{})
+	alloc, err = AssetToExternalAllocation(asset, []string{"namespace", "label:foo"}, nil)
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
@@ -1033,134 +1045,314 @@ func TestAssetToExternalAllocation(t *testing.T) {
 		t.Fatalf("expected external allocation with TotalCost %f; got %f", 10.00, alloc.TotalCost())
 	}
 
-	// 3) no match
-	alloc, err = AssetToExternalAllocation(asset, []string{"cluster"}, map[string]string{})
+	// 4) label alias match(es)
+	alloc, err = AssetToExternalAllocation(asset, []string{"product", "department", "environment", "owner", "team"}, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if alloc.Name != "app/department/env/owner/team/__external__" {
+		t.Fatalf("expected external allocation with name '%s'; got '%s'", "app/department/env/owner/team/__external__", alloc.Name)
+	}
+	if alloc.Properties.Labels[labelConfig.ProductLabel] != "app" {
+		t.Fatalf("expected external allocation with label %s equal to %s; got %s", labelConfig.ProductLabel, "app", alloc.Properties.Labels[labelConfig.ProductLabel])
+	}
+	if alloc.Properties.Labels[labelConfig.DepartmentLabel] != "department" {
+		t.Fatalf("expected external allocation with label %s equal to %s; got %s", labelConfig.DepartmentLabel, "department", alloc.Properties.Labels[labelConfig.DepartmentLabel])
+	}
+	if alloc.Properties.Labels[labelConfig.EnvironmentLabel] != "env" {
+		t.Fatalf("expected external allocation with label %s equal to %s; got %s", labelConfig.EnvironmentLabel, "env", alloc.Properties.Labels[labelConfig.EnvironmentLabel])
+	}
+	if alloc.Properties.Labels[labelConfig.OwnerLabel] != "owner" {
+		t.Fatalf("expected external allocation with label %s equal to %s; got %s", labelConfig.OwnerLabel, "owner", alloc.Properties.Labels[labelConfig.OwnerLabel])
+	}
+	if alloc.Properties.Labels[labelConfig.TeamLabel] != "team" {
+		t.Fatalf("expected external allocation with label %s equal to %s; got %s", labelConfig.TeamLabel, "team", alloc.Properties.Labels[labelConfig.TeamLabel])
+	}
+	if alloc.ExternalCost != 10.00 {
+		t.Fatalf("expected external allocation with ExternalCost %f; got %f", 10.00, alloc.ExternalCost)
+	}
+	if alloc.TotalCost() != 10.00 {
+		t.Fatalf("expected external allocation with TotalCost %f; got %f", 10.00, alloc.TotalCost())
+	}
+
+	// 5) no match
+	_, err = AssetToExternalAllocation(asset, []string{"cluster"}, nil)
 	if err == nil {
 		t.Fatalf("expected 'no match' error")
 	}
 
-	alloc, err = AssetToExternalAllocation(asset, []string{"namespace", "label:app"}, map[string]string{"app": "product"})
+	// other cases
+
+	alloc, err = AssetToExternalAllocation(asset, []string{"namespace", "label:app"}, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
 	if alloc.ExternalCost != 10.00 {
 		t.Fatalf("expected external allocation with ExternalCost %f; got %f", 10.00, alloc.ExternalCost)
 	}
 	if alloc.TotalCost() != 10.00 {
 		t.Fatalf("expected external allocation with TotalCost %f; got %f", 10.00, alloc.TotalCost())
 	}
+}
+
+func TestAssetSetRange_Start(t *testing.T) {
+	tests := []struct {
+		name string
+		arg  *AssetSetRange
+
+		expectError bool
+		expected    time.Time
+	}{
+		{
+			name: "Empty ASR",
+			arg:  nil,
+
+			expectError: true,
+		},
+		{
+			name: "Single asset",
+			arg: &AssetSetRange{
+				assets: []*AssetSet{
+					&AssetSet{
+						assets: map[string]Asset{
+							"a": &Node{
+								start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "Two assets",
+			arg: &AssetSetRange{
+				assets: []*AssetSet{
+					&AssetSet{
+						assets: map[string]Asset{
+							"a": &Node{
+								start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+							"b": &Node{
+								start: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "Two AssetSets",
+			arg: &AssetSetRange{
+				assets: []*AssetSet{
+					&AssetSet{
+						assets: map[string]Asset{
+							"a": &Node{
+								start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+					&AssetSet{
+						assets: map[string]Asset{
+							"b": &Node{
+								start: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+	}
+
+	for _, test := range tests {
+		result, err := test.arg.Start()
+		if test.expectError && err != nil {
+			continue
+		}
 
+		if test.expectError && err == nil {
+			t.Errorf("%s: expected error and got none", test.name)
+		} else if result != test.expected {
+			t.Errorf("%s: expected %s but got %s", test.name, test.expected, result)
+		}
+	}
 }
 
-// TODO merge conflict had this:
-
-// as.Each(func(key string, a Asset) {
-// 	if exp, ok := exps[key]; ok {
-// 		if math.Round(a.TotalCost()*100) != math.Round(exp*100) {
-// 			t.Fatalf("AssetSet.AggregateBy[%s]: key %s expected total cost %.2f, actual %.2f", msg, key, exp, a.TotalCost())
-// 		}
-// 		if !a.Window().Equal(window) {
-// 			t.Fatalf("AssetSet.AggregateBy[%s]: key %s expected window %s, actual %s", msg, key, window, as.Window)
-// 		}
-// 	} else {
-// 		t.Fatalf("AssetSet.AggregateBy[%s]: unexpected asset: %s", msg, key)
-// 	}
-// })
-// }
-
-// // GenerateMockAssetSet generates the following topology:
-// //
-// // | Asset                        | Cost |  Adj |
-// // +------------------------------+------+------+
-// //   cluster1:
-// //     node1:                        6.00   1.00
-// //     node2:                        4.00   1.50
-// //     node3:                        7.00  -0.50
-// //     disk1:                        2.50   0.00
-// //     disk2:                        1.50   0.00
-// //     clusterManagement1:           3.00   0.00
-// // +------------------------------+------+------+
-// //   cluster1 subtotal              24.00   2.00
-// // +------------------------------+------+------+
-// //   cluster2:
-// //     node4:                       12.00  -1.00
-// //     disk3:                        2.50   0.00
-// //     disk4:                        1.50   0.00
-// //     clusterManagement2:           0.00   0.00
-// // +------------------------------+------+------+
-// //   cluster2 subtotal              16.00  -1.00
-// // +------------------------------+------+------+
-// //   cluster3:
-// //     node5:                       17.00   2.00
-// // +------------------------------+------+------+
-// //   cluster3 subtotal              17.00   2.00
-// // +------------------------------+------+------+
-// //   total                          57.00   3.00
-// // +------------------------------+------+------+
-// func GenerateMockAssetSet(start time.Time) *AssetSet {
-// end := start.Add(day)
-// window := NewWindow(&start, &end)
-
-// hours := window.Duration().Hours()
-
-// node1 := NewNode("node1", "cluster1", "gcp-node1", *window.Clone().start, *window.Clone().end, window.Clone())
-// node1.CPUCost = 4.0
-// node1.RAMCost = 4.0
-// node1.GPUCost = 2.0
-// node1.Discount = 0.5
-// node1.CPUCoreHours = 2.0 * hours
-// node1.RAMByteHours = 4.0 * gb * hours
-// node1.SetAdjustment(1.0)
-// node1.SetLabels(map[string]string{"test": "test"})
-
-// node2 := NewNode("node2", "cluster1", "gcp-node2", *window.Clone().start, *window.Clone().end, window.Clone())
-// node2.CPUCost = 4.0
-// node2.RAMCost = 4.0
-// node2.GPUCost = 0.0
-// node2.Discount = 0.5
-// node2.CPUCoreHours = 2.0 * hours
-// node2.RAMByteHours = 4.0 * gb * hours
-// node2.SetAdjustment(1.5)
-
-// node3 := NewNode("node3", "cluster1", "gcp-node3", *window.Clone().start, *window.Clone().end, window.Clone())
-// node3.CPUCost = 4.0
-// node3.RAMCost = 4.0
-// node3.GPUCost = 3.0
-// node3.Discount = 0.5
-// node3.CPUCoreHours = 2.0 * hours
-// node3.RAMByteHours = 4.0 * gb * hours
-// node3.SetAdjustment(-0.5)
-
-// node4 := NewNode("node4", "cluster2", "gcp-node4", *window.Clone().start, *window.Clone().end, window.Clone())
-// node4.CPUCost = 10.0
-// node4.RAMCost = 6.0
-// node4.GPUCost = 0.0
-// node4.Discount = 0.25
-// node4.CPUCoreHours = 4.0 * hours
-// node4.RAMByteHours = 12.0 * gb * hours
-// node4.SetAdjustment(-1.0)
-
-// node5 := NewNode("node5", "cluster3", "aws-node5", *window.Clone().start, *window.Clone().end, window.Clone())
-// node5.CPUCost = 10.0
-// node5.RAMCost = 7.0
-// node5.GPUCost = 0.0
-// node5.Discount = 0.0
-// node5.CPUCoreHours = 8.0 * hours
-// node5.RAMByteHours = 24.0 * gb * hours
-// node5.SetAdjustment(2.0)
-
-// disk1 := NewDisk("disk1", "cluster1", "gcp-disk1", *window.Clone().start, *window.Clone().end, window.Clone())
-// disk1.Cost = 2.5
-// disk1.ByteHours = 100 * gb * hours
-
-// disk2 := NewDisk("disk2", "cluster1", "gcp-disk2", *window.Clone().start, *window.Clone().end, window.Clone())
-// disk2.Cost = 1.5
-// disk2.ByteHours = 60 * gb * hours
-
-// disk3 := NewDisk("disk3", "cluster2", "gcp-disk3", *window.Clone().start, *window.Clone().end, window.Clone())
-// disk3.Cost = 2.5
-// disk3.ByteHours = 100 * gb * hours
-
-// disk4 := NewDisk("disk4", "cluster2", "gcp-disk4", *window.Clone().start, *window.Clone().end, window.Clone())
-// disk4.Cost = 1.5
-// disk4.ByteHours = 100 * gb * hours
-
-// cm1 := NewClusterManagement("gcp", "cluster1", window.Clone())
-// cm1.Cost = 3.0
+func TestAssetSetRange_End(t *testing.T) {
+	tests := []struct {
+		name string
+		arg  *AssetSetRange
+
+		expectError bool
+		expected    time.Time
+	}{
+		{
+			name: "Empty ASR",
+			arg:  nil,
+
+			expectError: true,
+		},
+		{
+			name: "Single asset",
+			arg: &AssetSetRange{
+				assets: []*AssetSet{
+					&AssetSet{
+						assets: map[string]Asset{
+							"a": &Node{
+								end: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "Two assets",
+			arg: &AssetSetRange{
+				assets: []*AssetSet{
+					&AssetSet{
+						assets: map[string]Asset{
+							"a": &Node{
+								end: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+							"b": &Node{
+								end: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "Two AssetSets",
+			arg: &AssetSetRange{
+				assets: []*AssetSet{
+					&AssetSet{
+						assets: map[string]Asset{
+							"a": &Node{
+								end: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+					&AssetSet{
+						assets: map[string]Asset{
+							"b": &Node{
+								end: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+		},
+	}
+
+	for _, test := range tests {
+		result, err := test.arg.End()
+		if test.expectError && err != nil {
+			continue
+		}
+
+		if test.expectError && err == nil {
+			t.Errorf("%s: expected error and got none", test.name)
+		} else if result != test.expected {
+			t.Errorf("%s: expected %s but got %s", test.name, test.expected, result)
+		}
+	}
+}
+
+func TestAssetSetRange_Minutes(t *testing.T) {
+	tests := []struct {
+		name string
+		arg  *AssetSetRange
+
+		expected float64
+	}{
+		{
+			name: "Empty ASR",
+			arg:  nil,
+
+			expected: 0,
+		},
+		{
+			name: "Single asset",
+			arg: &AssetSetRange{
+				assets: []*AssetSet{
+					&AssetSet{
+						assets: map[string]Asset{
+							"a": &Node{
+								start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+								end:   time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: 24 * 60,
+		},
+		{
+			name: "Two assets",
+			arg: &AssetSetRange{
+				assets: []*AssetSet{
+					&AssetSet{
+						assets: map[string]Asset{
+							"a": &Node{
+								start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+								end:   time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+							"b": &Node{
+								start: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+								end:   time.Date(1970, 1, 3, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: 2 * 24 * 60,
+		},
+		{
+			name: "Two AssetSets",
+			arg: &AssetSetRange{
+				assets: []*AssetSet{
+					&AssetSet{
+						assets: map[string]Asset{
+							"a": &Node{
+								start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+								end:   time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+					&AssetSet{
+						assets: map[string]Asset{
+							"b": &Node{
+								start: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+								end:   time.Date(1970, 1, 3, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: 2 * 24 * 60,
+		},
+	}
+
+	for _, test := range tests {
+		result := test.arg.Minutes()
+		if result != test.expected {
+			t.Errorf("%s: expected %f but got %f", test.name, test.expected, result)
+		}
+	}
+}

pkg/kubecost/assetprops.go

@@ -65,20 +65,6 @@ func ParseAssetProperty(text string) (AssetProperty, error) {
 	return AssetNilProp, fmt.Errorf("invalid asset property: %s", text)
 }
 
-func propsEqual(p1, p2 []AssetProperty) bool {
-	if len(p1) != len(p2) {
-		return false
-	}
-
-	for _, p := range p1 {
-		if !hasProp(p2, p) {
-			return false
-		}
-	}
-
-	return true
-}
-
 // Category options
 
 // ComputeCategory signifies the Compute Category

pkg/kubecost/bingen.go

@@ -1,11 +1,16 @@
 package kubecost
 
+// Default Version Set (uses -version flag passed) includes shared resources
+// @bingen:generate:Window
+
+// Asset Version Set: Includes Asset pipeline specific resources
+// @bingen:set[name=Assets,version=15]
 // @bingen:generate:Any
 // @bingen:generate:Asset
 // @bingen:generate:AssetLabels
 // @bingen:generate:AssetProperties
 // @bingen:generate:AssetProperty
-// @bingen:generate:AssetSet
+// @bingen:generate[stringtable]:AssetSet
 // @bingen:generate:AssetSetRange
 // @bingen:generate:Breakdown
 // @bingen:generate:Cloud
@@ -15,10 +20,12 @@ package kubecost
 // @bingen:generate:Network
 // @bingen:generate:Node
 // @bingen:generate:SharedAsset
-// @bingen:generate:Window
+// @bingen:end
 
+// Allocation Version Set: Includes Allocation pipeline specific resources
+// @bingen:set[name=Allocation,version=15]
 // @bingen:generate:Allocation
-// @bingen:generate:AllocationSet
+// @bingen:generate[stringtable]:AllocationSet
 // @bingen:generate:AllocationSetRange
 // @bingen:generate:AllocationProperties
 // @bingen:generate:AllocationProperty
@@ -28,5 +35,6 @@ package kubecost
 // @bingen:generate:PVAllocations
 // @bingen:generate:PVKey
 // @bingen:generate:PVAllocation
+// @bingen:end
 
-//go:generate bingen -package=kubecost -version=14 -buffer=github.com/kubecost/cost-model/pkg/util
+//go:generate bingen -package=kubecost -version=15 -buffer=github.com/kubecost/cost-model/pkg/util

pkg/kubecost/config.go

@@ -3,6 +3,9 @@ package kubecost
 import (
 	"fmt"
 	"strings"
+
+	"github.com/kubecost/cost-model/pkg/prom"
+	"github.com/kubecost/cost-model/pkg/util/cloudutil"
 )
 
 // LabelConfig is a port of type AnalyzerConfig. We need to be more thoughtful
@@ -29,6 +32,30 @@ type LabelConfig struct {
 	TeamExternalLabel        string `json:"team_external_label"`
 }
 
+// NewLabelConfig creates a new LabelConfig instance with default values.
+func NewLabelConfig() *LabelConfig {
+	return &LabelConfig{
+		DepartmentLabel:          "department",
+		EnvironmentLabel:         "env",
+		OwnerLabel:               "owner",
+		ProductLabel:             "app",
+		TeamLabel:                "team",
+		ClusterExternalLabel:     "kubernetes_cluster",
+		NamespaceExternalLabel:   "kubernetes_namespace",
+		ControllerExternalLabel:  "kubernetes_controller",
+		DaemonsetExternalLabel:   "kubernetes_daemonset",
+		DeploymentExternalLabel:  "kubernetes_deployment",
+		StatefulsetExternalLabel: "kubernetes_statefulset",
+		ServiceExternalLabel:     "kubernetes_service",
+		PodExternalLabel:         "kubernetes_pod",
+		DepartmentExternalLabel:  "kubernetes_label_department",
+		EnvironmentExternalLabel: "kubernetes_label_env",
+		OwnerExternalLabel:       "kubernetes_label_owner",
+		ProductExternalLabel:     "kubernetes_label_app",
+		TeamExternalLabel:        "kubernetes_label_team",
+	}
+}
+
 // Map returns the config as a basic string map, with default values if not set
 func (lc *LabelConfig) Map() map[string]string {
 	// Start with default values
@@ -142,57 +169,111 @@ func (lc *LabelConfig) Map() map[string]string {
 	return m
 }
 
-// ExternalQueryLabels returns the config's external labels as a mapping of the
-// query column to the label it should set;
-// e.g. if the config stores "statefulset_external_label": "kubernetes_sset",
-//      then this would return "kubernetes_sset": "statefulset"
-func (lc *LabelConfig) ExternalQueryLabels() map[string]string {
-	queryLabels := map[string]string{}
+// Sanitize returns a sanitized version of the given string, which converts
+// all illegal characters to underscores. Illegal characters are those that
+// Prometheus does not support; i.e. [^a-zA-Z0-9_]
+func (lc *LabelConfig) Sanitize(label string) string {
+	return prom.SanitizeLabelName(strings.TrimSpace(label))
+}
+
+// GetExternalAllocationName derives an external allocation name from a set of
+// labels, given an aggregation property. If the aggregation property is,
+// itself, a label (e.g. label:app) then this function looks for a
+// corresponding value under "app" and returns "app=thatvalue". If the
+// aggregation property is not a label but a Kubernetes concept
+// (e.g. namespace) then this function first finds the "external label"
+// configured to represent it (e.g. NamespaceExternalLabel: "kubens") and uses
+// that label to determine an external allocation name. If no label value can
+// be found, return an empty string.
+func (lc *LabelConfig) GetExternalAllocationName(labels map[string]string, aggregateBy string) string {
+	labelNames := []string{}
+	aggByLabel := false
+
+	// Determine if the aggregation property is, itself, a label or not. If
+	// not, determine the label associated with the given aggregation property.
+	if strings.HasPrefix(aggregateBy, "label:") {
+		labelNames = append(labelNames, prom.SanitizeLabelName(strings.TrimPrefix(aggregateBy, "label:")))
+		aggByLabel = true
+	} else {
+		// If lc is nil, use a default LabelConfig to do a best-effort match
+		if lc == nil {
+			lc = NewLabelConfig()
+		}
+
+		switch strings.ToLower(aggregateBy) {
+		case AllocationClusterProp:
+			labelNames = strings.Split(lc.ClusterExternalLabel, ",")
+		case AllocationControllerProp:
+			labelNames = strings.Split(lc.ControllerExternalLabel, ",")
+		case AllocationNamespaceProp:
+			labelNames = strings.Split(lc.NamespaceExternalLabel, ",")
+		case AllocationPodProp:
+			labelNames = strings.Split(lc.PodExternalLabel, ",")
+		case AllocationServiceProp:
+			labelNames = strings.Split(lc.ServiceExternalLabel, ",")
+		case AllocationDeploymentProp:
+			labelNames = strings.Split(lc.DeploymentExternalLabel, ",")
+		case AllocationStatefulSetProp:
+			labelNames = strings.Split(lc.StatefulsetExternalLabel, ",")
+		case AllocationDaemonSetProp:
+			labelNames = strings.Split(lc.DaemonsetExternalLabel, ",")
+		case AllocationDepartmentProp:
+			labelNames = strings.Split(lc.DepartmentExternalLabel, ",")
+		case AllocationEnvironmentProp:
+			labelNames = strings.Split(lc.EnvironmentExternalLabel, ",")
+		case AllocationOwnerProp:
+			labelNames = strings.Split(lc.OwnerExternalLabel, ",")
+		case AllocationProductProp:
+			labelNames = strings.Split(lc.ProductExternalLabel, ",")
+		case AllocationTeamProp:
+			labelNames = strings.Split(lc.TeamExternalLabel, ",")
+		}
 
-	for label, query := range lc.Map() {
-		if strings.HasSuffix(label, "external_label") && query != "" {
-			queryLabels[query] = label
+		for i, labelName := range labelNames {
+			labelNames[i] = prom.SanitizeLabelName(strings.TrimSpace(labelName))
 		}
 	}
 
-	return queryLabels
-}
+	// No label is set for the given aggregation property.
+	if len(labelNames) == 0 {
+		return ""
+	}
 
-// AllocationPropertyLabels returns the config's external resource labels
-// as a mapping from k8s resource-to-label name.
-// e.g. if the config stores "statefulset_external_label": "kubernetes_sset",
-//      then this would return "statefulset": "kubernetes_sset"
-// e.g. if the config stores "owner_label": "product_owner",
-//      then this would return "label:product_owner": "product_owner"
-func (lc *LabelConfig) AllocationPropertyLabels() map[string]string {
-	labels := map[string]string{}
-
-	for labelKind, labelName := range lc.Map() {
-		if labelName != "" {
-			switch labelKind {
-			case "namespace_external_label":
-				labels["namespace"] = labelName
-			case "cluster_external_label":
-				labels["cluster"] = labelName
-			case "controller_external_label":
-				labels["controller"] = labelName
-			case "product_external_label":
-				labels["product"] = labelName
-			case "service_external_label":
-				labels["service"] = labelName
-			case "deployment_external_label":
-				labels["deployment"] = labelName
-			case "statefulset_external_label":
-				labels["statefulset"] = labelName
-			case "daemonset_external_label":
-				labels["daemonset"] = labelName
-			case "pod_external_label":
-				labels["pod"] = labelName
-			default:
-				labels[fmt.Sprintf("label:%s", labelName)] = labelName
+	// The relevant label is not present in the set of labels provided.
+	labelName := ""
+	labelValue := ""
+	for _, ln := range labelNames {
+		if lv, ok := labels[ln]; ok {
+			// Match found for given label
+			labelName = ln
+			labelValue = lv
+			break
+		} else {
+			// Convert the label name to a format compatible with AWS Glue and
+			// Athena column naming and check again. If not found after that,
+			// then consider the label not present.
+			ln = cloudutil.ConvertToGlueColumnFormat(ln)
+			if lv, ok = labels[ln]; ok {
+				// Match found for given label after converting to AWS format
+				labelName = ln
+				labelValue = lv
+				break
 			}
 		}
 	}
 
-	return labels
+	// No match found
+	if labelName == "" {
+		return ""
+	}
+
+	// When aggregating by some label (i.e. not by a Kubernetes concept),
+	// prepend the label value with the label name (e.g. "app=cost-analyzer").
+	// This step is not necessary for Kubernetes concepts (e.g. for namespace,
+	// we do not need "namespace=kubecost"; just "kubecost" will do).
+	if aggByLabel {
+		return fmt.Sprintf("%s=%s", labelName, labelValue)
+	}
+
+	return labelValue
 }

pkg/kubecost/config_test.go

@@ -1,6 +1,10 @@
 package kubecost
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/kubecost/cost-model/pkg/util/cloudutil"
+)
 
 func TestLabelConfig_Map(t *testing.T) {
 	var m map[string]string
@@ -32,63 +36,117 @@ func TestLabelConfig_Map(t *testing.T) {
 	}
 }
 
-func TestLabelConfig_ExternalQueryLabels(t *testing.T) {
-	var qls map[string]string
-	var lc *LabelConfig
+func TestLabelConfig_GetExternalAllocationName(t *testing.T) {
+	// Make sure that AWS's Glue/Athena column formatting is supported
+	glueFormattedLabel := cloudutil.ConvertToGlueColumnFormat("Non__GlueFormattedLabel")
 
-	qls = lc.ExternalQueryLabels()
-	if len(qls) != 13 {
-		t.Fatalf("ExternalQueryLabels: expected length %d; got length %d", 13, len(qls))
-	}
-	if val, ok := qls["kubernetes_deployment"]; !ok || val != "deployment_external_label" {
-		t.Fatalf("ExternalQueryLabels: expected %s; got %s", "deployment_external_label", val)
-	}
-	if val, ok := qls["kubernetes_namespace"]; !ok || val != "namespace_external_label" {
-		t.Fatalf("ExternalQueryLabels: expected %s; got %s", "namespace_external_label", val)
+	labels := map[string]string{
+		"kubens":                      "kubecost-staging",
+		"kubeowner":                   "kubecost-owner",
+		"env":                         "env1",
+		"app":                         "app1",
+		"team":                        "team1",
+		glueFormattedLabel:            "glue",
+		"prom_sanitization_test":      "pass",
+		"kubernetes_cluster":          "cluster-one",
+		"kubernetes_namespace":        "kubecost",
+		"kubernetes_controller":       "kubecost-controller",
+		"kubernetes_daemonset":        "kubecost-daemonset",
+		"kubernetes_deployment":       "kubecost-deployment",
+		"kubernetes_statefulset":      "kubecost-statefulset",
+		"kubernetes_service":          "kubecost-service",
+		"kubernetes_pod":              "kubecost-cost-analyzer-abc123",
+		"kubernetes_label_department": "kubecost-department",
+		"kubernetes_label_env":        "kubecost-env",
+		"kubernetes_label_owner":      "kubecost-owner",
+		"kubernetes_label_app":        "kubecost-app",
+		"kubernetes_label_team":       "kubecost-team",
 	}
 
-	lc = &LabelConfig{
-		DaemonsetExternalLabel: "kubernetes_ds",
-	}
-	qls = lc.ExternalQueryLabels()
-	if len(qls) != 13 {
-		t.Fatalf("ExternalQueryLabels: expected length %d; got length %d", 13, len(qls))
-	}
-	if val, ok := qls["kubernetes_ds"]; !ok || val != "daemonset_external_label" {
-		t.Fatalf("ExternalQueryLabels: expected %s; got %s", "daemonset_external_label", val)
+	testCases := []struct {
+		aggBy    string
+		expected string
+	}{
+		{"label:env", "env=env1"},
+		{"label:app", "app=app1"},
+		{"label:Non__GlueFormattedLabel", "non_glue_formatted_label=glue"},
+		{"cluster", "cluster-one"},
+		{"namespace", "kubecost"},
+		{"controller", "kubecost-controller"},
+		{"daemonset", "kubecost-daemonset"},
+		{"deployment", "kubecost-deployment"},
+		{"statefulset", "kubecost-statefulset"},
+		{"service", "kubecost-service"},
+		{"pod", "kubecost-cost-analyzer-abc123"},
+		{"pod", "kubecost-cost-analyzer-abc123"},
+		{"notathing", ""},
+		{"", ""},
 	}
-	if val, ok := qls["kubernetes_namespace"]; !ok || val != "namespace_external_label" {
-		t.Fatalf("ExternalQueryLabels: expected %s; got %s", "namespace_external_label", val)
-	}
-}
 
-func TestTestLabelConfig_AllocationPropertyLabels(t *testing.T) {
-	var labels map[string]string
 	var lc *LabelConfig
 
-	labels = lc.AllocationPropertyLabels()
-	if len(labels) != 18 {
-		t.Fatalf("AllocationPropertyLabels: expected length %d; got length %d", 18, len(labels))
-	}
-	if val, ok := labels["namespace"]; !ok || val != "kubernetes_namespace" {
-		t.Fatalf("AllocationPropertyLabels: expected %s; got %s", "kubernetes_namespace", val)
-	}
-	if val, ok := labels["label:env"]; !ok || val != "env" {
-		t.Fatalf("AllocationPropertyLabels: expected %s; got %s", "env", val)
+	// If lc is nil, everything should still work off of defaults.
+	for _, tc := range testCases {
+		actual := lc.GetExternalAllocationName(labels, tc.aggBy)
+		if actual != tc.expected {
+			t.Fatalf("GetExternalAllocationName failed; expected '%s'; got '%s'", tc.expected, actual)
+		}
 	}
 
-	lc = &LabelConfig{
-		NamespaceExternalLabel: "kubens",
-		EnvironmentLabel:       "kubeenv",
+	// If lc is default, everything should work, just like the nil.
+	lc = NewLabelConfig()
+	for _, tc := range testCases {
+		actual := lc.GetExternalAllocationName(labels, tc.aggBy)
+		if actual != tc.expected {
+			t.Fatalf("GetExternalAllocationName failed; expected '%s'; got '%s'", tc.expected, actual)
+		}
 	}
-	labels = lc.AllocationPropertyLabels()
-	if len(labels) != 18 {
-		t.Fatalf("AllocationPropertyLabels: expected length %d; got length %d", 18, len(labels))
+
+	// Change the external label for namespace and confirm it still works
+	lc.NamespaceExternalLabel = "kubens"
+	lc.ServiceExternalLabel = "prom/sanitization-test"
+	lc.PodExternalLabel = "Non__GlueFormattedLabel"
+	lc.OwnerExternalLabel = "kubeowner"
+	lc.DepartmentExternalLabel = "doesntexist,env"
+	lc.TeamExternalLabel = "team,env"
+
+	// TODO how is e.g. OwnerExternalLabel supposed to work?
+
+	testCases = []struct {
+		aggBy    string
+		expected string
+	}{
+		{"namespace", "kubecost-staging"},
+		{"service", "pass"},
+		{"pod", "glue"},
+		{"owner", "kubecost-owner"},
+		{"department", "env1"},
+		{"team", "team1"},
+	}
+	for _, tc := range testCases {
+		actual := lc.GetExternalAllocationName(labels, tc.aggBy)
+		if actual != tc.expected {
+			t.Fatalf("GetExternalAllocationName failed; expected '%s'; got '%s'", tc.expected, actual)
+		}
 	}
-	if val, ok := labels["namespace"]; !ok || val != "kubens" {
-		t.Fatalf("AllocationPropertyLabels: expected %s; got %s", "kubens", val)
+}
+
+func TestLabelConfig_Sanitize(t *testing.T) {
+	testCases := []struct {
+		label    string
+		expected string
+	}{
+		{"", ""},
+		{"simple", "simple"},
+		{"prom/sanitization-test", "prom_sanitization_test"},
+		{" prom/sanitization-test$  ", "prom_sanitization_test_"},
 	}
-	if val, ok := labels["label:kubeenv"]; !ok || val != "kubeenv" {
-		t.Fatalf("AllocationPropertyLabels: expected %s; got %s", "kubeenv", val)
+
+	lc := NewLabelConfig()
+	for _, tc := range testCases {
+		actual := lc.Sanitize(tc.label)
+		if actual != tc.expected {
+			t.Fatalf("Sanitize failed; expected '%s'; got '%s'", tc.expected, actual)
+		}
 	}
 }

pkg/kubecost/mock.go

@@ -4,8 +4,10 @@ import (
 	"fmt"
 	"time"
 )
+
 const gb = 1024 * 1024 * 1024
 const day = 24 * time.Hour
+
 var disk = PVKey{}
 
 // NewMockUnitAllocation creates an *Allocation with all of its float64 values set to 1 and generic properties if not provided in arg
@@ -304,7 +306,7 @@ func GenerateMockAllocationSet(start time.Time) *AllocationSet {
 }
 
 // GenerateMockAllocationSetWithAssetProperties with no idle and connections to Assets in properties
-func GenerateMockAllocationSetWithAssetProperties(start time.Time)  *AllocationSet {
+func GenerateMockAllocationSetWithAssetProperties(start time.Time) *AllocationSet {
 	as := GenerateMockAllocationSet(start)
 	disk1 := PVKey{
 		Cluster: "cluster2",

pkg/kubecost/status.go

@@ -29,7 +29,37 @@ type FileStatus struct {
 	Name         string            `json:"name"`
 	Size         string            `json:"size"`
 	LastModified time.Time         `json:"lastModified"`
+	IsRepairing  bool              `json:"isRepairing"`
 	Details      map[string]string `json:"details,omitempty"`
 	Errors       []string          `json:"errors,omitempty"`
 	Warnings     []string          `json:"warnings,omitempty"`
 }
+
+// CloudStatus describes CloudStore metadata
+type CloudStatus struct {
+	CloudConnectionStatus string                `json:"cloudConnectionStatus"`
+	CloudAssets           *CloudAssetStatus     `json:"cloudAssets,omitempty"`
+	Reconciliation        *ReconciliationStatus `json:"reconciliation,omitempty"`
+}
+
+// CloudAssetStatus describes CloudAsset metadata of a CloudStore
+type CloudAssetStatus struct {
+	Coverage    Window    `json:"coverage"`
+	LastRun     time.Time `json:"lastRun"`
+	NextRun     time.Time `json:"nextRun"`
+	Progress    float64   `json:"progress"`
+	RefreshRate string    `json:"refreshRate"`
+	Resolution  string    `json:"resolution"`
+	StartTime   time.Time `json:"startTime"`
+}
+
+// ReconciliationStatus describes Reconciliation metadata of a CloudStore
+type ReconciliationStatus struct {
+	Coverage    Window    `json:"coverage"`
+	LastRun     time.Time `json:"lastRun"`
+	NextRun     time.Time `json:"nextRun"`
+	Progress    float64   `json:"progress"`
+	RefreshRate string    `json:"refreshRate"`
+	Resolution  string    `json:"resolution"`
+	StartTime   time.Time `json:"startTime"`
+}

pkg/kubecost/window.go

@@ -3,6 +3,7 @@ package kubecost
 import (
 	"bytes"
 	"fmt"
+	"github.com/kubecost/cost-model/pkg/util/timeutil"
 	"math"
 	"regexp"
 	"strconv"
@@ -10,7 +11,6 @@ import (
 
 	"github.com/kubecost/cost-model/pkg/env"
 	"github.com/kubecost/cost-model/pkg/thanos"
-	"github.com/kubecost/cost-model/pkg/util"
 )
 
 const (
@@ -572,7 +572,7 @@ func (w Window) DurationOffset() (time.Duration, time.Duration, error) {
 	}
 
 	duration := w.Duration()
-	offset := time.Now().Sub(*w.End())
+	offset := time.Since(*w.End())
 
 	return duration, offset, nil
 }
@@ -611,7 +611,7 @@ func (w Window) DurationOffsetForPrometheus() (string, string, error) {
 		offset = 0
 	}
 
-	durStr, offStr := util.DurationOffsetStrings(duration, offset)
+	durStr, offStr := timeutil.DurationOffsetStrings(duration, offset)
 	if offset < time.Minute {
 		offStr = ""
 	} else {
@@ -630,7 +630,7 @@ func (w Window) DurationOffsetStrings() (string, string) {
 		return "", ""
 	}
 
-	return util.DurationOffsetStrings(dur, off)
+	return timeutil.DurationOffsetStrings(dur, off)
 }
 
 type BoundaryError struct {

pkg/kubecost/window_test.go

@@ -598,7 +598,7 @@ func TestWindow_DurationOffsetStrings(t *testing.T) {
 	if err != nil {
 		t.Fatalf(`unexpected error parsing "1589448338,1589534798": %s`, err)
 	}
-	dur, off = w.DurationOffsetStrings()
+	dur, _ = w.DurationOffsetStrings()
 	if dur != "1d" {
 		t.Fatalf(`expect: window to be "1d"; actual: "%s"`, dur)
 	}

pkg/log/log.go

@@ -63,7 +63,7 @@ func Profilef(format string, a ...interface{}) {
 }
 
 func Debugf(format string, a ...interface{}) {
-	klog.V(4).Infof(fmt.Sprintf("[Debug] %s", format), a...)
+	klog.V(5).Infof(fmt.Sprintf("[Debug] %s", format), a...)
 }
 
 func Profile(start time.Time, name string) {

pkg/metrics/deploymentmetrics.go

@@ -0,0 +1,255 @@
+package metrics
+
+import (
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/prom"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+)
+
+//--------------------------------------------------------------------------
+//  KubecostDeploymentCollector
+//--------------------------------------------------------------------------
+
+// KubecostDeploymentCollector is a prometheus collector that generates kubecost
+// specific deployment metrics.
+type KubecostDeploymentCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (kdc KubecostDeploymentCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("deployment_match_labels", "deployment match labels", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kdc KubecostDeploymentCollector) Collect(ch chan<- prometheus.Metric) {
+	ds := kdc.KubeClusterCache.GetAllDeployments()
+
+	for _, deployment := range ds {
+		deploymentName := deployment.GetName()
+		deploymentNS := deployment.GetNamespace()
+
+		labels, values := prom.KubeLabelsToLabels(deployment.Spec.Selector.MatchLabels)
+		if len(labels) > 0 {
+			m := newDeploymentMatchLabelsMetric(deploymentName, deploymentNS, "deployment_match_labels", labels, values)
+			ch <- m
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  DeploymentMatchLabelsMetric
+//--------------------------------------------------------------------------
+
+// DeploymentMatchLabelsMetric is a prometheus.Metric used to encode deployment match labels
+type DeploymentMatchLabelsMetric struct {
+	fqName         string
+	help           string
+	labelNames     []string
+	labelValues    []string
+	deploymentName string
+	namespace      string
+}
+
+// Creates a new DeploymentMatchLabelsMetric, implementation of prometheus.Metric
+func newDeploymentMatchLabelsMetric(name, namespace, fqname string, labelNames, labelvalues []string) DeploymentMatchLabelsMetric {
+	return DeploymentMatchLabelsMetric{
+		fqName:         fqname,
+		labelNames:     labelNames,
+		labelValues:    labelvalues,
+		help:           "deployment_match_labels Deployment Match Labels",
+		deploymentName: name,
+		namespace:      namespace,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (dmlm DeploymentMatchLabelsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"deployment": dmlm.deploymentName,
+		"namespace":  dmlm.namespace,
+	}
+	return prometheus.NewDesc(dmlm.fqName, dmlm.help, dmlm.labelNames, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (dmlm DeploymentMatchLabelsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+	var labels []*dto.LabelPair
+	for i := range dmlm.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &dmlm.labelNames[i],
+			Value: &dmlm.labelValues[i],
+		})
+	}
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("namespace"),
+		Value: &dmlm.namespace,
+	})
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("deployment"),
+		Value: &dmlm.deploymentName,
+	})
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeDeploymentCollector
+//--------------------------------------------------------------------------
+
+// KubeDeploymentCollector is a prometheus collector that generates
+type KubeDeploymentCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (kdc KubeDeploymentCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_deployment_spec_replicas", "Number of desired pods for a deployment.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_deployment_status_replicas_available", "The number of available replicas per deployment.", []string{}, nil)
+
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kdc KubeDeploymentCollector) Collect(ch chan<- prometheus.Metric) {
+	deployments := kdc.KubeClusterCache.GetAllDeployments()
+
+	for _, deployment := range deployments {
+		deploymentName := deployment.GetName()
+		deploymentNS := deployment.GetNamespace()
+
+		// Replicas Defined
+		var replicas int32
+		if deployment.Spec.Replicas == nil {
+			replicas = 1 // defaults to 1, documented on the 'Replicas' field
+		} else {
+			replicas = *deployment.Spec.Replicas
+		}
+
+		ch <- newKubeDeploymentReplicasMetric("kube_deployment_spec_replicas", deploymentName, deploymentNS, replicas)
+
+		// Replicas Available
+		ch <- newKubeDeploymentStatusAvailableReplicasMetric(
+			"kube_deployment_status_replicas_available",
+			deploymentName,
+			deploymentNS,
+			deployment.Status.AvailableReplicas)
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubeDeploymentReplicasMetric
+//--------------------------------------------------------------------------
+
+// KubeDeploymentReplicasMetric is a prometheus.Metric used to encode deployment match labels
+type KubeDeploymentReplicasMetric struct {
+	fqName     string
+	help       string
+	deployment string
+	namespace  string
+	replicas   float64
+}
+
+// Creates a new DeploymentMatchLabelsMetric, implementation of prometheus.Metric
+func newKubeDeploymentReplicasMetric(fqname, deployment, namespace string, replicas int32) KubeDeploymentReplicasMetric {
+	return KubeDeploymentReplicasMetric{
+		fqName:     fqname,
+		help:       "kube_deployment_spec_replicas Number of desired pods for a deployment.",
+		deployment: deployment,
+		namespace:  namespace,
+		replicas:   float64(replicas),
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kdr KubeDeploymentReplicasMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"deployment": kdr.deployment,
+		"namespace":  kdr.namespace,
+	}
+	return prometheus.NewDesc(kdr.fqName, kdr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kdr KubeDeploymentReplicasMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kdr.replicas,
+	}
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kdr.namespace,
+		},
+		{
+			Name:  toStringPtr("deployment"),
+			Value: &kdr.deployment,
+		},
+	}
+
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeDeploymentStatusAvailableReplicasMetric
+//--------------------------------------------------------------------------
+
+// KubeDeploymentStatusAvailableReplicasMetric is a prometheus.Metric used to encode deployment match labels
+type KubeDeploymentStatusAvailableReplicasMetric struct {
+	fqName            string
+	help              string
+	deployment        string
+	namespace         string
+	replicasAvailable float64
+}
+
+// Creates a new DeploymentMatchLabelsMetric, implementation of prometheus.Metric
+func newKubeDeploymentStatusAvailableReplicasMetric(fqname, deployment, namespace string, replicasAvailable int32) KubeDeploymentStatusAvailableReplicasMetric {
+	return KubeDeploymentStatusAvailableReplicasMetric{
+		fqName:            fqname,
+		help:              "kube_deployment_status_replicas_available The number of available replicas per deployment.",
+		deployment:        deployment,
+		namespace:         namespace,
+		replicasAvailable: float64(replicasAvailable),
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kdr KubeDeploymentStatusAvailableReplicasMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"deployment": kdr.deployment,
+		"namespace":  kdr.namespace,
+	}
+	return prometheus.NewDesc(kdr.fqName, kdr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kdr KubeDeploymentStatusAvailableReplicasMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kdr.replicasAvailable,
+	}
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kdr.namespace,
+		},
+		{
+			Name:  toStringPtr("deployment"),
+			Value: &kdr.deployment,
+		},
+	}
+
+	return nil
+}

pkg/metrics/jobmetrics.go

@@ -0,0 +1,116 @@
+package metrics
+
+import (
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	batchv1 "k8s.io/api/batch/v1"
+)
+
+var (
+	jobFailureReasons = []string{"BackoffLimitExceeded", "DeadLineExceeded", "Evicted"}
+)
+
+//--------------------------------------------------------------------------
+//  KubeJobCollector
+//--------------------------------------------------------------------------
+
+// KubeJobCollector is a prometheus collector that generates job sourced metrics.
+type KubeJobCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (kjc KubeJobCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_job_status_failed", "The number of pods which reached Phase Failed and the reason for failure.", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kjc KubeJobCollector) Collect(ch chan<- prometheus.Metric) {
+	jobs := kjc.KubeClusterCache.GetAllJobs()
+	for _, job := range jobs {
+		jobName := job.GetName()
+		jobNS := job.GetNamespace()
+
+		if job.Status.Failed == 0 {
+			ch <- newKubeJobStatusFailedMetric(jobName, jobNS, "kube_job_status_failed", "", 0)
+		} else {
+			for _, condition := range job.Status.Conditions {
+				if condition.Type == batchv1.JobFailed {
+					reasonKnown := false
+					for _, reason := range jobFailureReasons {
+						reasonKnown = reasonKnown || failureReason(&condition, reason)
+
+						ch <- newKubeJobStatusFailedMetric(jobName, jobNS, "kube_job_status_failed", reason, boolFloat64(failureReason(&condition, reason)))
+					}
+
+					// for unknown reasons
+					if !reasonKnown {
+						ch <- newKubeJobStatusFailedMetric(jobName, jobNS, "kube_job_status_failed", "", float64(job.Status.Failed))
+					}
+				}
+			}
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubeJobStatusFailedMetric
+//--------------------------------------------------------------------------
+
+// KubeJobStatusFailedMetric
+type KubeJobStatusFailedMetric struct {
+	fqName    string
+	help      string
+	job       string
+	namespace string
+	reason    string
+	value     float64
+}
+
+// Creates a new KubeJobStatusFailedMetric, implementation of prometheus.Metric
+func newKubeJobStatusFailedMetric(job, namespace, fqName, reason string, value float64) KubeJobStatusFailedMetric {
+	return KubeJobStatusFailedMetric{
+		fqName:    fqName,
+		help:      "kube_job_status_failed Failed job",
+		job:       job,
+		namespace: namespace,
+		reason:    reason,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kjsf KubeJobStatusFailedMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"job_name":  kjsf.job,
+		"namespace": kjsf.namespace,
+		"reason":    kjsf.reason,
+	}
+	return prometheus.NewDesc(kjsf.fqName, kjsf.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kjsf KubeJobStatusFailedMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kjsf.value,
+	}
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("job_name"),
+			Value: &kjsf.job,
+		},
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kjsf.namespace,
+		},
+		{
+			Name:  toStringPtr("reason"),
+			Value: &kjsf.reason,
+		},
+	}
+	return nil
+}

pkg/metrics/kubemetrics.go

@@ -0,0 +1,211 @@
+package metrics
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/prom"
+
+	"github.com/prometheus/client_golang/prometheus"
+	batchv1 "k8s.io/api/batch/v1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/validation"
+)
+
+//--------------------------------------------------------------------------
+//  Kube Metric Registration
+//--------------------------------------------------------------------------
+
+// initializer
+var kubeMetricInit sync.Once
+
+// KubeMetricsOpts represents our Kubernetes metrics emission options.
+type KubeMetricsOpts struct {
+	EmitKubecostControllerMetrics bool
+	EmitNamespaceAnnotations      bool
+	EmitPodAnnotations            bool
+	EmitKubeStateMetrics          bool
+}
+
+// DefaultKubeMetricsOpts returns KubeMetricsOpts with default values set
+func DefaultKubeMetricsOpts() *KubeMetricsOpts {
+	return &KubeMetricsOpts{
+		EmitKubecostControllerMetrics: true,
+		EmitNamespaceAnnotations:      false,
+		EmitPodAnnotations:            false,
+		EmitKubeStateMetrics:          true,
+	}
+}
+
+// InitKubeMetrics initializes kubernetes metric emission using the provided options.
+func InitKubeMetrics(clusterCache clustercache.ClusterCache, opts *KubeMetricsOpts) {
+	if opts == nil {
+		opts = DefaultKubeMetricsOpts()
+	}
+
+	kubeMetricInit.Do(func() {
+		if opts.EmitKubecostControllerMetrics {
+			prometheus.MustRegister(KubecostServiceCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubecostDeploymentCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubecostStatefulsetCollector{
+				KubeClusterCache: clusterCache,
+			})
+		}
+
+		if opts.EmitPodAnnotations {
+			prometheus.MustRegister(KubecostPodCollector{
+				KubeClusterCache: clusterCache,
+			})
+		}
+
+		if opts.EmitNamespaceAnnotations {
+			prometheus.MustRegister(KubecostNamespaceCollector{
+				KubeClusterCache: clusterCache,
+			})
+		}
+
+		if opts.EmitKubeStateMetrics {
+			prometheus.MustRegister(KubeNodeCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubeNamespaceCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubeDeploymentCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubePodCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubePVCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubePVCCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubeJobCollector{
+				KubeClusterCache: clusterCache,
+			})
+		}
+	})
+}
+
+//--------------------------------------------------------------------------
+//  Kube Metric Helpers
+//--------------------------------------------------------------------------
+
+// getPersistentVolumeClaimClass returns StorageClassName. If no storage class was
+// requested, it returns "".
+func getPersistentVolumeClaimClass(claim *v1.PersistentVolumeClaim) string {
+	// Use beta annotation first
+	if class, found := claim.Annotations[v1.BetaStorageClassAnnotation]; found {
+		return class
+	}
+
+	if claim.Spec.StorageClassName != nil {
+		return *claim.Spec.StorageClassName
+	}
+
+	// Special non-empty string to indicate absence of storage class.
+	return "<none>"
+}
+
+// toResourceUnitValue accepts a resource name and quantity and returns the sanitized resource, the unit, and the value in the units.
+// Returns an empty string for resource and unit if there was a failure.
+func toResourceUnitValue(resourceName v1.ResourceName, quantity resource.Quantity) (resource string, unit string, value float64) {
+	resource = prom.SanitizeLabelName(string(resourceName))
+
+	switch resourceName {
+	case v1.ResourceCPU:
+		unit = "core"
+		value = float64(quantity.MilliValue()) / 1000
+		return
+
+	case v1.ResourceStorage:
+		fallthrough
+	case v1.ResourceEphemeralStorage:
+		fallthrough
+	case v1.ResourceMemory:
+		unit = "byte"
+		value = float64(quantity.Value())
+		return
+	case v1.ResourcePods:
+		unit = "integer"
+		value = float64(quantity.Value())
+		return
+	default:
+		if isHugePageResourceName(resourceName) || isAttachableVolumeResourceName(resourceName) {
+			unit = "byte"
+			value = float64(quantity.Value())
+			return
+		}
+
+		if isExtendedResourceName(resourceName) {
+			unit = "integer"
+			value = float64(quantity.Value())
+			return
+		}
+	}
+
+	resource = ""
+	unit = ""
+	value = 0.0
+	return
+}
+
+// isHugePageResourceName checks for a huge page container resource name
+func isHugePageResourceName(name v1.ResourceName) bool {
+	return strings.HasPrefix(string(name), v1.ResourceHugePagesPrefix)
+}
+
+// isAttachableVolumeResourceName checks for attached volume container resource name
+func isAttachableVolumeResourceName(name v1.ResourceName) bool {
+	return strings.HasPrefix(string(name), v1.ResourceAttachableVolumesPrefix)
+}
+
+// isExtendedResourceName checks for extended container resource name
+func isExtendedResourceName(name v1.ResourceName) bool {
+	if isNativeResource(name) || strings.HasPrefix(string(name), v1.DefaultResourceRequestsPrefix) {
+		return false
+	}
+	// Ensure it satisfies the rules in IsQualifiedName() after converted into quota resource name
+	nameForQuota := fmt.Sprintf("%s%s", v1.DefaultResourceRequestsPrefix, string(name))
+	if errs := validation.IsQualifiedName(nameForQuota); len(errs) != 0 {
+		return false
+	}
+	return true
+}
+
+// isNativeResource checks for a kubernetes.io/ prefixed resource name
+func isNativeResource(name v1.ResourceName) bool {
+	return !strings.Contains(string(name), "/") || isPrefixedNativeResource(name)
+}
+
+func isPrefixedNativeResource(name v1.ResourceName) bool {
+	return strings.Contains(string(name), v1.ResourceDefaultNamespacePrefix)
+}
+
+func failureReason(jc *batchv1.JobCondition, reason string) bool {
+	if jc == nil {
+		return false
+	}
+	return jc.Reason == reason
+}
+
+// boolFloat64 converts a boolean input into a 1 or 0
+func boolFloat64(b bool) float64 {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+// toStringPtr is used to create a new string pointer from iteration vars
+func toStringPtr(s string) *string { return &s }

pkg/metrics/namespacemetrics.go

@@ -0,0 +1,178 @@
+package metrics
+
+import (
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/prom"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+)
+
+//--------------------------------------------------------------------------
+//  KubecostNamespaceCollector
+//--------------------------------------------------------------------------
+
+// KubecostNamespaceCollector is a prometheus collector that generates namespace sourced metrics
+type KubecostNamespaceCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (nsac KubecostNamespaceCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_namespace_annotations", "namespace annotations", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (nsac KubecostNamespaceCollector) Collect(ch chan<- prometheus.Metric) {
+	namespaces := nsac.KubeClusterCache.GetAllNamespaces()
+	for _, namespace := range namespaces {
+		nsName := namespace.GetName()
+
+		labels, values := prom.KubeAnnotationsToLabels(namespace.Annotations)
+		if len(labels) > 0 {
+			m := newNamespaceAnnotationsMetric("kube_namespace_annotations", nsName, labels, values)
+			ch <- m
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  NamespaceAnnotationsMetric
+//--------------------------------------------------------------------------
+
+// NamespaceAnnotationsMetric is a prometheus.Metric used to encode namespace annotations
+type NamespaceAnnotationsMetric struct {
+	fqName      string
+	help        string
+	namespace   string
+	labelNames  []string
+	labelValues []string
+}
+
+// Creates a new NamespaceAnnotationsMetric, implementation of prometheus.Metric
+func newNamespaceAnnotationsMetric(fqname, namespace string, labelNames []string, labelValues []string) NamespaceAnnotationsMetric {
+	return NamespaceAnnotationsMetric{
+		fqName:      fqname,
+		help:        "kube_namespace_annotations Namespace Annotations",
+		namespace:   namespace,
+		labelNames:  labelNames,
+		labelValues: labelValues,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam NamespaceAnnotationsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": nam.namespace,
+	}
+	return prometheus.NewDesc(nam.fqName, nam.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (nam NamespaceAnnotationsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	for i := range nam.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &nam.labelNames[i],
+			Value: &nam.labelValues[i],
+		})
+	}
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("namespace"),
+		Value: &nam.namespace,
+	})
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeNamespaceCollector
+//--------------------------------------------------------------------------
+
+// KubeNamespaceCollector is a prometheus collector that generates namespace sourced metrics
+type KubeNamespaceCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (nsac KubeNamespaceCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_namespace_labels", "namespace labels", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (nsac KubeNamespaceCollector) Collect(ch chan<- prometheus.Metric) {
+	namespaces := nsac.KubeClusterCache.GetAllNamespaces()
+	for _, namespace := range namespaces {
+		nsName := namespace.GetName()
+
+		labels, values := prom.KubeLabelsToLabels(namespace.Labels)
+		if len(labels) > 0 {
+			m := newNamespaceAnnotationsMetric("kube_namespace_labels", nsName, labels, values)
+			ch <- m
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  NamespaceAnnotationsMetric
+//--------------------------------------------------------------------------
+
+// NamespaceAnnotationsMetric is a prometheus.Metric used to encode namespace annotations
+type KubeNamespaceLabelsMetric struct {
+	fqName      string
+	help        string
+	namespace   string
+	labelNames  []string
+	labelValues []string
+}
+
+// Creates a new KubeNamespaceLabelsMetric, implementation of prometheus.Metric
+func newKubeNamespaceLabelsMetric(fqname, namespace string, labelNames []string, labelValues []string) KubeNamespaceLabelsMetric {
+	return KubeNamespaceLabelsMetric{
+		namespace:   namespace,
+		fqName:      fqname,
+		labelNames:  labelNames,
+		labelValues: labelValues,
+		help:        "kube_namespace_labels Namespace Labels",
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubeNamespaceLabelsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": nam.namespace,
+	}
+	return prometheus.NewDesc(nam.fqName, nam.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (nam KubeNamespaceLabelsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	for i := range nam.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &nam.labelNames[i],
+			Value: &nam.labelValues[i],
+		})
+	}
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("namespace"),
+		Value: &nam.namespace,
+	})
+	m.Label = labels
+	return nil
+}

pkg/metrics/nodemetrics.go

@@ -0,0 +1,558 @@
+package metrics
+
+import (
+	"strings"
+
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/log"
+	"github.com/kubecost/cost-model/pkg/prom"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	v1 "k8s.io/api/core/v1"
+)
+
+var (
+	conditionStatuses = []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionFalse, v1.ConditionUnknown}
+)
+
+//--------------------------------------------------------------------------
+//  KubeNodeCollector
+//--------------------------------------------------------------------------
+
+// KubeNodeCollector is a prometheus collector that generates node sourced metrics.
+type KubeNodeCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (nsac KubeNodeCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_node_status_capacity", "Node resource capacity.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_node_status_capacity_memory_bytes", "node capacity memory bytes", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_node_status_capacity_cpu_cores", "node capacity cpu cores", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_node_status_allocatable", "The allocatable for different resources of a node that are available for scheduling.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_node_status_allocatable_cpu_cores", "The allocatable cpu cores.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_node_status_allocatable_memory_bytes", "The allocatable memory in bytes.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_node_labels", "all labels for each node prefixed with label_", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_node_status_condition", "The condition of a cluster node.", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (nsac KubeNodeCollector) Collect(ch chan<- prometheus.Metric) {
+	nodes := nsac.KubeClusterCache.GetAllNodes()
+	for _, node := range nodes {
+		nodeName := node.GetName()
+
+		// Node Capacity
+		for resourceName, quantity := range node.Status.Capacity {
+			resource, unit, value := toResourceUnitValue(resourceName, quantity)
+
+			// failed to parse the resource type
+			if resource == "" {
+				log.DedupedWarningf(5, "Failed to parse resource units and quantity for resource: %s", resourceName)
+				continue
+			}
+
+			// KSM v1 Emission
+			if resource == "cpu" {
+				ch <- newKubeNodeStatusCapacityCPUCoresMetric("kube_node_status_capacity_cpu_cores", nodeName, value)
+
+			}
+			if resource == "memory" {
+				ch <- newKubeNodeStatusCapacityMemoryBytesMetric("kube_node_status_capacity_memory_bytes", nodeName, value)
+			}
+
+			ch <- newKubeNodeStatusCapacityMetric("kube_node_status_capacity", nodeName, resource, unit, value)
+		}
+
+		// Node Allocatable Resources
+		for resourceName, quantity := range node.Status.Allocatable {
+			resource, unit, value := toResourceUnitValue(resourceName, quantity)
+
+			// failed to parse the resource type
+			if resource == "" {
+				log.DedupedWarningf(5, "Failed to parse resource units and quantity for resource: %s", resourceName)
+				continue
+			}
+
+			// KSM v1 Emission
+			if resource == "cpu" {
+				ch <- newKubeNodeStatusAllocatableCPUCoresMetric("kube_node_status_allocatable_cpu_cores", nodeName, value)
+
+			}
+			if resource == "memory" {
+				ch <- newKubeNodeStatusAllocatableMemoryBytesMetric("kube_node_status_allocatable_memory_bytes", nodeName, value)
+			}
+
+			ch <- newKubeNodeStatusAllocatableMetric("kube_node_status_allocatable", nodeName, resource, unit, value)
+		}
+
+		// node labels
+		labelNames, labelValues := prom.KubePrependQualifierToLabels(node.GetLabels(), "label_")
+		ch <- newKubeNodeLabelsMetric(nodeName, "kube_node_labels", labelNames, labelValues)
+
+		// kube_node_status_condition
+		// Collect node conditions and while default to false.
+		for _, c := range node.Status.Conditions {
+			conditions := getConditions(c.Status)
+
+			for _, cond := range conditions {
+				ch <- newKubeNodeStatusConditionMetric(nodeName, "kube_node_status_condition", string(c.Type), cond.status, cond.value)
+			}
+		}
+
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusCapacityMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusCapacityMetric is a prometheus.Metric
+type KubeNodeStatusCapacityMetric struct {
+	fqName   string
+	help     string
+	resource string
+	unit     string
+	node     string
+	value    float64
+}
+
+// Creates a new KubeNodeStatusCapacityMetric, implementation of prometheus.Metric
+func newKubeNodeStatusCapacityMetric(fqname, node, resource, unit string, value float64) KubeNodeStatusCapacityMetric {
+	return KubeNodeStatusCapacityMetric{
+		fqName:   fqname,
+		help:     "kube_node_status_capacity node capacity",
+		node:     node,
+		resource: resource,
+		unit:     unit,
+		value:    value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubeNodeStatusCapacityMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"node":     kpcrr.node,
+		"resource": kpcrr.resource,
+		"unit":     kpcrr.unit,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (kpcrr KubeNodeStatusCapacityMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("node"),
+			Value: &kpcrr.node,
+		},
+		{
+			Name:  toStringPtr("resource"),
+			Value: &kpcrr.resource,
+		},
+		{
+			Name:  toStringPtr("unit"),
+			Value: &kpcrr.unit,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusCapacityMemoryBytesMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusCapacityMemoryBytesMetric is a prometheus.Metric used to encode
+// a duplicate of the deprecated kube-state-metrics metric
+// kube_node_status_capacity_memory_bytes
+type KubeNodeStatusCapacityMemoryBytesMetric struct {
+	fqName string
+	help   string
+	bytes  float64
+	node   string
+}
+
+// Creates a new KubeNodeStatusCapacityMemoryBytesMetric, implementation of prometheus.Metric
+func newKubeNodeStatusCapacityMemoryBytesMetric(fqname string, node string, bytes float64) KubeNodeStatusCapacityMemoryBytesMetric {
+	return KubeNodeStatusCapacityMemoryBytesMetric{
+		fqName: fqname,
+		help:   "kube_node_status_capacity_memory_bytes Node Capacity Memory Bytes",
+		node:   node,
+		bytes:  bytes,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubeNodeStatusCapacityMemoryBytesMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{"node": nam.node}
+	return prometheus.NewDesc(nam.fqName, nam.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (nam KubeNodeStatusCapacityMemoryBytesMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &nam.bytes,
+	}
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("node"),
+			Value: &nam.node,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusCapacityCPUCoresMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusCapacityCPUCoresMetric is a prometheus.Metric used to encode
+// a duplicate of the deprecated kube-state-metrics metric
+// kube_node_status_capacity_memory_bytes
+type KubeNodeStatusCapacityCPUCoresMetric struct {
+	fqName string
+	help   string
+	cores  float64
+	node   string
+}
+
+// Creates a new KubeNodeStatusCapacityCPUCoresMetric, implementation of prometheus.Metric
+func newKubeNodeStatusCapacityCPUCoresMetric(fqname string, node string, cores float64) KubeNodeStatusCapacityCPUCoresMetric {
+	return KubeNodeStatusCapacityCPUCoresMetric{
+		fqName: fqname,
+		help:   "kube_node_status_capacity_cpu_cores Node Capacity CPU Cores",
+		cores:  cores,
+		node:   node,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubeNodeStatusCapacityCPUCoresMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{"node": nam.node}
+	return prometheus.NewDesc(nam.fqName, nam.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (nam KubeNodeStatusCapacityCPUCoresMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &nam.cores,
+	}
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("node"),
+			Value: &nam.node,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeLabelsCollector
+//--------------------------------------------------------------------------
+//
+// We use this to emit kube_node_labels with all of a node's labels, regardless
+// of the whitelist setting introduced in KSM v2. See
+// https://github.com/kubernetes/kube-state-metrics/issues/1270#issuecomment-712986441
+
+//--------------------------------------------------------------------------
+//  KubeNodeLabelsMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeLabelsMetric is a prometheus.Metric used to encode
+// a duplicate of the deprecated kube-state-metrics metric
+// kube_node_labels
+type KubeNodeLabelsMetric struct {
+	fqName      string
+	help        string
+	labelNames  []string
+	labelValues []string
+	node        string
+}
+
+// Creates a new KubeNodeLabelsMetric, implementation of prometheus.Metric
+func newKubeNodeLabelsMetric(node string, fqname string, labelNames []string, labelValues []string) KubeNodeLabelsMetric {
+	return KubeNodeLabelsMetric{
+		fqName:      fqname,
+		labelNames:  labelNames,
+		labelValues: labelValues,
+		help:        "kube_node_labels all labels for each node prefixed with label_",
+		node:        node,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubeNodeLabelsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"node": nam.node,
+	}
+	return prometheus.NewDesc(nam.fqName, nam.help, nam.labelNames, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (nam KubeNodeLabelsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	for i := range nam.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &nam.labelNames[i],
+			Value: &nam.labelValues[i],
+		})
+	}
+
+	nodeString := "node"
+	labels = append(labels, &dto.LabelPair{Name: &nodeString, Value: &nam.node})
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusConditionMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusConditionMetric
+type KubeNodeStatusConditionMetric struct {
+	fqName    string
+	help      string
+	node      string
+	condition string
+	status    string
+	value     float64
+}
+
+// Creates a new KubeNodeStatusConditionMetric, implementation of prometheus.Metric
+func newKubeNodeStatusConditionMetric(node, fqname, condition, status string, value float64) KubeNodeStatusConditionMetric {
+	return KubeNodeStatusConditionMetric{
+		fqName:    fqname,
+		help:      "kube_node_status_condition condition status for nodes",
+		node:      node,
+		condition: condition,
+		status:    status,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubeNodeStatusConditionMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"node":      nam.node,
+		"condition": nam.condition,
+		"status":    nam.status,
+	}
+	return prometheus.NewDesc(nam.fqName, nam.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (nam KubeNodeStatusConditionMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &nam.value,
+	}
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("node"),
+			Value: &nam.node,
+		},
+		{
+			Name:  toStringPtr("condition"),
+			Value: &nam.condition,
+		},
+		{
+			Name:  toStringPtr("status"),
+			Value: &nam.status,
+		},
+	}
+	return nil
+}
+
+// helper type for status condition reporting and metric rollup
+type statusCondition struct {
+	status string
+	value  float64
+}
+
+// retrieves the total status conditions and the comparison to the provided condition
+func getConditions(cs v1.ConditionStatus) []*statusCondition {
+	ms := make([]*statusCondition, len(conditionStatuses))
+
+	for i, status := range conditionStatuses {
+		ms[i] = &statusCondition{
+			status: strings.ToLower(string(status)),
+			value:  boolFloat64(cs == status),
+		}
+	}
+
+	return ms
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusAllocatableMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusAllocatableMetric is a prometheus.Metric
+type KubeNodeStatusAllocatableMetric struct {
+	fqName   string
+	help     string
+	resource string
+	unit     string
+	node     string
+	value    float64
+}
+
+// Creates a new KubeNodeStatusAllocatableMetric, implementation of prometheus.Metric
+func newKubeNodeStatusAllocatableMetric(fqname, node, resource, unit string, value float64) KubeNodeStatusAllocatableMetric {
+	return KubeNodeStatusAllocatableMetric{
+		fqName:   fqname,
+		help:     "kube_node_status_allocatable node allocatable",
+		node:     node,
+		resource: resource,
+		unit:     unit,
+		value:    value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubeNodeStatusAllocatableMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"node":     kpcrr.node,
+		"resource": kpcrr.resource,
+		"unit":     kpcrr.unit,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (kpcrr KubeNodeStatusAllocatableMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("node"),
+			Value: &kpcrr.node,
+		},
+		{
+			Name:  toStringPtr("resource"),
+			Value: &kpcrr.resource,
+		},
+		{
+			Name:  toStringPtr("unit"),
+			Value: &kpcrr.unit,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusAllocatableCPUCoresMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusAllocatableCPUCoresMetric is a prometheus.Metric
+type KubeNodeStatusAllocatableCPUCoresMetric struct {
+	fqName   string
+	help     string
+	resource string
+	unit     string
+	node     string
+	value    float64
+}
+
+// Creates a new KubeNodeStatusAllocatableCPUCoresMetric, implementation of prometheus.Metric
+func newKubeNodeStatusAllocatableCPUCoresMetric(fqname, node string, value float64) KubeNodeStatusAllocatableCPUCoresMetric {
+	return KubeNodeStatusAllocatableCPUCoresMetric{
+		fqName: fqname,
+		help:   "kube_node_status_allocatable_cpu_cores node allocatable cpu cores",
+		node:   node,
+		value:  value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubeNodeStatusAllocatableCPUCoresMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"node": kpcrr.node,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (kpcrr KubeNodeStatusAllocatableCPUCoresMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("node"),
+			Value: &kpcrr.node,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusAllocatableMemoryBytesMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusAllocatableMemoryBytesMetric is a prometheus.Metric
+type KubeNodeStatusAllocatableMemoryBytesMetric struct {
+	fqName   string
+	help     string
+	resource string
+	unit     string
+	node     string
+	value    float64
+}
+
+// Creates a new KubeNodeStatusAllocatableMemoryBytesMetric, implementation of prometheus.Metric
+func newKubeNodeStatusAllocatableMemoryBytesMetric(fqname, node string, value float64) KubeNodeStatusAllocatableMemoryBytesMetric {
+	return KubeNodeStatusAllocatableMemoryBytesMetric{
+		fqName: fqname,
+		help:   "kube_node_status_allocatable_memory_bytes node allocatable memory in bytes",
+		node:   node,
+		value:  value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubeNodeStatusAllocatableMemoryBytesMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"node": kpcrr.node,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (kpcrr KubeNodeStatusAllocatableMemoryBytesMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("node"),
+			Value: &kpcrr.node,
+		},
+	}
+	return nil
+}

pkg/metrics/podmetrics.go

@@ -0,0 +1,1016 @@
+package metrics
+
+import (
+	"fmt"
+
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/log"
+	"github.com/kubecost/cost-model/pkg/prom"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	v1 "k8s.io/api/core/v1"
+)
+
+//--------------------------------------------------------------------------
+//  KubecostPodCollector
+//--------------------------------------------------------------------------
+
+// KubecostPodCollector is a prometheus collector that emits pod metrics
+type KubecostPodCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (kpmc KubecostPodCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_pod_annotations", "All annotations for each pod prefix with annotation_", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kpmc KubecostPodCollector) Collect(ch chan<- prometheus.Metric) {
+	pods := kpmc.KubeClusterCache.GetAllPods()
+	for _, pod := range pods {
+		podName := pod.GetName()
+		podNS := pod.GetNamespace()
+
+		// Pod Annotations
+		labels, values := prom.KubeAnnotationsToLabels(pod.Annotations)
+		if len(labels) > 0 {
+			ch <- newPodAnnotationMetric("kube_pod_annotations", podNS, podName, labels, values)
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubePodCollector
+//--------------------------------------------------------------------------
+
+// KubePodMetricCollector is a prometheus collector that emits pod metrics
+type KubePodCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (kpmc KubePodCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_pod_labels", "All labels for each pod prefixed with label_", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_owner", "Information about the Pod's owner", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_container_status_running", "Describes whether the container is currently in running state", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_container_status_terminated_reason", "Describes the reason the container is currently in terminated state.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_container_status_restarts_total", "The number of container restarts per container.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_container_resource_requests", "The number of requested resource by a container", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_container_resource_limits", "The number of requested limit resource by a container.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_container_resource_limits_cpu_cores", "The number of requested limit cpu core resource by a container.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_container_resource_limits_memory_bytes", "The number of requested limit memory resource by a container.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_status_phase", "The pods current phase.", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kpmc KubePodCollector) Collect(ch chan<- prometheus.Metric) {
+	pods := kpmc.KubeClusterCache.GetAllPods()
+	for _, pod := range pods {
+		podName := pod.GetName()
+		podNS := pod.GetNamespace()
+		podUID := string(pod.GetUID())
+		node := pod.Spec.NodeName
+		phase := pod.Status.Phase
+
+		// Pod Status Phase
+		if phase != "" {
+			phases := []struct {
+				v bool
+				n string
+			}{
+				{phase == v1.PodPending, string(v1.PodPending)},
+				{phase == v1.PodSucceeded, string(v1.PodSucceeded)},
+				{phase == v1.PodFailed, string(v1.PodFailed)},
+				{phase == v1.PodUnknown, string(v1.PodUnknown)},
+				{phase == v1.PodRunning, string(v1.PodRunning)},
+			}
+
+			for _, p := range phases {
+				ch <- newKubePodStatusPhaseMetric("kube_pod_status_phase", podNS, podName, podUID, p.n, boolFloat64(p.v))
+			}
+		}
+
+		// Pod Labels
+		labelNames, labelValues := prom.KubePrependQualifierToLabels(pod.GetLabels(), "label_")
+		ch <- newKubePodLabelsMetric("kube_pod_labels", podNS, podName, podUID, labelNames, labelValues)
+
+		// Owner References
+		for _, owner := range pod.OwnerReferences {
+			ch <- newKubePodOwnerMetric("kube_pod_owner", podNS, podName, owner.Name, owner.Kind, owner.Controller != nil)
+		}
+
+		// Container Status
+		for _, status := range pod.Status.ContainerStatuses {
+			ch <- newKubePodContainerStatusRestartsTotalMetric("kube_pod_container_status_restarts_total", podNS, podName, podUID, status.Name, float64(status.RestartCount))
+			if status.State.Running != nil {
+				ch <- newKubePodContainerStatusRunningMetric("kube_pod_container_status_running", podNS, podName, podUID, status.Name)
+			}
+
+			if status.State.Terminated != nil {
+				ch <- newKubePodContainerStatusTerminatedReasonMetric(
+					"kube_pod_container_status_terminated_reason",
+					podNS,
+					podName,
+					podUID,
+					status.Name,
+					status.State.Terminated.Reason)
+			}
+		}
+
+		for _, container := range pod.Spec.Containers {
+			// Requests
+			for resourceName, quantity := range container.Resources.Requests {
+				resource, unit, value := toResourceUnitValue(resourceName, quantity)
+
+				// failed to parse the resource type
+				if resource == "" {
+					log.DedupedWarningf(5, "Failed to parse resource units and quantity for resource: %s", resourceName)
+					continue
+				}
+
+				ch <- newKubePodContainerResourceRequestsMetric(
+					"kube_pod_container_resource_requests",
+					podNS,
+					podName,
+					podUID,
+					container.Name,
+					node,
+					resource,
+					unit,
+					value)
+			}
+
+			// Limits
+			for resourceName, quantity := range container.Resources.Limits {
+				resource, unit, value := toResourceUnitValue(resourceName, quantity)
+
+				// failed to parse the resource type
+				if resource == "" {
+					log.DedupedWarningf(5, "Failed to parse resource units and quantity for resource: %s", resourceName)
+					continue
+				}
+
+				// KSM v1 Emission
+				if resource == "cpu" {
+					ch <- newKubePodContainerResourceLimitsCPUCoresMetric(
+						"kube_pod_container_resource_limits_cpu_cores",
+						podNS,
+						podName,
+						podUID,
+						container.Name,
+						node,
+						value)
+				}
+				if resource == "memory" {
+					ch <- newKubePodContainerResourceLimitsMemoryBytesMetric(
+						"kube_pod_container_resource_limits_memory_bytes",
+						podNS,
+						podName,
+						podUID,
+						container.Name,
+						node,
+						value)
+				}
+
+				ch <- newKubePodContainerResourceLimitsMetric(
+					"kube_pod_container_resource_limits",
+					podNS,
+					podName,
+					podUID,
+					container.Name,
+					node,
+					resource,
+					unit,
+					value)
+			}
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  PodAnnotationsMetric
+//--------------------------------------------------------------------------
+
+// PodAnnotationsMetric is a prometheus.Metric used to encode namespace annotations
+type PodAnnotationsMetric struct {
+	fqName      string
+	help        string
+	namespace   string
+	pod         string
+	labelNames  []string
+	labelValues []string
+}
+
+// Creates a new PodAnnotationsMetric, implementation of prometheus.Metric
+func newPodAnnotationMetric(fqname, namespace, pod string, labelNames, labelValues []string) PodAnnotationsMetric {
+	return PodAnnotationsMetric{
+		fqName:      fqname,
+		help:        "kube_pod_annotations Pod Annotations",
+		namespace:   namespace,
+		pod:         pod,
+		labelNames:  labelNames,
+		labelValues: labelValues,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (pam PodAnnotationsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": pam.namespace,
+		"pod":       pam.pod,
+	}
+	return prometheus.NewDesc(pam.fqName, pam.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (pam PodAnnotationsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	for i := range pam.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &pam.labelNames[i],
+			Value: &pam.labelValues[i],
+		})
+	}
+	labels = append(labels,
+		&dto.LabelPair{
+			Name:  toStringPtr("namespace"),
+			Value: &pam.namespace,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("pod"),
+			Value: &pam.pod,
+		})
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodLabelsMetric
+//--------------------------------------------------------------------------
+
+// KubePodLabelsMetric is a prometheus.Metric used to encode
+// a duplicate of the deprecated kube-state-metrics metric
+// kube_pod_labels
+type KubePodLabelsMetric struct {
+	fqName      string
+	help        string
+	pod         string
+	namespace   string
+	uid         string
+	labelNames  []string
+	labelValues []string
+}
+
+// Creates a new KubePodLabelsMetric, implementation of prometheus.Metric
+func newKubePodLabelsMetric(fqname, namespace, pod, uid string, labelNames []string, labelValues []string) KubePodLabelsMetric {
+	return KubePodLabelsMetric{
+		fqName:      fqname,
+		help:        "kube_pod_labels all labels for each pod prefixed with label_",
+		pod:         pod,
+		namespace:   namespace,
+		uid:         uid,
+		labelNames:  labelNames,
+		labelValues: labelValues,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubePodLabelsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": nam.namespace,
+		"pod":       nam.pod,
+		"uid":       nam.uid,
+	}
+	return prometheus.NewDesc(nam.fqName, nam.help, nam.labelNames, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (nam KubePodLabelsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	for i := range nam.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &nam.labelNames[i],
+			Value: &nam.labelValues[i],
+		})
+	}
+
+	labels = append(labels,
+		&dto.LabelPair{
+			Name:  toStringPtr("pod"),
+			Value: &nam.pod,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("namespace"),
+			Value: &nam.namespace,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("uid"),
+			Value: &nam.uid,
+		},
+	)
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodContainerStatusRestartsTotalMetric
+//--------------------------------------------------------------------------
+
+// KubePodContainerStatusRestartsTotalMetric is a prometheus.Metric emitting container restarts metrics.
+type KubePodContainerStatusRestartsTotalMetric struct {
+	fqName    string
+	help      string
+	pod       string
+	namespace string
+	container string
+	uid       string
+	value     float64
+}
+
+// Creates a new KubePodContainerStatusRestartsTotalMetric, implementation of prometheus.Metric
+func newKubePodContainerStatusRestartsTotalMetric(fqname, namespace, pod, uid, container string, value float64) KubePodContainerStatusRestartsTotalMetric {
+	return KubePodContainerStatusRestartsTotalMetric{
+		fqName:    fqname,
+		help:      "kube_pod_container_status_restarts_total total container restarts",
+		pod:       pod,
+		namespace: namespace,
+		uid:       uid,
+		container: container,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcs KubePodContainerStatusRestartsTotalMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": kpcs.namespace,
+		"pod":       kpcs.pod,
+		"uid":       kpcs.uid,
+		"container": kpcs.container,
+	}
+	return prometheus.NewDesc(kpcs.fqName, kpcs.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (kpcs KubePodContainerStatusRestartsTotalMetric) Write(m *dto.Metric) error {
+	m.Counter = &dto.Counter{
+		Value: &kpcs.value,
+	}
+
+	var labels []*dto.LabelPair
+	labels = append(labels,
+		&dto.LabelPair{
+			Name:  toStringPtr("namespace"),
+			Value: &kpcs.namespace,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("pod"),
+			Value: &kpcs.pod,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("container"),
+			Value: &kpcs.container,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("uid"),
+			Value: &kpcs.uid,
+		},
+	)
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodContainerStatusTerminatedReasonMetric
+//--------------------------------------------------------------------------
+
+// KubePodContainerStatusTerminatedReasonMetric is a prometheus.Metric emitting container termination reasons.
+type KubePodContainerStatusTerminatedReasonMetric struct {
+	fqName    string
+	help      string
+	pod       string
+	namespace string
+	container string
+	uid       string
+	reason    string
+}
+
+// Creates a new KubePodContainerStatusRestartsTotalMetric, implementation of prometheus.Metric
+func newKubePodContainerStatusTerminatedReasonMetric(fqname, namespace, pod, uid, container, reason string) KubePodContainerStatusTerminatedReasonMetric {
+	return KubePodContainerStatusTerminatedReasonMetric{
+		fqName:    fqname,
+		help:      "kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.",
+		pod:       pod,
+		namespace: namespace,
+		uid:       uid,
+		container: container,
+		reason:    reason,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcs KubePodContainerStatusTerminatedReasonMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": kpcs.namespace,
+		"pod":       kpcs.pod,
+		"uid":       kpcs.uid,
+		"container": kpcs.container,
+		"reason":    kpcs.reason,
+	}
+	return prometheus.NewDesc(kpcs.fqName, kpcs.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (kpcs KubePodContainerStatusTerminatedReasonMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	labels = append(labels,
+		&dto.LabelPair{
+			Name:  toStringPtr("namespace"),
+			Value: &kpcs.namespace,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("pod"),
+			Value: &kpcs.pod,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("container"),
+			Value: &kpcs.container,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("uid"),
+			Value: &kpcs.uid,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("reason"),
+			Value: &kpcs.reason,
+		},
+	)
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodStatusPhaseMetric
+//--------------------------------------------------------------------------
+
+// KubePodStatusPhaseMetric is a prometheus.Metric emitting all phases for a pod
+type KubePodStatusPhaseMetric struct {
+	fqName    string
+	help      string
+	pod       string
+	namespace string
+	uid       string
+	phase     string
+	value     float64
+}
+
+// Creates a new KubePodContainerStatusRestartsTotalMetric, implementation of prometheus.Metric
+func newKubePodStatusPhaseMetric(fqname, namespace, pod, uid, phase string, value float64) KubePodStatusPhaseMetric {
+	return KubePodStatusPhaseMetric{
+		fqName:    fqname,
+		help:      "kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.",
+		pod:       pod,
+		namespace: namespace,
+		uid:       uid,
+		phase:     phase,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcs KubePodStatusPhaseMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": kpcs.namespace,
+		"pod":       kpcs.pod,
+		"uid":       kpcs.uid,
+		"phase":     kpcs.phase,
+	}
+	return prometheus.NewDesc(kpcs.fqName, kpcs.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (kpcs KubePodStatusPhaseMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcs.value,
+	}
+
+	var labels []*dto.LabelPair
+	labels = append(labels,
+		&dto.LabelPair{
+			Name:  toStringPtr("namespace"),
+			Value: &kpcs.namespace,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("pod"),
+			Value: &kpcs.pod,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("uid"),
+			Value: &kpcs.uid,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("phase"),
+			Value: &kpcs.phase,
+		},
+	)
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodContainerStatusRunningMetric
+//--------------------------------------------------------------------------
+
+// KubePodLabelsMetric is a prometheus.Metric used to encode
+// a duplicate of the deprecated kube-state-metrics metric
+// kube_pod_labels
+type KubePodContainerStatusRunningMetric struct {
+	fqName    string
+	help      string
+	pod       string
+	namespace string
+	container string
+	uid       string
+}
+
+// Creates a new KubePodContainerStatusRunningMetric, implementation of prometheus.Metric
+func newKubePodContainerStatusRunningMetric(fqname, namespace, pod, uid, container string) KubePodContainerStatusRunningMetric {
+	return KubePodContainerStatusRunningMetric{
+		fqName:    fqname,
+		help:      "kube_pod_container_status_running pods container status",
+		pod:       pod,
+		namespace: namespace,
+		uid:       uid,
+		container: container,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcs KubePodContainerStatusRunningMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": kpcs.namespace,
+		"pod":       kpcs.pod,
+		"uid":       kpcs.uid,
+		"container": kpcs.container,
+	}
+	return prometheus.NewDesc(kpcs.fqName, kpcs.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpcs KubePodContainerStatusRunningMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	labels = append(labels,
+		&dto.LabelPair{
+			Name:  toStringPtr("namespace"),
+			Value: &kpcs.namespace,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("pod"),
+			Value: &kpcs.pod,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("container"),
+			Value: &kpcs.container,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("uid"),
+			Value: &kpcs.uid,
+		},
+	)
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodContainerResourceRequestMetric
+//--------------------------------------------------------------------------
+
+// KubePodContainerResourceRequestsMetric is a prometheus.Metric
+type KubePodContainerResourceRequestsMetric struct {
+	fqName    string
+	help      string
+	pod       string
+	namespace string
+	container string
+	uid       string
+	resource  string
+	unit      string
+	node      string
+	value     float64
+}
+
+// Creates a new newKubePodContainerResourceRequestsMetric, implementation of prometheus.Metric
+func newKubePodContainerResourceRequestsMetric(fqname, namespace, pod, uid, container, node, resource, unit string, value float64) KubePodContainerResourceRequestsMetric {
+	return KubePodContainerResourceRequestsMetric{
+		fqName:    fqname,
+		help:      "kube_pod_container_resource_requests pods container resource requests",
+		pod:       pod,
+		namespace: namespace,
+		uid:       uid,
+		container: container,
+		node:      node,
+		resource:  resource,
+		unit:      unit,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubePodContainerResourceRequestsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": kpcrr.namespace,
+		"pod":       kpcrr.pod,
+		"uid":       kpcrr.uid,
+		"container": kpcrr.container,
+		"node":      kpcrr.node,
+		"resource":  kpcrr.resource,
+		"unit":      kpcrr.unit,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpcrr KubePodContainerResourceRequestsMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kpcrr.namespace,
+		},
+		{
+			Name:  toStringPtr("pod"),
+			Value: &kpcrr.pod,
+		},
+		{
+			Name:  toStringPtr("container"),
+			Value: &kpcrr.container,
+		},
+		{
+			Name:  toStringPtr("uid"),
+			Value: &kpcrr.uid,
+		},
+		{
+			Name:  toStringPtr("node"),
+			Value: &kpcrr.node,
+		},
+		{
+			Name:  toStringPtr("resource"),
+			Value: &kpcrr.resource,
+		},
+		{
+			Name:  toStringPtr("unit"),
+			Value: &kpcrr.unit,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodContainerResourceLimitsMetric
+//--------------------------------------------------------------------------
+
+// KubePodContainerResourceLimitsMetric is a prometheus.Metric
+type KubePodContainerResourceLimitsMetric struct {
+	fqName    string
+	help      string
+	pod       string
+	namespace string
+	container string
+	uid       string
+	resource  string
+	unit      string
+	node      string
+	value     float64
+}
+
+// Creates a new KubePodContainerResourceLimitsMetric, implementation of prometheus.Metric
+func newKubePodContainerResourceLimitsMetric(fqname, namespace, pod, uid, container, node, resource, unit string, value float64) KubePodContainerResourceLimitsMetric {
+	return KubePodContainerResourceLimitsMetric{
+		fqName:    fqname,
+		help:      "kube_pod_container_resource_limits pods container resource limits",
+		pod:       pod,
+		namespace: namespace,
+		uid:       uid,
+		container: container,
+		node:      node,
+		resource:  resource,
+		unit:      unit,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubePodContainerResourceLimitsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": kpcrr.namespace,
+		"pod":       kpcrr.pod,
+		"uid":       kpcrr.uid,
+		"container": kpcrr.container,
+		"node":      kpcrr.node,
+		"resource":  kpcrr.resource,
+		"unit":      kpcrr.unit,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpcrr KubePodContainerResourceLimitsMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kpcrr.namespace,
+		},
+		{
+			Name:  toStringPtr("pod"),
+			Value: &kpcrr.pod,
+		},
+		{
+			Name:  toStringPtr("container"),
+			Value: &kpcrr.container,
+		},
+		{
+			Name:  toStringPtr("uid"),
+			Value: &kpcrr.uid,
+		},
+		{
+			Name:  toStringPtr("node"),
+			Value: &kpcrr.node,
+		},
+		{
+			Name:  toStringPtr("resource"),
+			Value: &kpcrr.resource,
+		},
+		{
+			Name:  toStringPtr("unit"),
+			Value: &kpcrr.unit,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodContainerResourceLimitsCPUCoresMetric (KSM v1)
+//--------------------------------------------------------------------------
+
+// KubePodContainerResourceLimitsCPUCoresMetric is a prometheus.Metric
+type KubePodContainerResourceLimitsCPUCoresMetric struct {
+	fqName    string
+	help      string
+	pod       string
+	namespace string
+	container string
+	uid       string
+	node      string
+	value     float64
+}
+
+// Creates a new KubePodContainerResourceLimitsMetric, implementation of prometheus.Metric
+func newKubePodContainerResourceLimitsCPUCoresMetric(fqname, namespace, pod, uid, container, node string, value float64) KubePodContainerResourceLimitsCPUCoresMetric {
+	return KubePodContainerResourceLimitsCPUCoresMetric{
+		fqName:    fqname,
+		help:      "kube_pod_container_resource_limits_cpu_cores pods container cpu cores resource limits",
+		pod:       pod,
+		namespace: namespace,
+		uid:       uid,
+		container: container,
+		node:      node,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubePodContainerResourceLimitsCPUCoresMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": kpcrr.namespace,
+		"pod":       kpcrr.pod,
+		"uid":       kpcrr.uid,
+		"container": kpcrr.container,
+		"node":      kpcrr.node,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpcrr KubePodContainerResourceLimitsCPUCoresMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kpcrr.namespace,
+		},
+		{
+			Name:  toStringPtr("pod"),
+			Value: &kpcrr.pod,
+		},
+		{
+			Name:  toStringPtr("container"),
+			Value: &kpcrr.container,
+		},
+		{
+			Name:  toStringPtr("uid"),
+			Value: &kpcrr.uid,
+		},
+		{
+			Name:  toStringPtr("node"),
+			Value: &kpcrr.node,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodContainerResourceLimitsMemoryBytesMetric (KSM v1)
+//--------------------------------------------------------------------------
+
+// KubePodContainerResourceLimitsMemoryBytesMetric is a prometheus.Metric
+type KubePodContainerResourceLimitsMemoryBytesMetric struct {
+	fqName    string
+	help      string
+	pod       string
+	namespace string
+	container string
+	uid       string
+	node      string
+	value     float64
+}
+
+// Creates a new KubePodContainerResourceLimitsMemoryBytesMetric, implementation of prometheus.Metric
+func newKubePodContainerResourceLimitsMemoryBytesMetric(fqname, namespace, pod, uid, container, node string, value float64) KubePodContainerResourceLimitsMemoryBytesMetric {
+	return KubePodContainerResourceLimitsMemoryBytesMetric{
+		fqName:    fqname,
+		help:      "kube_pod_container_resource_limits_memory_bytes pods container memory bytes resource limits",
+		pod:       pod,
+		namespace: namespace,
+		uid:       uid,
+		container: container,
+		node:      node,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubePodContainerResourceLimitsMemoryBytesMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": kpcrr.namespace,
+		"pod":       kpcrr.pod,
+		"uid":       kpcrr.uid,
+		"container": kpcrr.container,
+		"node":      kpcrr.node,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpcrr KubePodContainerResourceLimitsMemoryBytesMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kpcrr.namespace,
+		},
+		{
+			Name:  toStringPtr("pod"),
+			Value: &kpcrr.pod,
+		},
+		{
+			Name:  toStringPtr("container"),
+			Value: &kpcrr.container,
+		},
+		{
+			Name:  toStringPtr("uid"),
+			Value: &kpcrr.uid,
+		},
+		{
+			Name:  toStringPtr("node"),
+			Value: &kpcrr.node,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodOwnerMetric
+//--------------------------------------------------------------------------
+
+// KubePodOwnerMetric is a prometheus.Metric
+type KubePodOwnerMetric struct {
+	fqName            string
+	help              string
+	namespace         string
+	pod               string
+	ownerIsController bool
+	ownerName         string
+	ownerKind         string
+}
+
+// Creates a new KubePodOwnerMetric, implementation of prometheus.Metric
+func newKubePodOwnerMetric(fqname, namespace, pod, ownerName, ownerKind string, ownerIsController bool) KubePodOwnerMetric {
+	return KubePodOwnerMetric{
+		fqName:            fqname,
+		help:              "kube_pod_owner Information about the Pod's owner",
+		namespace:         namespace,
+		pod:               pod,
+		ownerName:         ownerName,
+		ownerKind:         ownerKind,
+		ownerIsController: ownerIsController,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpo KubePodOwnerMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace":           kpo.namespace,
+		"pod":                 kpo.pod,
+		"owner_name":          kpo.ownerName,
+		"owner_kind":          kpo.ownerKind,
+		"owner_is_controller": fmt.Sprintf("%t", kpo.ownerIsController),
+	}
+	return prometheus.NewDesc(kpo.fqName, kpo.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpo KubePodOwnerMetric) Write(m *dto.Metric) error {
+	v := float64(1.0)
+	m.Gauge = &dto.Gauge{
+		Value: &v,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kpo.namespace,
+		},
+		{
+			Name:  toStringPtr("pod"),
+			Value: &kpo.pod,
+		},
+		{
+			Name:  toStringPtr("owner_name"),
+			Value: &kpo.ownerName,
+		},
+		{
+			Name:  toStringPtr("owner_kind"),
+			Value: &kpo.ownerKind,
+		},
+		{
+			Name:  toStringPtr("owner_is_controller"),
+			Value: toStringPtr(fmt.Sprintf("%t", kpo.ownerIsController)),
+		},
+	}
+	return nil
+}

pkg/metrics/pvcmetrics.go

@@ -0,0 +1,159 @@
+package metrics
+
+import (
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	v1 "k8s.io/api/core/v1"
+)
+
+//--------------------------------------------------------------------------
+//  KubePVCCollector
+//--------------------------------------------------------------------------
+
+// KubePVCCollector is a prometheus collector that generates pvc sourced metrics
+type KubePVCCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics collected by this Collector.
+func (kpvc KubePVCCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_persistentvolumeclaim_resource_requests_storage_bytes", "The pvc storage resource requests in bytes", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_persistentvolumeclaim_info", "The pvc storage resource requests in bytes", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kpvc KubePVCCollector) Collect(ch chan<- prometheus.Metric) {
+	pvcs := kpvc.KubeClusterCache.GetAllPersistentVolumeClaims()
+	for _, pvc := range pvcs {
+		storageClass := getPersistentVolumeClaimClass(pvc)
+		volume := pvc.Spec.VolumeName
+
+		ch <- newKubePVCInfoMetric("kube_persistentvolumeclaim_info", pvc.Name, pvc.Namespace, storageClass, volume)
+
+		if storage, ok := pvc.Spec.Resources.Requests[v1.ResourceStorage]; ok {
+			ch <- newKubePVCResourceRequestsStorageBytesMetric("kube_persistentvolumeclaim_resource_requests_storage_bytes", pvc.Name, pvc.Namespace, float64(storage.Value()))
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubePVCResourceRequestsStorageBytesMetric
+//--------------------------------------------------------------------------
+
+// KubePVCResourceRequestsStorageBytesMetric is a prometheus.Metric
+type KubePVCResourceRequestsStorageBytesMetric struct {
+	fqName    string
+	help      string
+	namespace string
+	pvc       string
+	value     float64
+}
+
+// Creates a new KubePVCResourceRequestsStorageBytesMetric, implementation of prometheus.Metric
+func newKubePVCResourceRequestsStorageBytesMetric(fqname, pvc, namespace string, value float64) KubePVCResourceRequestsStorageBytesMetric {
+	return KubePVCResourceRequestsStorageBytesMetric{
+		fqName:    fqname,
+		help:      "kube_persistentvolumeclaim_resource_requests_storage_bytes pvc storage resource requests in bytes",
+		pvc:       pvc,
+		namespace: namespace,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpvcrr KubePVCResourceRequestsStorageBytesMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"persistentvolumeclaim": kpvcrr.pvc,
+		"namespace":             kpvcrr.namespace,
+	}
+	return prometheus.NewDesc(kpvcrr.fqName, kpvcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpvcrr KubePVCResourceRequestsStorageBytesMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpvcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("persistentvolumeclaim"),
+			Value: &kpvcrr.pvc,
+		},
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kpvcrr.namespace,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePVCInfoMetric
+//--------------------------------------------------------------------------
+
+// KubePVCInfoMetric is a prometheus.Metric
+type KubePVCInfoMetric struct {
+	fqName       string
+	help         string
+	namespace    string
+	pvc          string
+	storageclass string
+	volume       string
+}
+
+// Creates a new KubePVCInfoMetric, implementation of prometheus.Metric
+func newKubePVCInfoMetric(fqname, pvc, namespace, storageclass, volume string) KubePVCInfoMetric {
+	return KubePVCInfoMetric{
+		fqName:       fqname,
+		help:         "kube_persistentvolumeclaim_info pvc storage resource requests in bytes",
+		pvc:          pvc,
+		namespace:    namespace,
+		storageclass: storageclass,
+		volume:       volume,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpvcrr KubePVCInfoMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"persistentvolumeclaim": kpvcrr.pvc,
+		"namespace":             kpvcrr.namespace,
+		"storageclass":          kpvcrr.storageclass,
+		"volumename":            kpvcrr.volume,
+	}
+	return prometheus.NewDesc(kpvcrr.fqName, kpvcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpvci KubePVCInfoMetric) Write(m *dto.Metric) error {
+	v := float64(1.0)
+	m.Gauge = &dto.Gauge{
+		Value: &v,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kpvci.namespace,
+		},
+		{
+			Name:  toStringPtr("persistentvolumeclaim"),
+			Value: &kpvci.pvc,
+		},
+		{
+			Name:  toStringPtr("storageclass"),
+			Value: &kpvci.storageclass,
+		},
+		{
+			Name:  toStringPtr("volumename"),
+			Value: &kpvci.volume,
+		},
+	}
+	return nil
+}

pkg/metrics/pvmetrics.go

@@ -0,0 +1,154 @@
+package metrics
+
+import (
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	v1 "k8s.io/api/core/v1"
+)
+
+//--------------------------------------------------------------------------
+//  KubePVCollector
+//--------------------------------------------------------------------------
+
+// KubePVCollector is a prometheus collector that generates PV metrics
+type KubePVCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (kpvcb KubePVCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_persistentvolume_capacity_bytes", "The pv storage capacity in bytes", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_persistentvolume_status_phase", "The phase indicates if a volume is available, bound to a claim, or released by a claim.", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kpvcb KubePVCollector) Collect(ch chan<- prometheus.Metric) {
+	pvs := kpvcb.KubeClusterCache.GetAllPersistentVolumes()
+	for _, pv := range pvs {
+		phase := pv.Status.Phase
+		if phase != "" {
+			phases := []struct {
+				v bool
+				n string
+			}{
+				{phase == v1.VolumePending, string(v1.VolumePending)},
+				{phase == v1.VolumeAvailable, string(v1.VolumeAvailable)},
+				{phase == v1.VolumeBound, string(v1.VolumeBound)},
+				{phase == v1.VolumeReleased, string(v1.VolumeReleased)},
+				{phase == v1.VolumeFailed, string(v1.VolumeFailed)},
+			}
+
+			for _, p := range phases {
+				ch <- newKubePVStatusPhaseMetric("kube_persistentvolume_status_phase", pv.Name, p.n, boolFloat64(p.v))
+			}
+		}
+
+		storage := pv.Spec.Capacity[v1.ResourceStorage]
+		m := newKubePVCapacityBytesMetric("kube_persistentvolume_capacity_bytes", pv.Name, float64(storage.Value()))
+
+		ch <- m
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubePVCapacityBytesMetric
+//--------------------------------------------------------------------------
+
+// KubePVCapacityBytesMetric is a prometheus.Metric
+type KubePVCapacityBytesMetric struct {
+	fqName string
+	help   string
+	pv     string
+	value  float64
+}
+
+// Creates a new KubePVCapacityBytesMetric, implementation of prometheus.Metric
+func newKubePVCapacityBytesMetric(fqname, pv string, value float64) KubePVCapacityBytesMetric {
+	return KubePVCapacityBytesMetric{
+		fqName: fqname,
+		help:   "kube_persistentvolume_capacity_bytes pv storage capacity in bytes",
+		pv:     pv,
+		value:  value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubePVCapacityBytesMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"persistentvolume": kpcrr.pv,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpcrr KubePVCapacityBytesMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("persistentvolume"),
+			Value: &kpcrr.pv,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePVStatusPhaseMetric
+//--------------------------------------------------------------------------
+
+// KubePVStatusPhaseMetric is a prometheus.Metric
+type KubePVStatusPhaseMetric struct {
+	fqName string
+	help   string
+	pv     string
+	phase  string
+	value  float64
+}
+
+// Creates a new KubePVCapacityBytesMetric, implementation of prometheus.Metric
+func newKubePVStatusPhaseMetric(fqname, pv, phase string, value float64) KubePVStatusPhaseMetric {
+	return KubePVStatusPhaseMetric{
+		fqName: fqname,
+		help:   "kube_persistentvolume_status_phase pv status phase",
+		pv:     pv,
+		phase:  phase,
+		value:  value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubePVStatusPhaseMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"persistentvolume": kpcrr.pv,
+		"phase":            kpcrr.phase,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpcrr KubePVStatusPhaseMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("persistentvolume"),
+			Value: &kpcrr.pv,
+		},
+		{
+			Name:  toStringPtr("phase"),
+			Value: &kpcrr.phase,
+		},
+	}
+	return nil
+}

pkg/metrics/servicemetrics.go

@@ -0,0 +1,101 @@
+package metrics
+
+import (
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/prom"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+)
+
+//--------------------------------------------------------------------------
+//  KubecostServiceCollector
+//--------------------------------------------------------------------------
+
+// KubecostServiceCollector is a prometheus collector that generates service sourced metrics.
+type KubecostServiceCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (sc KubecostServiceCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("service_selector_labels", "service selector labels", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (sc KubecostServiceCollector) Collect(ch chan<- prometheus.Metric) {
+	svcs := sc.KubeClusterCache.GetAllServices()
+	for _, svc := range svcs {
+		serviceName := svc.GetName()
+		serviceNS := svc.GetNamespace()
+
+		labels, values := prom.KubeLabelsToLabels(svc.Spec.Selector)
+		if len(labels) > 0 {
+			m := newServiceSelectorLabelsMetric(serviceName, serviceNS, "service_selector_labels", labels, values)
+			ch <- m
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  ServiceSelectorLabelsMetric
+//--------------------------------------------------------------------------
+
+// ServiceSelectorLabelsMetric is a prometheus.Metric used to encode service selector labels
+type ServiceSelectorLabelsMetric struct {
+	fqName      string
+	help        string
+	labelNames  []string
+	labelValues []string
+	serviceName string
+	namespace   string
+}
+
+// Creates a new ServiceMetric, implementation of prometheus.Metric
+func newServiceSelectorLabelsMetric(name, namespace, fqname string, labelNames, labelvalues []string) ServiceSelectorLabelsMetric {
+	return ServiceSelectorLabelsMetric{
+		fqName:      fqname,
+		labelNames:  labelNames,
+		labelValues: labelvalues,
+		help:        "service_selector_labels Service Selector Labels",
+		serviceName: name,
+		namespace:   namespace,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (s ServiceSelectorLabelsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"service":   s.serviceName,
+		"namespace": s.namespace,
+	}
+	return prometheus.NewDesc(s.fqName, s.help, s.labelNames, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (s ServiceSelectorLabelsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+	var labels []*dto.LabelPair
+	for i := range s.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &s.labelNames[i],
+			Value: &s.labelValues[i],
+		})
+	}
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("namespace"),
+		Value: &s.namespace,
+	})
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("service"),
+		Value: &s.serviceName,
+	})
+	m.Label = labels
+	return nil
+}

pkg/metrics/statefulsetmetrics.go

@@ -0,0 +1,101 @@
+package metrics
+
+import (
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/prom"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+)
+
+//--------------------------------------------------------------------------
+//  KubecostStatefulsetCollector
+//--------------------------------------------------------------------------
+
+// StatefulsetCollector is a prometheus collector that generates StatefulsetMetrics
+type KubecostStatefulsetCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (sc KubecostStatefulsetCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("statefulSet_match_labels", "statfulSet match labels", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (sc KubecostStatefulsetCollector) Collect(ch chan<- prometheus.Metric) {
+	ds := sc.KubeClusterCache.GetAllStatefulSets()
+	for _, statefulset := range ds {
+		statefulsetName := statefulset.GetName()
+		statefulsetNS := statefulset.GetNamespace()
+
+		labels, values := prom.KubeLabelsToLabels(statefulset.Spec.Selector.MatchLabels)
+		if len(labels) > 0 {
+			m := newStatefulsetMatchLabelsMetric(statefulsetName, statefulsetNS, "statefulSet_match_labels", labels, values)
+			ch <- m
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  StatefulsetMatchLabelsMetric
+//--------------------------------------------------------------------------
+
+// StatefulsetMetric is a prometheus.Metric used to encode statefulset match labels
+type StatefulsetMatchLabelsMetric struct {
+	fqName          string
+	help            string
+	labelNames      []string
+	labelValues     []string
+	statefulsetName string
+	namespace       string
+}
+
+// Creates a new StatefulsetMetric, implementation of prometheus.Metric
+func newStatefulsetMatchLabelsMetric(name, namespace, fqname string, labelNames, labelvalues []string) StatefulsetMatchLabelsMetric {
+	return StatefulsetMatchLabelsMetric{
+		fqName:          fqname,
+		labelNames:      labelNames,
+		labelValues:     labelvalues,
+		help:            "statefulSet_match_labels StatefulSet Match Labels",
+		statefulsetName: name,
+		namespace:       namespace,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (s StatefulsetMatchLabelsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"statefulSet": s.statefulsetName,
+		"namespace":   s.namespace,
+	}
+	return prometheus.NewDesc(s.fqName, s.help, s.labelNames, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (s StatefulsetMatchLabelsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+	var labels []*dto.LabelPair
+	for i := range s.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &s.labelNames[i],
+			Value: &s.labelValues[i],
+		})
+	}
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("namespace"),
+		Value: &s.namespace,
+	})
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("statefulSet"),
+		Value: &s.statefulsetName,
+	})
+	m.Label = labels
+	return nil
+}

pkg/prom/contextnames.go

@@ -0,0 +1,27 @@
+package prom
+
+const (
+	// AllocationContextName is the name we assign the allocation query context [metadata]
+	AllocationContextName = "allocation"
+
+	// ClusterContextName is the name we assign the cluster query context [metadata]
+	ClusterContextName = "cluster"
+
+	// ClusterContextName is the name we assign the optional cluster query context [metadata]
+	ClusterOptionalContextName = "cluster-optional"
+
+	// ComputeCostDataContextName is the name we assign the compute cost data query context [metadata]
+	ComputeCostDataContextName = "compute-cost-data"
+
+	// ComputeCostDataContextName is the name we assign the compute cost data range query context [metadata]
+	ComputeCostDataRangeContextName = "compute-cost-data-range"
+
+	// ClusterMapContextName is the name we assign the cluster map query context [metadata]
+	ClusterMapContextName = "cluster-map"
+
+	// FrontendContextName is the name we assign queries proxied from the frontend [metadata]
+	FrontendContextName = "frontend"
+
+	// DiagnosticContextName is the name we assign queries that check the state of the prometheus connection
+	DiagnosticContextName = "diagnostic"
+)

pkg/prom/diagnostics.go

@@ -0,0 +1,173 @@
+package prom
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/kubecost/cost-model/pkg/env"
+	"github.com/kubecost/cost-model/pkg/log"
+	prometheus "github.com/prometheus/client_golang/api"
+)
+
+// QueuedPromRequest is a representation of a request waiting to be sent by the prometheus
+// client.
+type QueuedPromRequest struct {
+	Context   string `json:"context"`
+	Query     string `json:"query"`
+	QueueTime int64  `json:"queueTime"`
+}
+
+// PrometheusQueueState contains diagnostic information concerning the state of the prometheus request
+// queue
+type PrometheusQueueState struct {
+	QueuedRequests      []*QueuedPromRequest `json:"queuedRequests"`
+	OutboundRequests    int                  `json:"outboundRequests"`
+	TotalRequests       int                  `json:"totalRequests"`
+	MaxQueryConcurrency int                  `json:"maxQueryConcurrency"`
+}
+
+// GetPrometheusQueueState is a diagnostic function that probes the prometheus request queue and gathers
+// query, context, and queue statistics.
+func GetPrometheusQueueState(client prometheus.Client) (*PrometheusQueueState, error) {
+	rlpc, ok := client.(*RateLimitedPrometheusClient)
+	if !ok {
+		return nil, fmt.Errorf("Failed to get prometheus queue state for the provided client. Must be of type RateLimitedPrometheusClient.")
+	}
+
+	outbound := rlpc.TotalOutboundRequests()
+
+	requests := []*QueuedPromRequest{}
+	rlpc.queue.Each(func(_ int, entry interface{}) {
+		if req, ok := entry.(*workRequest); ok {
+			requests = append(requests, &QueuedPromRequest{
+				Context:   req.contextName,
+				Query:     req.query,
+				QueueTime: time.Since(req.start).Milliseconds(),
+			})
+		}
+	})
+
+	return &PrometheusQueueState{
+		QueuedRequests:      requests,
+		OutboundRequests:    outbound,
+		TotalRequests:       outbound + len(requests),
+		MaxQueryConcurrency: env.GetMaxQueryConcurrency(),
+	}, nil
+}
+
+// LogPrometheusClientState logs the current state, with respect to outbound requests, if that
+// information is available.
+func LogPrometheusClientState(client prometheus.Client) {
+	if rc, ok := client.(requestCounter); ok {
+		queued := rc.TotalQueuedRequests()
+		outbound := rc.TotalOutboundRequests()
+		total := queued + outbound
+
+		log.Infof("Outbound Requests: %d, Queued Requests: %d, Total Requests: %d", outbound, queued, total)
+	}
+}
+
+// GetPrometheusMetrics returns a list of the state of Prometheus metric used by kubecost using the provided client
+func GetPrometheusMetrics(client prometheus.Client, offset string) ([]*PrometheusDiagnostic, error) {
+	docs := "https://github.com/kubecost/docs/blob/master/diagnostics.md"
+	ctx := NewNamedContext(client, DiagnosticContextName)
+
+	result := []*PrometheusDiagnostic{
+		{
+			ID:          "cadvisorMetric",
+			Query:       fmt.Sprintf(`absent_over_time(container_cpu_usage_seconds_total[5m] %s)`, offset),
+			Label:       "cAdvsior metrics available",
+			Description: "Determine if cAdvisor metrics are available during last 5 minutes.",
+			DocLink:     fmt.Sprintf("%s#cadvisor-metrics-available", docs),
+		},
+		{
+			ID:          "ksmMetric",
+			Query:       fmt.Sprintf(`absent_over_time(kube_pod_container_resource_requests{resource="memory", unit="byte"}[5m]  %s)`, offset),
+			Label:       "Kube-state-metrics available",
+			Description: "Determine if metrics from kube-state-metrics are available during last 5 minutes.",
+			DocLink:     fmt.Sprintf("%s#kube-state-metrics-metrics-available", docs),
+		},
+		{
+			ID:          "kubecostMetric",
+			Query:       fmt.Sprintf(`absent_over_time(node_cpu_hourly_cost[5m]  %s)`, offset),
+			Label:       "Kubecost metrics available",
+			Description: "Determine if metrics from Kubecost are available during last 5 minutes.",
+		},
+		{
+			ID:          "neMetric",
+			Query:       fmt.Sprintf(`absent_over_time(node_cpu_seconds_total[5m]  %s)`, offset),
+			Label:       "Node-exporter metrics available",
+			Description: "Determine if metrics from node-exporter are available during last 5 minutes.",
+			DocLink:     fmt.Sprintf("%s#node-exporter-metrics-available", docs),
+		},
+		{
+			ID:          "recordingMetric",
+			Query:       fmt.Sprintf(`absent_over_time(node_cpu_seconds_total[5m]  %s)`, offset),
+			Label:       "Recording rules are available",
+			Description: "Determine if metrics defined by kubecost recording rules are available during last 5 minutes.",
+			DocLink:     fmt.Sprintf("%s#recording-rules-are-available", docs),
+		},
+		{
+			ID:          "cadvisorLabel",
+			Query:       fmt.Sprintf(`absent_over_time(container_cpu_usage_seconds_total{container!="",pod!=""}[5m]  %s)`, offset),
+			Label:       "Expected cAdvsior labels available",
+			Description: "Determine if expected cAdvisor labels are present during last 5 minutes.",
+			DocLink:     fmt.Sprintf("%s#cadvisor-metrics-available", docs),
+		},
+		{
+			ID:          "ksmVersion",
+			Query:       fmt.Sprintf(`absent_over_time(kube_persistentvolume_capacity_bytes[5m]  %s)`, offset),
+			Label:       "Expected kube-state-metrics version found",
+			Description: "Determine if metric in required kube-state-metrics version are present during last 5 minutes.",
+			DocLink:     fmt.Sprintf("%s#expected-kube-state-metrics-version-found", docs),
+		},
+		{
+			ID:          "scrapeInterval",
+			Query:       fmt.Sprintf(`absent_over_time(prometheus_target_interval_length_seconds[5m]  %s)`, offset),
+			Label:       "Expected Prometheus self-scrape metrics available",
+			Description: "Determine if prometheus has its own self-scraped metrics during the last 5 minutes.",
+		},
+		{
+			ID: "cpuThrottling",
+			Query: `avg(increase(container_cpu_cfs_throttled_periods_total{container="cost-model"}[10m])) by (container_name, pod_name, namespace)
+		/ avg(increase(container_cpu_cfs_periods_total{container="cost-model"}[10m])) by (container_name, pod_name, namespace) > 0.2`,
+			Label:       "Kubecost is not CPU throttled",
+			Description: "Kubecost loading slowly? A kubecost component might be CPU throttled",
+		},
+	}
+
+	for _, pd := range result {
+		err := pd.executePrometheusDiagnosticQuery(ctx)
+		if err != nil {
+			log.Errorf(err.Error())
+		}
+	}
+
+	return result, nil
+}
+
+// PrometheusDiagnostic holds information about a metric and the query to ensure it is functional
+type PrometheusDiagnostic struct {
+	ID          string         `json:"id"`
+	Query       string         `json:"query"`
+	Label       string         `json:"label"`
+	Description string         `json:"description"`
+	DocLink     string         `json:"docLink"`
+	Result      []*QueryResult `json:"result"`
+	Passed      bool           `json:"passed"`
+}
+
+// executePrometheusDiagnosticQuery executes a PrometheusDiagnostic query using the given context
+func (pd *PrometheusDiagnostic) executePrometheusDiagnosticQuery(ctx *Context) error {
+	resultCh := ctx.Query(pd.Query)
+	result, err := resultCh.Await()
+	if err != nil {
+		return fmt.Errorf("prometheus diagnostic %s failed with error: %s", pd.ID, err)
+	}
+	if result == nil {
+		result = []*QueryResult{}
+	}
+	pd.Result = result
+	pd.Passed = len(result) == 0
+	return nil
+}

pkg/prom/error.go

@@ -82,7 +82,7 @@ type ErrorsAndWarningStrings struct {
 }
 
 // QueryErrorCollector is used to collect prometheus query errors and warnings, and also meets the
-// Error
+// Error interface
 type QueryErrorCollector struct {
 	m        sync.RWMutex
 	errors   []*QueryError

pkg/prom/helpers.go

@@ -0,0 +1,62 @@
+package prom
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	prometheus "github.com/prometheus/client_golang/api"
+	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+
+	"gopkg.in/yaml.v2"
+)
+
+const PrometheusTroubleshootingURL = "http://docs.kubecost.com/custom-prom#troubleshoot"
+
+// ScrapeConfig is the minimalized view of a prometheus scrape configuration
+type ScrapeConfig struct {
+	JobName        string `yaml:"job_name,omitempty"`
+	ScrapeInterval string `yaml:"scrape_interval,omitempty"`
+}
+
+// PrometheusConfig is the minimalized view of a prometheus configuration
+type PrometheusConfig struct {
+	ScrapeConfigs []ScrapeConfig `yaml:"scrape_configs,omitempty"`
+}
+
+// GetPrometheusConfig uses the provided yaml string to parse the minimalized prometheus config
+func GetPrometheusConfig(pcfg string) (PrometheusConfig, error) {
+	var promCfg PrometheusConfig
+	err := yaml.Unmarshal([]byte(pcfg), &promCfg)
+	return promCfg, err
+}
+
+// ScrapeIntervalFor uses the provided prometheus client to locate a scrape interval for a specific job name
+func ScrapeIntervalFor(client prometheus.Client, jobName string) (time.Duration, error) {
+	api := v1.NewAPI(client)
+	promConfig, err := api.Config(context.Background())
+	if err != nil {
+		return 0, err
+	}
+
+	cfg, err := GetPrometheusConfig(promConfig.YAML)
+	if err != nil {
+		return 0, err
+	}
+
+	for _, sc := range cfg.ScrapeConfigs {
+		if sc.JobName == jobName {
+			if sc.ScrapeInterval != "" {
+				si := sc.ScrapeInterval
+				sid, err := time.ParseDuration(si)
+				if err != nil {
+					return 0, fmt.Errorf("Error parsing scrape config for %s", sc.JobName)
+				} else {
+					return sid, nil
+				}
+			}
+		}
+	}
+
+	return 0, fmt.Errorf("Failed to locate scrape config for %s", jobName)
+}

pkg/prom/metrics_test.go

@@ -0,0 +1,95 @@
+package prom
+
+import (
+	"fmt"
+	"testing"
+)
+
+func checkSlice(s1, s2 []string) error {
+	if len(s1) != len(s2) {
+		return fmt.Errorf("len(s1) [%d] != len(s2) [%d]", len(s1), len(s2))
+	}
+
+	for i := 0; i < len(s1); i++ {
+		if s1[i] != s2[i] {
+			return fmt.Errorf("At Index: %d. Different Values %s (s1) != %s (s2)", i, s1[i], s2[i])
+		}
+	}
+	return nil
+}
+
+func TestEmptyKubeLabelsToPromLabels(t *testing.T) {
+	labels, values := KubeLabelsToLabels(nil)
+
+	if len(labels) != 0 {
+		t.Errorf("Labels length is non-zero\n")
+	}
+	if len(values) != 0 {
+		t.Errorf("Values length is non-zero\n")
+	}
+
+	labels, values = KubeLabelsToLabels(map[string]string{})
+
+	if len(labels) != 0 {
+		t.Errorf("Labels length is non-zero\n")
+	}
+	if len(values) != 0 {
+		t.Errorf("Values length is non-zero\n")
+	}
+}
+
+func TestKubeLabelsToPromLabels(t *testing.T) {
+	var expectedLabels []string = []string{
+		"label_app",
+		"label_chart",
+		"label_control_plane",
+		"label_gatekeeper_sh_operation",
+		"label_heritage",
+		"label_pod_template_hash",
+		"label_release",
+	}
+	var expectedValues []string = []string{
+		"gatekeeper",
+		"gatekeeper",
+		"audit-controller",
+		"audit",
+		"Helm",
+		"5599859cd4",
+		"gatekeeper",
+	}
+
+	kubeLabels := map[string]string{
+		"app":                     "gatekeeper",
+		"chart":                   "gatekeeper",
+		"control-plane":           "audit-controller",
+		"gatekeeper.sh/operation": "audit",
+		"heritage":                "Helm",
+		"pod-template-hash":       "5599859cd4",
+		"release":                 "gatekeeper",
+	}
+
+	labels, values := KubePrependQualifierToLabels(kubeLabels, "label_")
+	l2, v2 := KubeLabelsToLabels(kubeLabels)
+
+	// Check to make sure we get expected labels and values returned
+	err := checkSlice(labels, expectedLabels)
+	if err != nil {
+		t.Errorf("%s", err)
+	}
+	err = checkSlice(values, expectedValues)
+	if err != nil {
+		t.Errorf("%s", err)
+	}
+
+	// Check to make sure the helper function returns what the prependqualifier func
+	// returns
+	err = checkSlice(l2, labels)
+	if err != nil {
+		t.Errorf("%s", err)
+	}
+
+	err = checkSlice(v2, values)
+	if err != nil {
+		t.Errorf("%s", err)
+	}
+}

pkg/prom/prom.go

@@ -9,9 +9,12 @@ import (
 	"os"
 	"time"
 
+	"github.com/kubecost/cost-model/pkg/collections"
 	"github.com/kubecost/cost-model/pkg/env"
 	"github.com/kubecost/cost-model/pkg/log"
-	"github.com/kubecost/cost-model/pkg/util"
+	"github.com/kubecost/cost-model/pkg/util/atomic"
+	"github.com/kubecost/cost-model/pkg/util/fileutil"
+	"github.com/kubecost/cost-model/pkg/util/httputil"
 
 	golog "log"
 
@@ -63,16 +66,16 @@ type RateLimitedPrometheusClient struct {
 	id         string
 	client     prometheus.Client
 	auth       *ClientAuth
-	queue      util.BlockingQueue
+	queue      collections.BlockingQueue
 	decorator  QueryParamsDecorator
-	outbound   *util.AtomicInt32
+	outbound   *atomic.AtomicInt32
 	fileLogger *golog.Logger
 }
 
 // requestCounter is used to determine if the prometheus client keeps track of
 // the concurrent outbound requests
 type requestCounter interface {
-	TotalRequests() int
+	TotalQueuedRequests() int
 	TotalOutboundRequests() int
 }
 
@@ -84,12 +87,12 @@ func NewRateLimitedClient(id string, config prometheus.Config, maxConcurrency in
 		return nil, err
 	}
 
-	queue := util.NewBlockingQueue()
-	outbound := util.NewAtomicInt32(0)
+	queue := collections.NewBlockingQueue()
+	outbound := atomic.NewAtomicInt32(0)
 
 	var logger *golog.Logger
 	if queryLogFile != "" {
-		exists, err := util.FileExists(queryLogFile)
+		exists, err := fileutil.FileExists(queryLogFile)
 		if exists {
 			os.Remove(queryLogFile)
 		}
@@ -127,7 +130,7 @@ func (rlpc *RateLimitedPrometheusClient) ID() string {
 
 // TotalRequests returns the total number of requests that are either waiting to be sent and/or
 // are currently outbound.
-func (rlpc *RateLimitedPrometheusClient) TotalRequests() int {
+func (rlpc *RateLimitedPrometheusClient) TotalQueuedRequests() int {
 	return rlpc.queue.Length()
 }
 
@@ -150,6 +153,9 @@ type workRequest struct {
 	respChan chan *workResponse
 	// used as a sentinel value to close the worker goroutine
 	closer bool
+	// request metadata for diagnostics
+	contextName string
+	query       string
 }
 
 // workResponse is the response payload returned to the Do method
@@ -214,12 +220,21 @@ func (rlpc *RateLimitedPrometheusClient) Do(ctx context.Context, req *http.Reque
 	respChan := make(chan *workResponse)
 	defer close(respChan)
 
+	// request names are used as a debug utility to identify requests in queue
+	contextName := "<none>"
+	if n, ok := httputil.GetName(req); ok {
+		contextName = n
+	}
+	query, _ := httputil.GetQuery(req)
+
 	rlpc.queue.Enqueue(&workRequest{
-		ctx:      ctx,
-		req:      req,
-		start:    time.Now(),
-		respChan: respChan,
-		closer:   false,
+		ctx:         ctx,
+		req:         req,
+		start:       time.Now(),
+		respChan:    respChan,
+		closer:      false,
+		contextName: contextName,
+		query:       query,
 	})
 
 	workRes := <-respChan
@@ -256,24 +271,12 @@ func NewPrometheusClient(address string, timeout, keepAlive time.Duration, query
 	return NewRateLimitedClient(PrometheusClientID, pc, queryConcurrency, auth, nil, queryLogFile)
 }
 
-// LogPrometheusClientState logs the current state, with respect to outbound requests, if that
-// information is available.
-func LogPrometheusClientState(client prometheus.Client) {
-	if rc, ok := client.(requestCounter); ok {
-		total := rc.TotalRequests()
-		outbound := rc.TotalOutboundRequests()
-		queued := total - outbound
-
-		log.Infof("Outbound Requests: %d, Queued Requests: %d, Total Requests: %d", outbound, queued, total)
-	}
-}
-
 // LogQueryRequest logs the query that was send to prom/thanos with the time in queue and total time after being sent
 func LogQueryRequest(l *golog.Logger, req *http.Request, queueTime time.Duration, sendTime time.Duration) {
 	if l == nil {
 		return
 	}
-	qp := util.NewQueryParams(req.URL.Query())
+	qp := httputil.NewQueryParams(req.URL.Query())
 	query := qp.Get("query", "<Unknown>")
 
 	l.Printf("[Queue: %fs, Outbound: %fs][Query: %s]\n", queueTime.Seconds(), sendTime.Seconds(), query)

pkg/prom/query.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/kubecost/cost-model/pkg/errors"
 	"github.com/kubecost/cost-model/pkg/log"
-	"github.com/kubecost/cost-model/pkg/util"
+	"github.com/kubecost/cost-model/pkg/util/httputil"
 	"github.com/kubecost/cost-model/pkg/util/json"
 	prometheus "github.com/prometheus/client_golang/api"
 )
@@ -25,6 +25,7 @@ const (
 // parsing query responses and errors.
 type Context struct {
 	Client         prometheus.Client
+	name           string
 	errorCollector *QueryErrorCollector
 }
 
@@ -34,10 +35,18 @@ func NewContext(client prometheus.Client) *Context {
 
 	return &Context{
 		Client:         client,
+		name:           "",
 		errorCollector: &ec,
 	}
 }
 
+// NewNamedContext creates a new named Promethues querying context from the given client
+func NewNamedContext(client prometheus.Client, name string) *Context {
+	ctx := NewContext(client)
+	ctx.name = name
+	return ctx
+}
+
 // Warnings returns the warnings collected from the Context's ErrorCollector
 func (ctx *Context) Warnings() []*QueryWarning {
 	return ctx.errorCollector.Warnings()
@@ -157,7 +166,8 @@ func runQuery(query string, ctx *Context, resCh QueryResultsChan, profileLabel s
 	resCh <- results
 }
 
-func (ctx *Context) query(query string) (interface{}, prometheus.Warnings, error) {
+// RawQuery is a direct query to the prometheus client and returns the body of the response
+func (ctx *Context) RawQuery(query string) ([]byte, error) {
 	u := ctx.Client.URL(epQuery, nil)
 	q := u.Query()
 	q.Set("query", query)
@@ -165,38 +175,59 @@ func (ctx *Context) query(query string) (interface{}, prometheus.Warnings, error
 
 	req, err := http.NewRequest(http.MethodPost, u.String(), nil)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	resp, body, warnings, err := ctx.Client.Do(context.Background(), req)
-	for _, w := range warnings {
-		// NoStoreAPIWarning is a warning that we would consider an error. It returns partial data relating only to the
-		// store apis which were reachable. In order to ensure integrity of data across all clusters, we'll need to identify
-		// this warning and convert it to an error.
-		if IsNoStoreAPIWarning(w) {
-			return nil, warnings, NewCommError(fmt.Sprintf("Error: %s, Body: %s, Query: %s", w, body, query))
-		}
-
-		log.Warningf("fetching query '%s': %s", query, w)
+	// Set QueryContext name if non empty
+	if ctx.name != "" {
+		req = httputil.SetName(req, ctx.name)
 	}
+	req = httputil.SetQuery(req, query)
+
+	// Note that the warnings return value from client.Do() is always nil using this
+	// version of the prometheus client library. We parse the warnings out of the response
+	// body after json decodidng completes.
+	resp, body, _, err := ctx.Client.Do(context.Background(), req)
 	if err != nil {
 		if resp == nil {
-			return nil, warnings, fmt.Errorf("query error: '%s' fetching query '%s'", err.Error(), query)
+			return nil, fmt.Errorf("query error: '%s' fetching query '%s'", err.Error(), query)
 		}
 
-		return nil, warnings, fmt.Errorf("query error %d: '%s' fetching query '%s'", resp.StatusCode, err.Error(), query)
+		return nil, fmt.Errorf("query error %d: '%s' fetching query '%s'", resp.StatusCode, err.Error(), query)
 	}
+
 	// Unsuccessful Status Code, log body and status
 	statusCode := resp.StatusCode
 	statusText := http.StatusText(statusCode)
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return nil, warnings, CommErrorf("%d (%s) URL: '%s' Headers: '%s', Body: '%s' Query: '%s'", statusCode, statusText, req.URL, util.HeaderString(resp.Header), body, query)
+		return nil, CommErrorf("%d (%s) URL: '%s', Request Headers: '%s', Headers: '%s', Body: '%s' Query: '%s'", statusCode, statusText, req.URL, req.Header, httputil.HeaderString(resp.Header), body, query)
+	}
+
+	return body, err
+}
+
+func (ctx *Context) query(query string) (interface{}, prometheus.Warnings, error) {
+	body, err := ctx.RawQuery(query)
+	if err != nil {
+		return nil, nil, err
 	}
 
 	var toReturn interface{}
 	err = json.Unmarshal(body, &toReturn)
 	if err != nil {
-		return nil, warnings, fmt.Errorf("query error: '%s' fetching query '%s'", err.Error(), query)
+		return nil, nil, fmt.Errorf("Unmarshal Error: %s\nQuery: %s", err, query)
+	}
+
+	warnings := warningsFrom(toReturn)
+	for _, w := range warnings {
+		// NoStoreAPIWarning is a warning that we would consider an error. It returns partial data relating only to the
+		// store apis which were reachable. In order to ensure integrity of data across all clusters, we'll need to identify
+		// this warning and convert it to an error.
+		if IsNoStoreAPIWarning(w) {
+			return nil, warnings, CommErrorf("Error: %s, Body: %s, Query: %s", w, body, query)
+		}
+
+		log.Warningf("fetching query '%s': %s", query, w)
 	}
 
 	return toReturn, warnings, nil
@@ -256,7 +287,8 @@ func runQueryRange(query string, start, end time.Time, step time.Duration, ctx *
 	resCh <- results
 }
 
-func (ctx *Context) queryRange(query string, start, end time.Time, step time.Duration) (interface{}, prometheus.Warnings, error) {
+// RawQuery is a direct query to the prometheus client and returns the body of the response
+func (ctx *Context) RawQueryRange(query string, start, end time.Time, step time.Duration) ([]byte, error) {
 	u := ctx.Client.URL(epQueryRange, nil)
 	q := u.Query()
 	q.Set("query", query)
@@ -267,40 +299,75 @@ func (ctx *Context) queryRange(query string, start, end time.Time, step time.Dur
 
 	req, err := http.NewRequest(http.MethodPost, u.String(), nil)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	resp, body, warnings, err := ctx.Client.Do(context.Background(), req)
-	for _, w := range warnings {
-		// NoStoreAPIWarning is a warning that we would consider an error. It returns partial data relating only to the
-		// store apis which were reachable. In order to ensure integrity of data across all clusters, we'll need to identify
-		// this warning and convert it to an error.
-		if IsNoStoreAPIWarning(w) {
-			return nil, warnings, NewCommError(fmt.Sprintf("Error: %s, Body: %s, Query: %s", w, body, query))
-		}
-
-		log.Warningf("fetching query '%s': %s", query, w)
+	// Set QueryContext name if non empty
+	if ctx.name != "" {
+		req = httputil.SetName(req, ctx.name)
 	}
+	req = httputil.SetQuery(req, query)
+
+	// Note that the warnings return value from client.Do() is always nil using this
+	// version of the prometheus client library. We parse the warnings out of the response
+	// body after json decodidng completes.
+	resp, body, _, err := ctx.Client.Do(context.Background(), req)
 	if err != nil {
 		if resp == nil {
-			return nil, warnings, fmt.Errorf("Error: %s, Body: %s Query: %s", err.Error(), body, query)
+			return nil, fmt.Errorf("Error: %s, Body: %s Query: %s", err.Error(), body, query)
 		}
 
-		return nil, warnings, fmt.Errorf("%d (%s) Headers: %s Error: %s Body: %s Query: %s", resp.StatusCode, http.StatusText(resp.StatusCode), util.HeaderString(resp.Header), body, err.Error(), query)
+		return nil, fmt.Errorf("%d (%s) Headers: %s Error: %s Body: %s Query: %s", resp.StatusCode, http.StatusText(resp.StatusCode), httputil.HeaderString(resp.Header), body, err.Error(), query)
 	}
 
 	// Unsuccessful Status Code, log body and status
 	statusCode := resp.StatusCode
 	statusText := http.StatusText(statusCode)
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return nil, warnings, CommErrorf("%d (%s) Headers: %s, Body: %s Query: %s", statusCode, statusText, util.HeaderString(resp.Header), body, query)
+		return nil, CommErrorf("%d (%s) Headers: %s, Body: %s Query: %s", statusCode, statusText, httputil.HeaderString(resp.Header), body, query)
+	}
+
+	return body, err
+}
+
+func (ctx *Context) queryRange(query string, start, end time.Time, step time.Duration) (interface{}, prometheus.Warnings, error) {
+	body, err := ctx.RawQueryRange(query, start, end, step)
+	if err != nil {
+		return nil, nil, err
 	}
 
 	var toReturn interface{}
 	err = json.Unmarshal(body, &toReturn)
 	if err != nil {
-		return nil, warnings, fmt.Errorf("%d (%s) Headers: %s Error: %s Body: %s Query: %s", statusCode, statusText, util.HeaderString(resp.Header), err.Error(), body, query)
+		return nil, nil, fmt.Errorf("Unmarshal Error: %s\nQuery: %s", err, query)
+	}
+
+	warnings := warningsFrom(toReturn)
+	for _, w := range warnings {
+		// NoStoreAPIWarning is a warning that we would consider an error. It returns partial data relating only to the
+		// store apis which were reachable. In order to ensure integrity of data across all clusters, we'll need to identify
+		// this warning and convert it to an error.
+		if IsNoStoreAPIWarning(w) {
+			return nil, warnings, CommErrorf("Error: %s, Body: %s, Query: %s", w, body, query)
+		}
+
+		log.Warningf("fetching query '%s': %s", query, w)
 	}
 
 	return toReturn, warnings, nil
 }
+
+// Extracts the warnings from the resulting json if they exist (part of the prometheus response api).
+func warningsFrom(result interface{}) prometheus.Warnings {
+	var warnings prometheus.Warnings
+
+	if resultMap, ok := result.(map[string]interface{}); ok {
+		if warningProp, ok := resultMap["warnings"]; ok {
+			if w, ok := warningProp.([]string); ok {
+				warnings = w
+			}
+		}
+	}
+
+	return warnings
+}

pkg/prom/query_test.go

@@ -0,0 +1,27 @@
+package prom
+
+import "testing"
+
+func TestWarningsFrom(t *testing.T) {
+	var results interface{}
+
+	results = map[string]interface{}{
+		"status": "success",
+		"warnings": []string{
+			"Warning #1",
+			"Warning #2",
+		},
+	}
+
+	warnings := warningsFrom(results)
+	if len(warnings) != 2 {
+		t.Errorf("Unexpected warnings length: %d, Expected 2.", len(warnings))
+	}
+
+	if warnings[0] != "Warning #1" {
+		t.Errorf("Unexpected first warning: %s", warnings[0])
+	}
+	if warnings[1] != "Warning #2" {
+		t.Errorf("Unexpected second warning: %s", warnings[1])
+	}
+}

pkg/prom/result.go

@@ -90,8 +90,8 @@ type QueryResults struct {
 // QueryResult contains a single result from a prometheus query. It's common
 // to refer to query results as a slice of QueryResult
 type QueryResult struct {
-	Metric map[string]interface{}
-	Values []*util.Vector
+	Metric map[string]interface{} `json:"metric"`
+	Values []*util.Vector         `json:"values"`
 }
 
 // NewQueryResults accepts the raw prometheus query result and returns an array of

pkg/clustermanager/boltdbstorage.go → pkg/services/clusters/boltdbstorage.go

@@ -1,15 +1,16 @@
-package clustermanager
+package clusters
 
 import (
 	bolt "go.etcd.io/bbolt"
-	_ "k8s.io/klog"
 )
 
+// BoltDBClusterStorage is a boltdb implementation of a database used to store cluster definitions
 type BoltDBClusterStorage struct {
 	bucket []byte
 	db     *bolt.DB
 }
 
+// NewBoltDBClusterStorage creates a new boltdb backed ClusterStorage implementation
 func NewBoltDBClusterStorage(bucket string, db *bolt.DB) (ClusterStorage, error) {
 	bucketKey := []byte(bucket)
 
@@ -32,7 +33,7 @@ func NewBoltDBClusterStorage(bucket string, db *bolt.DB) (ClusterStorage, error)
 	}, nil
 }
 
-// Adds the entry if the key does not exist
+// AddIfNotExists Adds the entry if the key does not exist
 func (cs *BoltDBClusterStorage) AddIfNotExists(key string, cluster []byte) error {
 	return cs.db.Update(func(tx *bolt.Tx) error {
 		k := []byte(key)
@@ -45,7 +46,7 @@ func (cs *BoltDBClusterStorage) AddIfNotExists(key string, cluster []byte) error
 	})
 }
 
-// Adds the encoded cluster to storage if it doesn't exist. Otherwise, update the existing
+// AddOrUpdate Adds the encoded cluster to storage if it doesn't exist. Otherwise, update the existing
 // value with the provided.
 func (cs *BoltDBClusterStorage) AddOrUpdate(key string, cluster []byte) error {
 	return cs.db.Update(func(tx *bolt.Tx) error {
@@ -55,7 +56,7 @@ func (cs *BoltDBClusterStorage) AddOrUpdate(key string, cluster []byte) error {
 	})
 }
 
-// Removes a key from the cluster storage
+// Remove Removes a key from the cluster storage
 func (cs *BoltDBClusterStorage) Remove(key string) error {
 	return cs.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket(cs.bucket)
@@ -64,7 +65,7 @@ func (cs *BoltDBClusterStorage) Remove(key string) error {
 	})
 }
 
-// Iterates through all key/values for the storage and calls the handler func. If a handler returns
+// Each Iterates through all key/values for the storage and calls the handler func. If a handler returns
 // an error, the iteration stops.
 func (cs *BoltDBClusterStorage) Each(handler func(string, []byte) error) error {
 	return cs.db.View(func(tx *bolt.Tx) error {
@@ -87,7 +88,7 @@ func (cs *BoltDBClusterStorage) Each(handler func(string, []byte) error) error {
 	})
 }
 
-// Closes the backing storage
+// Close Closes the backing storage
 func (cs *BoltDBClusterStorage) Close() error {
 	return cs.db.Close()
 }

pkg/clustermanager/clustermanager.go → pkg/services/clusters/clustermanager.go

@@ -1,4 +1,4 @@
-package clustermanager
+package clusters
 
 import (
 	"encoding/base64"
@@ -8,7 +8,7 @@ import (
 
 	"github.com/google/uuid"
 
-	"github.com/kubecost/cost-model/pkg/util"
+	"github.com/kubecost/cost-model/pkg/util/fileutil"
 	"github.com/kubecost/cost-model/pkg/util/json"
 
 	"k8s.io/klog"
@@ -71,6 +71,7 @@ type ClusterStorage interface {
 	Close() error
 }
 
+// ClusterManager provides an implementation
 type ClusterManager struct {
 	storage ClusterStorage
 	// cache   map[string]*ClusterDefinition
@@ -88,7 +89,7 @@ func NewClusterManager(storage ClusterStorage) *ClusterManager {
 func NewConfiguredClusterManager(storage ClusterStorage, config string) *ClusterManager {
 	clusterManager := NewClusterManager(storage)
 
-	exists, err := util.FileExists(config)
+	exists, err := fileutil.FileExists(config)
 	if !exists {
 		if err != nil {
 			klog.V(1).Infof("[Error] Failed to load config file: %s. Error: %s", config, err.Error())
@@ -133,7 +134,7 @@ func NewConfiguredClusterManager(storage ClusterStorage, config string) *Cluster
 	return clusterManager
 }
 
-// Adds, but will not update an existing entry.
+// Add Adds a cluster definition, but will not update an existing entry.
 func (cm *ClusterManager) Add(cluster ClusterDefinition) (*ClusterDefinition, error) {
 	// First time add
 	if cluster.ID == "" {
@@ -153,6 +154,8 @@ func (cm *ClusterManager) Add(cluster ClusterDefinition) (*ClusterDefinition, er
 	return &cluster, nil
 }
 
+// AddOrUpdate will add the cluster definition if it doesn't exist, or update the existing definition
+// if it does exist.
 func (cm *ClusterManager) AddOrUpdate(cluster ClusterDefinition) (*ClusterDefinition, error) {
 	// First time add
 	if cluster.ID == "" {
@@ -172,10 +175,12 @@ func (cm *ClusterManager) AddOrUpdate(cluster ClusterDefinition) (*ClusterDefini
 	return &cluster, nil
 }
 
+// Remove will remove a cluster definition by id.
 func (cm *ClusterManager) Remove(id string) error {
 	return cm.storage.Remove(id)
 }
 
+// GetAll will return all of the cluster definitions
 func (cm *ClusterManager) GetAll() []*ClusterDefinition {
 	clusters := []*ClusterDefinition{}
 
@@ -198,6 +203,7 @@ func (cm *ClusterManager) GetAll() []*ClusterDefinition {
 	return clusters
 }
 
+// Close will close the backing database
 func (cm *ClusterManager) Close() error {
 	return cm.storage.Close()
 }
@@ -212,7 +218,7 @@ func fileFromSecret(secretName string) string {
 
 func fromSecret(secretName string) (string, error) {
 	file := fileFromSecret(secretName)
-	exists, err := util.FileExists(file)
+	exists, err := fileutil.FileExists(file)
 	if !exists || err != nil {
 		return "", fmt.Errorf("Failed to locate secret: %s", file)
 	}

pkg/clustermanager/clustersendpoints.go → pkg/services/clusters/clustersendpoints.go

@@ -1,4 +1,4 @@
-package clustermanager
+package clusters
 
 import (
 	"errors"
@@ -18,27 +18,37 @@ type DataEnvelope struct {
 	Data   interface{} `json:"data"`
 }
 
-type ClusterManagerEndpoints struct {
+// ClusterManagerHTTPService is an implementation of HTTPService which provides
+// the frontend with the ability to manage stored cluster definitions.
+type ClusterManagerHTTPService struct {
 	manager *ClusterManager
 }
 
-func NewClusterManagerEndpoints(manager *ClusterManager) *ClusterManagerEndpoints {
-	return &ClusterManagerEndpoints{
+// NewClusterManagerHTTPService creates a new cluster management http service
+func NewClusterManagerHTTPService(manager *ClusterManager) *ClusterManagerHTTPService {
+	return &ClusterManagerHTTPService{
 		manager: manager,
 	}
 }
 
-func (cme *ClusterManagerEndpoints) GetAllClusters(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+// Register assigns the endpoints and returns an error on failure.
+func (cme *ClusterManagerHTTPService) Register(router *httprouter.Router) error {
+	router.GET("/clusters", cme.GetAllClusters)
+	router.PUT("/clusters", cme.PutCluster)
+	router.DELETE("/clusters/:id", cme.DeleteCluster)
+
+	return nil
+}
+
+func (cme *ClusterManagerHTTPService) GetAllClusters(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("Access-Control-Allow-Origin", "*")
 
 	clusters := cme.manager.GetAll()
 	w.Write(wrapData(clusters, nil))
 }
 
-func (cme *ClusterManagerEndpoints) PutCluster(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+func (cme *ClusterManagerHTTPService) PutCluster(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("Access-Control-Allow-Origin", "*")
 
 	data, err := ioutil.ReadAll(r.Body)
 	if err != nil {
@@ -62,9 +72,8 @@ func (cme *ClusterManagerEndpoints) PutCluster(w http.ResponseWriter, r *http.Re
 	w.Write(wrapData(cd, nil))
 }
 
-func (cme *ClusterManagerEndpoints) DeleteCluster(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+func (cme *ClusterManagerHTTPService) DeleteCluster(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("Access-Control-Allow-Origin", "*")
 
 	clusterID := ps.ByName("id")
 	if clusterID == "" {

pkg/clustermanager/mapdbstorage.go → pkg/services/clusters/mapdbstorage.go

@@ -1,20 +1,18 @@
-package clustermanager
-
-import (
-	_ "k8s.io/klog"
-)
+package clusters
 
+// MapDBClusterStorage is a map implementation of a database used to store cluster definitions
 type MapDBClusterStorage struct {
 	store map[string][]byte
 }
 
+// NewMapDBClusterStorage creates a new map backed ClusterStorage implementation
 func NewMapDBClusterStorage() ClusterStorage {
 	return &MapDBClusterStorage{
 		store: make(map[string][]byte),
 	}
 }
 
-// Adds the entry if the key does not exist
+// AddIfNotExists Adds the entry if the key does not exist
 func (cs *MapDBClusterStorage) AddIfNotExists(key string, cluster []byte) error {
 	if _, ok := cs.store[key]; !ok {
 		cs.store[key] = cluster
@@ -22,20 +20,20 @@ func (cs *MapDBClusterStorage) AddIfNotExists(key string, cluster []byte) error
 	return nil
 }
 
-// Adds the encoded cluster to storage if it doesn't exist. Otherwise, update the existing
+// AddOrUpdate Adds the encoded cluster to storage if it doesn't exist. Otherwise, update the existing
 // value with the provided.
 func (cs *MapDBClusterStorage) AddOrUpdate(key string, cluster []byte) error {
 	cs.store[key] = cluster
 	return nil
 }
 
-// Removes a key from the cluster storage
+// Remove Removes a key from the cluster storage
 func (cs *MapDBClusterStorage) Remove(key string) error {
 	delete(cs.store, key)
 	return nil
 }
 
-// Iterates through all key/values for the storage and calls the handler func. If a handler returns
+// Each Iterates through all key/values for the storage and calls the handler func. If a handler returns
 // an error, the iteration stops.
 func (cs *MapDBClusterStorage) Each(handler func(string, []byte) error) error {
 	for k, v := range cs.store {
@@ -49,7 +47,7 @@ func (cs *MapDBClusterStorage) Each(handler func(string, []byte) error) error {
 	return nil
 }
 
-// Closes the backing storage
+// Close Closes the backing storage
 func (cs *MapDBClusterStorage) Close() error {
 	return nil
 }

pkg/services/clusterservice.go

@@ -0,0 +1,37 @@
+package services
+
+import "github.com/kubecost/cost-model/pkg/services/clusters"
+
+// NewClusterManagerService creates a new HTTPService implementation driving cluster definition management
+// for the frontend
+func NewClusterManagerService() HTTPService {
+	return clusters.NewClusterManagerHTTPService(newClusterManager())
+}
+
+// newClusterManager creates a new cluster manager instance for use in the service
+func newClusterManager() *clusters.ClusterManager {
+	clustersConfigFile := "/var/configs/clusters/default-clusters.yaml"
+
+	// Return a memory-backed cluster manager populated by configmap
+	return clusters.NewConfiguredClusterManager(clusters.NewMapDBClusterStorage(), clustersConfigFile)
+
+	// NOTE: The following should be used with a persistent disk store. Since the
+	// NOTE: configmap approach is currently the "persistent" source (entries are read-only
+	// NOTE: on the backend), we don't currently need to store on disk.
+	/*
+		path := env.GetConfigPath()
+		db, err := bolt.Open(path+"costmodel.db", 0600, nil)
+		if err != nil {
+			klog.V(1).Infof("[Error] Failed to create costmodel.db: %s", err.Error())
+			return cm.NewConfiguredClusterManager(cm.NewMapDBClusterStorage(), clustersConfigFile)
+		}
+
+		store, err := clusters.NewBoltDBClusterStorage("clusters", db)
+		if err != nil {
+			klog.V(1).Infof("[Error] Failed to Create Cluster Storage: %s", err.Error())
+			return clusters.NewConfiguredClusterManager(clusters.NewMapDBClusterStorage(), clustersConfigFile)
+		}
+
+		return clusters.NewConfiguredClusterManager(store, clustersConfigFile)
+	*/
+}

pkg/services/services.go

@@ -0,0 +1,65 @@
+package services
+
+import (
+	"sync"
+
+	"github.com/julienschmidt/httprouter"
+	"github.com/kubecost/cost-model/pkg/log"
+)
+
+// HTTPService defines an implementation prototype for an object capable of registering
+// endpoints on an http router which provide an service relevant to the cost-model.
+type HTTPService interface {
+	// Register assigns the endpoints and returns an error on failure.
+	Register(*httprouter.Router) error
+}
+
+// HTTPServices defines an implementation prototype for an object capable of managing and registering
+// predefined HTTPService routes
+type HTTPServices interface {
+	// Add a HTTPService implementation for registration
+	Add(service HTTPService)
+
+	// RegisterAll registers all the services added with the provided router
+	RegisterAll(*httprouter.Router) error
+}
+
+type defaultHTTPServices struct {
+	sync.Mutex
+	services []HTTPService
+}
+
+// Add a HTTPService implementation for
+func (dhs *defaultHTTPServices) Add(service HTTPService) {
+	if service == nil {
+		log.Warningf("Attempting to Add nil HTTPService")
+		return
+	}
+
+	dhs.Lock()
+	defer dhs.Unlock()
+
+	dhs.services = append(dhs.services, service)
+}
+
+// RegisterAll registers all the services added with the provided router
+func (dhs *defaultHTTPServices) RegisterAll(router *httprouter.Router) error {
+	dhs.Lock()
+	defer dhs.Unlock()
+
+	for _, svc := range dhs.services {
+		svc.Register(router)
+	}
+
+	return nil
+}
+
+// NewCostModelServices creates an HTTPServices implementation containing any predefined
+// http services used with the cost-model
+func NewCostModelServices() HTTPServices {
+	return &defaultHTTPServices{
+		services: []HTTPService{
+			NewClusterManagerService(),
+		},
+	}
+}

pkg/util/atomic.go → pkg/util/atomic/atomicint.go

@@ -1,4 +1,4 @@
-package util
+package atomic
 
 import "sync/atomic"
 

pkg/util/buffer.go

@@ -8,6 +8,8 @@ import (
 	"math"
 	"reflect"
 	"unsafe"
+
+	"github.com/kubecost/cost-model/pkg/util/stringutil"
 )
 
 // NonPrimitiveTypeError represents an error where the user provided a non-primitive data type for reading/writing
@@ -394,7 +396,7 @@ func bytesToString(b []byte) string {
 	// future optimization.
 	//return *(*string)(unsafe.Pointer(&b))
 
-	return string(b)
+	return stringutil.Bank(string(b))
 }
 
 // Direct string to byte conversion that doesn't allocate.

pkg/util/cloudutil/aws.go

@@ -0,0 +1,54 @@
+package cloudutil
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/kubecost/cost-model/pkg/log"
+)
+
+// ConvertToGlueColumnFormat takes a string and runs through various regex
+// and string replacement statements to convert it to a format compatible
+// with AWS Glue and Athena column names.
+// Following guidance from AWS provided here ('Column Names' section):
+// https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/run-athena-sql.html
+// It returns a string containing the column name in proper column name format and length.
+func ConvertToGlueColumnFormat(columnName string) string {
+	log.Debugf("Converting string \"%s\" to proper AWS Glue column name.", columnName)
+
+	// An underscore is added in front of uppercase letters
+	capitalUnderscore := regexp.MustCompile(`[A-Z]`)
+	final := capitalUnderscore.ReplaceAllString(columnName, `_$0`)
+
+	// Any non-alphanumeric characters are replaced with an underscore
+	noSpacePunc := regexp.MustCompile(`[\s]{1,}|[^A-Za-z0-9]`)
+	final = noSpacePunc.ReplaceAllString(final, "_")
+
+	// Duplicate underscores are removed
+	noDupUnderscore := regexp.MustCompile(`_{2,}`)
+	final = noDupUnderscore.ReplaceAllString(final, "_")
+
+	// Any leading and trailing underscores are removed
+	noFrontUnderscore := regexp.MustCompile(`(^\_|\_$)`)
+	final = noFrontUnderscore.ReplaceAllString(final, "")
+
+	// Uppercase to lowercase
+	final = strings.ToLower(final)
+
+	// Longer column name than expected - remove _ left to right
+	allowedColLen := 128
+	underscoreToRemove := len(final) - allowedColLen
+	if underscoreToRemove > 0 {
+		final = strings.Replace(final, "_", "", underscoreToRemove)
+	}
+
+	// If removing all of the underscores still didn't
+	// make the column name < 128 characters, trim it!
+	if len(final) > allowedColLen {
+		final = final[:allowedColLen]
+	}
+
+	log.Debugf("Column name being returned: \"%s\". Length: \"%d\".", final, len(final))
+
+	return final
+}

pkg/util/file.go → pkg/util/fileutil/fileutil.go

@@ -1,4 +1,4 @@
-package util
+package fileutil
 
 import "os"
 

pkg/util/http.go → pkg/util/httputil/httputil.go

@@ -1,4 +1,4 @@
-package util
+package httputil
 
 import (
 	"context"
@@ -43,6 +43,8 @@ func NewQueryParams(values url.Values) QueryParams {
 
 const (
 	ContextWarning string = "Warning"
+	ContextName    string = "Name"
+	ContextQuery   string = "Query"
 )
 
 // GetWarning Extracts a warning message from the request context if it exists
@@ -58,6 +60,32 @@ func SetWarning(r *http.Request, warning string) *http.Request {
 	return r.WithContext(ctx)
 }
 
+// GetName Extracts a name value from the request context if it exists
+func GetName(r *http.Request) (name string, ok bool) {
+	name, ok = r.Context().Value(ContextName).(string)
+	return
+}
+
+// SetName Sets the name value on the provided request and returns a new instance of the request
+// with the new context.
+func SetName(r *http.Request, name string) *http.Request {
+	ctx := context.WithValue(r.Context(), ContextName, name)
+	return r.WithContext(ctx)
+}
+
+// GetQuery Extracts a query value from the request context if it exists
+func GetQuery(r *http.Request) (name string, ok bool) {
+	name, ok = r.Context().Value(ContextQuery).(string)
+	return
+}
+
+// SetQuery Sets the query value on the provided request and returns a new instance of the request
+// with the new context.
+func SetQuery(r *http.Request, query string) *http.Request {
+	ctx := context.WithValue(r.Context(), ContextQuery, query)
+	return r.WithContext(ctx)
+}
+
 //--------------------------------------------------------------------------
 //  Package Funcs
 //--------------------------------------------------------------------------

test/util_test.go → pkg/util/httputil/httputil_test.go

@@ -1,10 +1,8 @@
-package test
+package httputil
 
 import (
 	"net/http"
 	"testing"
-
-	"github.com/kubecost/cost-model/pkg/util"
 )
 
 func TestHeaderString(t *testing.T) {
@@ -14,7 +12,7 @@ func TestHeaderString(t *testing.T) {
 	h.Add("bar", "foo")
 	h.Add("Content-Type", "application/octet-stream")
 
-	s := util.HeaderString(h)
+	s := HeaderString(h)
 	if len(s) == 0 {
 		t.Errorf("Header String failed to produce a valid output")
 		return
@@ -26,7 +24,7 @@ func TestHeaderString(t *testing.T) {
 func TestEmptyHeader(t *testing.T) {
 	h := make(http.Header)
 
-	s := util.HeaderString(h)
+	s := HeaderString(h)
 	if len(s) == 0 {
 		t.Errorf("Header String failed to produce a valid output")
 		return
@@ -38,7 +36,7 @@ func TestEmptyHeader(t *testing.T) {
 func TestNilHeader(t *testing.T) {
 	var h http.Header
 
-	s := util.HeaderString(h)
+	s := HeaderString(h)
 	if len(s) == 0 {
 		t.Errorf("Header String failed to produce a valid output")
 		return

pkg/util/mapper/mapper.go

@@ -1,7 +1,7 @@
 package mapper
 
 import (
-	"fmt"
+	"github.com/kubecost/cost-model/pkg/util/timeutil"
 	"strconv"
 	"strings"
 	"time"
@@ -397,7 +397,7 @@ func (rom *readOnlyMapper) GetBool(key string, defaultValue bool) bool {
 func (rom *readOnlyMapper) GetDuration(key string, defaultValue time.Duration) time.Duration {
 	r := rom.getter.Get(key)
 
-	d, err := parseDuration(r)
+	d, err := timeutil.ParseDuration(r)
 	if err != nil {
 		return defaultValue
 	}
@@ -488,7 +488,7 @@ func (wom *writeOnlyMapper) SetBool(key string, value bool) error {
 
 // SetDuration sets the map to a string formatted bool value.
 func (wom *writeOnlyMapper) SetDuration(key string, value time.Duration) error {
-	return wom.setter.Set(key, durationString(value))
+	return wom.setter.Set(key, timeutil.DurationString(value))
 }
 
 // SetList sets the map's value at key to a string consistent of each value in the list separated
@@ -496,67 +496,3 @@ func (wom *writeOnlyMapper) SetDuration(key string, value time.Duration) error {
 func (wom *writeOnlyMapper) SetList(key string, values []string, delimiter string) error {
 	return wom.setter.Set(key, strings.Join(values, delimiter))
 }
-
-const (
-	secsPerMin  = 60.0
-	secsPerHour = 3600.0
-	secsPerDay  = 86400.0
-)
-
-// durationString converts duration to a string of the form "4d", "4h", "4m", or "4s" if
-// the number of seconds in the string is evenly divisible into an integer number of
-// days, hours, minutes, or seconds respectively.
-func durationString(duration time.Duration) string {
-	durSecs := int64(duration.Seconds())
-
-	durStr := ""
-	if durSecs > 0 {
-		if durSecs%secsPerDay == 0 {
-			// convert to days
-			durStr = fmt.Sprintf("%dd", durSecs/secsPerDay)
-		} else if durSecs%secsPerHour == 0 {
-			// convert to hours
-			durStr = fmt.Sprintf("%dh", durSecs/secsPerHour)
-		} else if durSecs%secsPerMin == 0 {
-			// convert to mins
-			durStr = fmt.Sprintf("%dm", durSecs/secsPerMin)
-		} else if durSecs > 0 {
-			// default to mins, as long as duration is positive
-			durStr = fmt.Sprintf("%ds", durSecs)
-		}
-	}
-
-	return durStr
-}
-
-func parseDuration(duration string) (time.Duration, error) {
-	var amountStr string
-	var unit time.Duration
-	switch {
-	case strings.HasSuffix(duration, "s"):
-		unit = time.Second
-		amountStr = strings.TrimSuffix(duration, "s")
-	case strings.HasSuffix(duration, "m"):
-		unit = time.Minute
-		amountStr = strings.TrimSuffix(duration, "m")
-	case strings.HasSuffix(duration, "h"):
-		unit = time.Hour
-		amountStr = strings.TrimSuffix(duration, "h")
-	case strings.HasSuffix(duration, "d"):
-		unit = 24.0 * time.Hour
-		amountStr = strings.TrimSuffix(duration, "d")
-	default:
-		return 0, fmt.Errorf("error parsing duration: %s did not match expected format [0-9+](s|m|d|h)", duration)
-	}
-
-	if len(amountStr) == 0 {
-		return 0, fmt.Errorf("error parsing duration: %s did not match expected format [0-9+](s|m|d|h)", duration)
-	}
-
-	amount, err := strconv.ParseInt(amountStr, 10, 64)
-	if err != nil {
-		return 0, fmt.Errorf("error parsing duration: %s did not match expected format [0-9+](s|m|d|h)", duration)
-	}
-
-	return time.Duration(amount) * unit, nil
-}

pkg/util/retry/retry.go

@@ -36,7 +36,7 @@ func Retry(ctx context.Context, f func() (interface{}, error), attempts uint, de
 
 		time.Sleep(d)
 
-		jitter := time.Duration(rand.Int63n(int64(d)))
+		jitter := time.Duration(rand.Int63n(int64(d))) // #nosec No need for a cryptographic strength random here
 		d = d + jitter/2
 	}
 

pkg/util/retry/retry_test.go

@@ -13,6 +13,7 @@ type Obj struct {
 }
 
 func TestPtrSliceRetry(t *testing.T) {
+	t.Parallel()
 	const Expected uint64 = 3
 
 	var count uint64 = 0
@@ -42,6 +43,7 @@ func TestPtrSliceRetry(t *testing.T) {
 }
 
 func TestSuccessRetry(t *testing.T) {
+	t.Parallel()
 	const Expected uint64 = 3
 
 	var count uint64 = 0
@@ -64,6 +66,7 @@ func TestSuccessRetry(t *testing.T) {
 }
 
 func TestFailRetry(t *testing.T) {
+	t.Parallel()
 	const Expected uint64 = 5
 
 	expectedError := fmt.Sprintf("Failed: %d", Expected)
@@ -86,6 +89,8 @@ func TestFailRetry(t *testing.T) {
 }
 
 func TestCancelRetry(t *testing.T) {
+	t.Parallel()
+
 	const Expected uint64 = 5
 
 	var count uint64 = 0

pkg/util/strings.go → pkg/util/stringutil/stringutil.go

@@ -1,19 +1,13 @@
-package util
+package stringutil
 
 import (
 	"fmt"
 	"math"
 	"math/rand"
+	"sync"
 	"time"
 )
 
-func init() {
-	rand.Seed(time.Now().UnixNano())
-}
-
-var alpha = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
-var alphanumeric = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
-
 const (
 	_ = 1 << (10 * iota)
 	// KiB is bytes per Kibibyte
@@ -26,11 +20,30 @@ const (
 	TiB
 )
 
+var alpha = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+var alphanumeric = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
+
+// Any strings created at runtime, duplicate or not, are copied, even though by specification,
+// a go string is immutable. This utility allows us to cache runtime strings and retrieve them
+// when we expect heavy duplicates.
+var strings sync.Map
+
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+// Bank will return a non-copy of a string if it has been used before. Otherwise, it will store
+// the string as the unique instance.
+func Bank(s string) string {
+	ss, _ := strings.LoadOrStore(s, s)
+	return ss.(string)
+}
+
 // RandSeq generates a pseudo-random alphabetic string of the given length
 func RandSeq(n int) string {
 	b := make([]rune, n)
 	for i := range b {
-		b[i] = alpha[rand.Intn(len(alpha))]
+		b[i] = alpha[rand.Intn(len(alpha))] // #nosec No need for a cryptographic strength random here
 	}
 	return string(b)
 }

pkg/util/time_test.go

@@ -1,56 +0,0 @@
-package util
-
-import (
-	"testing"
-	"time"
-)
-
-func TestDurationOffsetStrings(t *testing.T) {
-	dur, off := "", ""
-
-	dur, off = DurationOffsetStrings(0, 0)
-	if dur != "" || off != "" {
-		t.Fatalf("DurationOffsetStrings: exp (%s %s); act (%s, %s)", "", "", dur, off)
-	}
-
-	dur, off = DurationOffsetStrings(24*time.Hour, 0)
-	if dur != "1d" || off != "" {
-		t.Fatalf("DurationOffsetStrings: exp (%s %s); act (%s, %s)", "1d", "", dur, off)
-	}
-
-	dur, off = DurationOffsetStrings(24*time.Hour+5*time.Minute, 0)
-	if dur != "1445m" || off != "" {
-		t.Fatalf("DurationOffsetStrings: exp (%s %s); act (%s, %s)", "1445m", "", dur, off)
-	}
-
-	dur, off = DurationOffsetStrings(25*time.Hour, 5*time.Minute)
-	if dur != "25h" || off != "5m" {
-		t.Fatalf("DurationOffsetStrings: exp (%s %s); act (%s, %s)", "25h", "5m", dur, off)
-	}
-
-	dur, off = DurationOffsetStrings(25*time.Hour, 60*time.Minute)
-	if dur != "25h" || off != "1h" {
-		t.Fatalf("DurationOffsetStrings: exp (%s %s); act (%s, %s)", "25h", "1h", dur, off)
-	}
-
-	dur, off = DurationOffsetStrings(72*time.Hour, 1440*time.Minute)
-	if dur != "3d" || off != "1d" {
-		t.Fatalf("DurationOffsetStrings: exp (%s %s); act (%s, %s)", "3d", "1d", dur, off)
-	}
-
-	dur, off = DurationOffsetStrings(25*time.Hour, 1*time.Second)
-	if dur != "25h" || off != "1s" {
-		t.Fatalf("DurationOffsetStrings: exp (%s %s); act (%s, %s)", "25h", "1s", dur, off)
-	}
-
-	dur, off = DurationOffsetStrings(24*time.Hour+time.Second, 1*time.Second)
-	if dur != "86401s" || off != "1s" {
-		t.Fatalf("DurationOffsetStrings: exp (%s %s); act (%s, %s)", "86401s", "1s", dur, off)
-	}
-
-	// Expect empty strings if durations are negative
-	dur, off = DurationOffsetStrings(-25*time.Hour, -1*time.Second)
-	if dur != "" || off != "" {
-		t.Fatalf("DurationOffsetStrings: exp (%s %s); act (%s, %s)", "", "", dur, off)
-	}
-}

pkg/util/time.go → pkg/util/timeutil/timeutil.go

@@ -1,8 +1,10 @@
-package util
+package timeutil
 
 import (
 	"fmt"
+	"regexp"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 )
@@ -50,7 +52,7 @@ func DurationString(duration time.Duration) string {
 			// convert to mins
 			durStr = fmt.Sprintf("%dm", durSecs/SecsPerMin)
 		} else if durSecs > 0 {
-			// default to mins, as long as duration is positive
+			// default to secs, as long as duration is positive
 			durStr = fmt.Sprintf("%ds", durSecs)
 		}
 	}
@@ -58,14 +60,39 @@ func DurationString(duration time.Duration) string {
 	return durStr
 }
 
+// DurationToPromOffsetString returns a Prometheus formatted string with leading offset or empty string if given a negative duration
+func DurationToPromOffsetString(duration time.Duration) string {
+	dirStr := DurationString(duration)
+	if dirStr != "" {
+		dirStr = fmt.Sprintf("offset %s", dirStr)
+	}
+	return dirStr
+}
+
 // DurationOffsetStrings converts a (duration, offset) pair to Prometheus-
 // compatible strings in terms of days, hours, minutes, or seconds.
 func DurationOffsetStrings(duration, offset time.Duration) (string, string) {
 	return DurationString(duration), DurationString(offset)
 }
 
+// FormatStoreResolution provides a clean notation for ETL store resolutions.
+// e.g. daily => 1d; hourly => 1h
+func FormatStoreResolution(dur time.Duration) string {
+	if dur >= 24*time.Hour {
+		return fmt.Sprintf("%dd", int(dur.Hours()/24.0))
+	} else if dur >= time.Hour {
+		return fmt.Sprintf("%dh", int(dur.Hours()))
+	}
+	return fmt.Sprint(dur)
+}
+
 // ParseDuration converts a Prometheus-style duration string into a Duration
-func ParseDuration(duration string) (*time.Duration, error) {
+func ParseDuration(duration string) (time.Duration, error) {
+	// Trim prefix of Prometheus format duration
+	duration = CleanDurationString(duration)
+	if len(duration) < 2 {
+		return 0, fmt.Errorf("error parsing duration: %s did not match expected format [0-9+](s|m|d|h)", duration)
+	}
 	unitStr := duration[len(duration)-1:]
 	var unit time.Duration
 	switch unitStr {
@@ -78,52 +105,51 @@ func ParseDuration(duration string) (*time.Duration, error) {
 	case "d":
 		unit = 24.0 * time.Hour
 	default:
-		return nil, fmt.Errorf("error parsing duration: %s did not match expected format [0-9+](s|m|d|h)", duration)
+		return 0, fmt.Errorf("error parsing duration: %s did not match expected format [0-9+](s|m|d|h)", duration)
 	}
 
 	amountStr := duration[:len(duration)-1]
 	amount, err := strconv.ParseInt(amountStr, 10, 64)
 	if err != nil {
-		return nil, fmt.Errorf("error parsing duration: %s did not match expected format [0-9+](s|m|d|h)", duration)
+		return 0, fmt.Errorf("error parsing duration: %s did not match expected format [0-9+](s|m|d|h)", duration)
 	}
 
-	dur := time.Duration(amount) * unit
-	return &dur, nil
+	return time.Duration(amount) * unit, nil
+}
+
+// CleanDurationString removes prometheus formatted prefix "offset " allong with leading a trailing whitespace
+// from duration string, leaving behind a string with format [0-9+](s|m|d|h)
+func CleanDurationString(duration string) string {
+	duration = strings.TrimSpace(duration)
+	duration = strings.TrimPrefix(duration, "offset ")
+	return duration
 }
 
 // ParseTimeRange returns a start and end time, respectively, which are converted from
 // a duration and offset, defined as strings with Prometheus-style syntax.
-func ParseTimeRange(duration, offset string) (*time.Time, *time.Time, error) {
+func ParseTimeRange(duration, offset time.Duration) (time.Time, time.Time) {
 	// endTime defaults to the current time, unless an offset is explicity declared,
 	// in which case it shifts endTime back by given duration
 	endTime := time.Now()
-	if offset != "" {
-		o, err := ParseDuration(offset)
-		if err != nil {
-			return nil, nil, fmt.Errorf("error parsing offset (%s): %s", offset, err)
-		}
-		endTime = endTime.Add(-1 * *o)
+	if offset > 0 {
+		endTime = endTime.Add(-1 * offset)
 	}
 
-	// if duration is defined in terms of days, convert to hours
-	// e.g. convert "2d" to "48h"
-	durationNorm, err := normalizeTimeParam(duration)
-	if err != nil {
-		return nil, nil, fmt.Errorf("error parsing duration (%s): %s", duration, err)
-	}
-
-	// convert time duration into start and end times, formatted
-	// as ISO datetime strings
-	dur, err := time.ParseDuration(durationNorm)
-	if err != nil {
-		return nil, nil, fmt.Errorf("errorf parsing duration (%s): %s", durationNorm, err)
-	}
-	startTime := endTime.Add(-1 * dur)
+	startTime := endTime.Add(-1 * duration)
 
-	return &startTime, &endTime, nil
+	return startTime, endTime
 }
 
-func normalizeTimeParam(param string) (string, error) {
+// FormatDurationStringDaysToHours converts string from format [0-9+]d to [0-9+]h
+func FormatDurationStringDaysToHours(param string) (string, error) {
+	//check that input matches format
+	ok, err := regexp.MatchString("[0-9+]d", param)
+	if !ok {
+		return param, fmt.Errorf("FormatDurationStringDaysToHours: input string (%s) not formatted as [0-9+]d", param)
+	}
+	if err != nil {
+		return "", err
+	}
 	// convert days to hours
 	if param[len(param)-1:] == "d" {
 		count := param[:len(param)-1]
@@ -138,17 +164,6 @@ func normalizeTimeParam(param string) (string, error) {
 	return param, nil
 }
 
-// FormatStoreResolution provides a clean notation for ETL store resolutions.
-// e.g. daily => 1d; hourly => 1h
-func FormatStoreResolution(dur time.Duration) string {
-	if dur >= 24*time.Hour {
-		return fmt.Sprintf("%dd", int(dur.Hours()/24.0))
-	} else if dur >= time.Hour {
-		return fmt.Sprintf("%dh", int(dur.Hours()))
-	}
-	return fmt.Sprint(dur)
-}
-
 // JobTicker is a ticker used to synchronize the next run of a repeating
 // process. The designated use-case is for infinitely-looping selects,
 // where a timeout or an exit channel might cancel the process, but otherwise

pkg/util/timeutil/timeutil_test.go

@@ -0,0 +1,378 @@
+package timeutil
+
+import (
+	"testing"
+	"time"
+)
+
+func Test_DurationString(t *testing.T) {
+	testCases := map[string]struct {
+		duration         time.Duration
+		expectedDuration string
+	}{
+		"1a": {
+			duration:         0,
+			expectedDuration: "",
+		},
+		"1b": {
+			duration:         24 * time.Hour,
+			expectedDuration: "1d",
+		},
+		"1c": {
+			duration:         24*time.Hour + 5*time.Minute,
+			expectedDuration: "1445m",
+		},
+		"1d": {
+			duration:         25 * time.Hour,
+			expectedDuration: "25h",
+		},
+		"1e": {
+			duration:         25 * time.Hour,
+			expectedDuration: "25h",
+		},
+		"1f": {
+			duration:         72 * time.Hour,
+			expectedDuration: "3d",
+		},
+		"1g": {
+			duration:         25 * time.Hour,
+			expectedDuration: "25h",
+		},
+		"1h": {
+			duration:         24*time.Hour + time.Second,
+			expectedDuration: "86401s",
+		},
+		// Expect empty strings if durations are negative
+		"1i": {
+			duration:         -25 * time.Hour,
+			expectedDuration: "",
+		},
+	}
+
+	for name, test := range testCases {
+		t.Run(name, func(t *testing.T) {
+			dur := DurationString(test.duration)
+			if dur != test.expectedDuration {
+				t.Fatalf("DurationOffsetStrings: exp (%s); act (%s)", test.expectedDuration, dur)
+			}
+		})
+	}
+}
+
+func Test_DurationToPromOffsetString(t *testing.T) {
+	testCases := map[string]struct {
+		duration         time.Duration
+		expectedDuration string
+	}{
+		"1a": {
+			duration:         0,
+			expectedDuration: "",
+		},
+		"1b": {
+			duration:         24 * time.Hour,
+			expectedDuration: "offset 1d",
+		},
+		"1c": {
+			duration:         24*time.Hour + 5*time.Minute,
+			expectedDuration: "offset 1445m",
+		},
+		"1d": {
+			duration:         25 * time.Hour,
+			expectedDuration: "offset 25h",
+		},
+		"1e": {
+			duration:         25 * time.Hour,
+			expectedDuration: "offset 25h",
+		},
+		"1f": {
+			duration:         72 * time.Hour,
+			expectedDuration: "offset 3d",
+		},
+		"1g": {
+			duration:         25 * time.Hour,
+			expectedDuration: "offset 25h",
+		},
+		"1h": {
+			duration:         24*time.Hour + time.Second,
+			expectedDuration: "offset 86401s",
+		},
+		// Expect empty strings if durations are negative
+		"1i": {
+			duration:         -25 * time.Hour,
+			expectedDuration: "",
+		},
+	}
+
+	for name, test := range testCases {
+		t.Run(name, func(t *testing.T) {
+			dur := DurationToPromOffsetString(test.duration)
+			if dur != test.expectedDuration {
+				t.Fatalf("DurationOffsetStrings: exp (%s); act (%s)", test.expectedDuration, dur)
+			}
+		})
+	}
+}
+
+func Test_FormatStoreResolution(t *testing.T) {
+	testCases := map[string]struct {
+		duration         time.Duration
+		expectedDuration string
+	}{
+		"1a": {
+			duration:         0,
+			expectedDuration: "0s",
+		},
+		"1b": {
+			duration:         24 * time.Hour,
+			expectedDuration: "1d",
+		},
+		"1c": {
+			duration:         24*time.Hour + 5*time.Minute,
+			expectedDuration: "1d",
+		},
+		"1d": {
+			duration:         25 * time.Hour,
+			expectedDuration: "1d",
+		},
+		"1e": {
+			duration:         25 * time.Hour,
+			expectedDuration: "1d",
+		},
+		"1f": {
+			duration:         72 * time.Hour,
+			expectedDuration: "3d",
+		},
+		"1g": {
+			duration:         25 * time.Hour,
+			expectedDuration: "1d",
+		},
+		"1h": {
+			duration:         24*time.Hour + time.Second,
+			expectedDuration: "1d",
+		},
+		// Expect empty strings if durations are negative
+		"1i": {
+			duration:         -25 * time.Hour,
+			expectedDuration: "-25h0m0s",
+		},
+	}
+
+	for name, test := range testCases {
+		t.Run(name, func(t *testing.T) {
+			dur := FormatStoreResolution(test.duration)
+			if dur != test.expectedDuration {
+				t.Fatalf("DurationOffsetStrings: exp (%s); act (%s)", test.expectedDuration, dur)
+			}
+		})
+	}
+}
+
+func Test_DurationOffsetStrings(t *testing.T) {
+	testCases := map[string]struct {
+		duration         time.Duration
+		offset           time.Duration
+		expectedDuration string
+		expectedOffset   string
+	}{
+		"1a": {
+			duration:         0,
+			offset:           0,
+			expectedDuration: "",
+			expectedOffset:   "",
+		},
+		"1b": {
+			duration:         24 * time.Hour,
+			offset:           0,
+			expectedDuration: "1d",
+			expectedOffset:   "",
+		},
+		"1c": {
+			duration:         24*time.Hour + 5*time.Minute,
+			offset:           0,
+			expectedDuration: "1445m",
+			expectedOffset:   "",
+		},
+		"1d": {
+			duration:         25 * time.Hour,
+			offset:           5 * time.Minute,
+			expectedDuration: "25h",
+			expectedOffset:   "5m",
+		},
+		"1e": {
+			duration:         25 * time.Hour,
+			offset:           60 * time.Minute,
+			expectedDuration: "25h",
+			expectedOffset:   "1h",
+		},
+		"1f": {
+			duration:         72 * time.Hour,
+			offset:           1440 * time.Minute,
+			expectedDuration: "3d",
+			expectedOffset:   "1d",
+		},
+		"1g": {
+			duration:         25 * time.Hour,
+			offset:           1 * time.Second,
+			expectedDuration: "25h",
+			expectedOffset:   "1s",
+		},
+		"1h": {
+			duration:         24*time.Hour + time.Second,
+			offset:           1 * time.Second,
+			expectedDuration: "86401s",
+			expectedOffset:   "1s",
+		},
+		// Expect empty strings if durations are negative
+		"1i": {
+			duration:         -25 * time.Hour,
+			offset:           -1 * time.Second,
+			expectedDuration: "",
+			expectedOffset:   "",
+		},
+	}
+
+	for name, test := range testCases {
+		t.Run(name, func(t *testing.T) {
+			dur, off := DurationOffsetStrings(test.duration, test.offset)
+			if dur != test.expectedDuration || off != test.expectedOffset {
+				t.Fatalf("DurationOffsetStrings: exp (%s %s); act (%s, %s)", test.expectedDuration, test.expectedOffset, dur, off)
+			}
+		})
+	}
+}
+
+func Test_ParseDuration(t *testing.T) {
+	testCases := map[string]struct {
+		input    string
+		expected time.Duration
+	}{
+		"expected": {
+			input:    "3h",
+			expected: time.Hour * 3,
+		},
+		"white space": {
+			input:    " 4s ",
+			expected: time.Second * 4,
+		},
+		"prom prefix": {
+			input:    "offset 3m",
+			expected: time.Minute * 3,
+		},
+		"prom prefix white space": {
+			input:    " offset 3d ",
+			expected: 24.0 * time.Hour * 3,
+		},
+		"zero": {
+			input:    "0h",
+			expected: time.Duration(0),
+		},
+		"empty": {
+			input:    "",
+			expected: time.Duration(0),
+		},
+		"bad string": {
+			input:    "oqwd3dk5hk",
+			expected: time.Duration(0),
+		},
+		"digit": {
+			input:    "3",
+			expected: time.Duration(0),
+		},
+		"unit": {
+			input:    "h",
+			expected: time.Duration(0),
+		},
+	}
+	for name, test := range testCases {
+		t.Run(name, func(t *testing.T) {
+			dur, _ := ParseDuration(test.input)
+			if dur != test.expected {
+				t.Errorf("Expected duration %v did not match result %v", test.expected, dur)
+			}
+		})
+	}
+}
+
+func Test_CleanDurationString(t *testing.T) {
+	testCases := map[string]struct {
+		input    string
+		expected string
+	}{
+		"white space": {
+			input:    " 1d ",
+			expected: "1d",
+		},
+		"no change": {
+			input:    "1d",
+			expected: "1d",
+		},
+		"prefix": {
+			input:    "offset 1d",
+			expected: "1d",
+		},
+		"prefix white space": {
+			input:    " offset 1d ",
+			expected: "1d",
+		},
+		"empty": {
+			input:    "",
+			expected: "",
+		},
+		"random": {
+			input:    "oqwd3dk5hk",
+			expected: "oqwd3dk5hk",
+		},
+	}
+	for name, test := range testCases {
+		t.Run(name, func(t *testing.T) {
+			res := CleanDurationString(test.input)
+			if res != test.expected {
+				t.Errorf("Expected output %s did not match result %s", test.expected, res)
+			}
+		})
+	}
+}
+
+func Test_FormatDurationStringDaysToHours(t *testing.T) {
+	testCases := map[string]struct {
+		input    string
+		expected string
+	}{
+		"1 day": {
+			input:    "1d",
+			expected: "24h",
+		},
+		"2 days": {
+			input:    "1d",
+			expected: "24h",
+		},
+		"500 days": {
+			input:    "500d",
+			expected: "12000h",
+		},
+		"1h": {
+			input:    "1h",
+			expected: "1h",
+		},
+		"empty": {
+			input:    "",
+			expected: "",
+		},
+		"no unit": {
+			input:    "1",
+			expected: "1",
+		},
+		"random": {
+			input:    "oqwd3dk5hk",
+			expected: "oqwd3dk5hk",
+		},
+	}
+	for name, test := range testCases {
+		t.Run(name, func(t *testing.T) {
+			res, _ := FormatDurationStringDaysToHours(test.input)
+			if res != test.expected {
+				t.Errorf("Expected output %s did not match result %s", test.expected, res)
+			}
+		})
+	}
+}

pkg/util/watcher/configwatcher_test.go

@@ -0,0 +1,142 @@
+package watcher
+
+import (
+	"testing"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	TestConfigMapName          = "test-config"
+	AlternateTestConfigMapName = "alternate-test-config"
+	TestDataProperty           = "test-prop"
+)
+
+func newTestWatcher(t *testing.T, configMapName string, instanceName string, didRun *bool) *ConfigMapWatcher {
+	return &ConfigMapWatcher{
+		ConfigMapName: configMapName,
+		WatchFunc: func(cmn string, data map[string]string) error {
+			t.Logf("ConfigMapWatcher[%s] triggered for ConfigMap: %s, data[\"test\"] = %s\n", instanceName, cmn, data[TestDataProperty])
+			*didRun = true
+			return nil
+		},
+	}
+}
+
+func newConfigMap(configMapName string, dataValue string) *v1.ConfigMap {
+	return &v1.ConfigMap{
+		Data: map[string]string{
+			TestDataProperty: dataValue,
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: configMapName,
+		},
+	}
+}
+
+func TestConfigWatcherSingleHandler(t *testing.T) {
+	// Test that a single watcher added for the configmap test-config is executed when
+	// triggered
+	var didRun bool = false
+
+	w := NewConfigMapWatchers(newTestWatcher(t, TestConfigMapName, "single", &didRun))
+	f := w.ToWatchFunc()
+
+	// Execute watch func with a 'test-config' configmap
+	f(newConfigMap(TestConfigMapName, "testing 1 2 3"))
+
+	if !didRun {
+		t.Errorf("Failed to run configmap handler for 'single'\n")
+	}
+}
+
+func TestConfigWatcherMultipleHandlers(t *testing.T) {
+	// Test that adding two different configmap watchers aren't both triggered on a configmap update
+	var firstDidRun bool = false
+	var secondDidRun bool = false
+
+	w := NewConfigMapWatchers(
+		newTestWatcher(t, TestConfigMapName, "single", &firstDidRun),
+		newTestWatcher(t, AlternateTestConfigMapName, "alternate", &secondDidRun))
+
+	f := w.ToWatchFunc()
+
+	// Execute watch func with a 'alternate-test-config' configmap
+	f(newConfigMap(AlternateTestConfigMapName, "oof!"))
+
+	// Assert that first did not run
+	if firstDidRun {
+		t.Errorf("Executed alternate-test-config map change, but test-config handler, 'single' executed!\n")
+	}
+
+	if !secondDidRun {
+		t.Errorf("Failed to run configmap handler for 'alternate'\n")
+	}
+}
+
+func TestConfigWatcherMultipleHandlersForSameConfig(t *testing.T) {
+	// Test that adding two different configmap watchers for the same configmap are both triggered
+	var firstDidRun bool = false
+	var secondDidRun bool = false
+	var thirdDidRun bool = false
+
+	w := NewConfigMapWatchers(
+		newTestWatcher(t, TestConfigMapName, "first", &firstDidRun),
+		newTestWatcher(t, AlternateTestConfigMapName, "alternate", &secondDidRun),
+		// third watcher watches for the same configmap as "first"
+		newTestWatcher(t, TestConfigMapName, "third", &thirdDidRun),
+	)
+
+	f := w.ToWatchFunc()
+
+	// Execute watch func with a 'test-config' configmap
+	f(newConfigMap(TestConfigMapName, "double trouble"))
+
+	// Assert that second did not run
+	if secondDidRun {
+		t.Errorf("Executed test-config map change, first handler, 'single', executed!\n")
+	}
+
+	if !firstDidRun {
+		t.Errorf("Failed to run configmap handler for 'first'\n")
+	}
+	if !thirdDidRun {
+		t.Errorf("Failed to run configmap handler for 'third'\n")
+	}
+}
+
+func TestConfigMapWatcherWithAdd(t *testing.T) {
+	// Test that adding two different configmap watchers for the same configmap are both triggered
+	// when using Add() and AddWatcher()
+	var firstDidRun bool = false
+	var secondDidRun bool = false
+	var thirdDidRun bool = false
+
+	a, b, c := newTestWatcher(t, TestConfigMapName, "first", &firstDidRun),
+		newTestWatcher(t, AlternateTestConfigMapName, "alternate", &secondDidRun),
+		// third watcher watches for the same configmap as "first"
+		newTestWatcher(t, TestConfigMapName, "third", &thirdDidRun)
+
+	w := NewConfigMapWatchers()
+	w.AddWatcher(a)
+	w.AddWatcher(b)
+	w.Add(c.ConfigMapName, c.WatchFunc)
+
+	f := w.ToWatchFunc()
+
+	// Execute watch func with a 'test-config' configmap
+	f(newConfigMap(TestConfigMapName, "double trouble"))
+
+	// Assert that second did not run
+	if secondDidRun {
+		t.Errorf("Executed test-config map change, first handler, 'single', executed!\n")
+	}
+
+	if !firstDidRun {
+		t.Errorf("Failed to run configmap handler for 'first'\n")
+	}
+	if !thirdDidRun {
+		t.Errorf("Failed to run configmap handler for 'third'\n")
+	}
+}

pkg/util/watcher/configwatchers.go

@@ -0,0 +1,74 @@
+package watcher
+
+import (
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/klog"
+)
+
+// ConfigMapWatcher represents a single configmap watcher
+type ConfigMapWatcher struct {
+	ConfigMapName string
+	WatchFunc     func(string, map[string]string) error
+}
+
+type ConfigMapWatchers struct {
+	watchers map[string][]*ConfigMapWatcher
+}
+
+func NewConfigMapWatchers(watchers ...*ConfigMapWatcher) *ConfigMapWatchers {
+	cmw := &ConfigMapWatchers{
+		watchers: make(map[string][]*ConfigMapWatcher),
+	}
+
+	for _, w := range watchers {
+		cmw.AddWatcher(w)
+	}
+
+	return cmw
+}
+
+func (cmw *ConfigMapWatchers) AddWatcher(watcher *ConfigMapWatcher) {
+	if watcher == nil {
+		return
+	}
+
+	name := watcher.ConfigMapName
+	cmw.watchers[name] = append(cmw.watchers[name], watcher)
+}
+
+func (cmw *ConfigMapWatchers) Add(configMapName string, watchFunc func(string, map[string]string) error) {
+	cmw.AddWatcher(&ConfigMapWatcher{
+		ConfigMapName: configMapName,
+		WatchFunc:     watchFunc,
+	})
+}
+
+func (cmw *ConfigMapWatchers) GetWatchedConfigs() []string {
+	configNames := []string{}
+
+	for k := range cmw.watchers {
+		configNames = append(configNames, k)
+	}
+
+	return configNames
+}
+
+func (cmw *ConfigMapWatchers) ToWatchFunc() func(interface{}) {
+	return func(c interface{}) {
+		conf, ok := c.(*v1.ConfigMap)
+		if !ok {
+			return
+		}
+
+		name := conf.GetName()
+		data := conf.Data
+		if watchers, ok := cmw.watchers[name]; ok {
+			for _, cw := range watchers {
+				err := cw.WatchFunc(name, data)
+				if err != nil {
+					klog.Infof("ERROR UPDATING %s CONFIG: %s", name, err.Error())
+				}
+			}
+		}
+	}
+}

test/cloud_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/kubecost/cost-model/pkg/costmodel"
 	"github.com/kubecost/cost-model/pkg/costmodel/clusters"
 
+	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 )
 
@@ -248,6 +249,10 @@ func (f FakeCache) GetAllNodes() []*v1.Node {
 	return f.nodes
 }
 
+func (f FakeCache) GetAllDaemonSets() []*appsv1.DaemonSet {
+	return nil
+}
+
 func NewFakeNodeCache(nodes []*v1.Node) FakeCache {
 	return FakeCache{
 		nodes: nodes,