Merge branch 'develop' of https://github.com/kubecost/cost-model into AjayTripathy-better-cadvisor-compatibility

Dockerfile.metrics

@@ -0,0 +1,23 @@
+FROM golang:latest as build-env
+
+RUN mkdir /app
+WORKDIR /app
+COPY go.mod .
+COPY go.sum .
+
+# Get dependencies - will also be cached if we won't change mod/sum
+RUN go mod download
+# COPY the source code as the last step
+COPY . .
+# Build the binary
+RUN set -e ;\
+    cd cmd/kubemetrics;\
+    CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
+    go build -a -installsuffix cgo -o /go/bin/app
+
+FROM alpine:latest
+RUN apk add --update --no-cache ca-certificates
+COPY --from=build-env /go/bin/app /go/bin/app
+
+USER 1001
+ENTRYPOINT ["/go/bin/app"]

cmd/kubemetrics/main.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/env"
+	"github.com/kubecost/cost-model/pkg/metrics"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/rs/cors"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog"
+)
+
+func Healthz(w http.ResponseWriter, _ *http.Request) {
+	w.WriteHeader(200)
+	w.Header().Set("Content-Length", "0")
+	w.Header().Set("Content-Type", "text/plain")
+}
+
+// initializes the kubernetes client cache
+func newKubernetesClusterCache() (clustercache.ClusterCache, error) {
+	var err error
+
+	// Kubernetes API setup
+	var kc *rest.Config
+	if kubeconfig := env.GetKubeConfigPath(); kubeconfig != "" {
+		kc, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
+	} else {
+		kc, err = rest.InClusterConfig()
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	kubeClientset, err := kubernetes.NewForConfig(kc)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create Kubernetes Cluster Cache + Watchers
+	k8sCache := clustercache.NewKubernetesClusterCache(kubeClientset)
+	k8sCache.Run()
+
+	return k8sCache, nil
+}
+
+func main() {
+	klog.InitFlags(nil)
+	flag.Set("v", "3")
+	flag.Parse()
+
+	klog.V(1).Infof("Starting kubecost-metrics...")
+
+	// initialize kubernetes client and cluster cache
+	clusterCache, err := newKubernetesClusterCache()
+	if err != nil {
+		panic(err.Error())
+	}
+
+	// initialize Kubernetes Metrics
+	metrics.InitKubeMetrics(clusterCache, &metrics.KubeMetricsOpts{
+		EmitKubecostControllerMetrics: true,
+		EmitNamespaceAnnotations:      env.IsEmitNamespaceAnnotationsMetric(),
+		EmitPodAnnotations:            env.IsEmitPodAnnotationsMetric(),
+		EmitKubeStateMetrics:          true,
+	})
+
+	rootMux := http.NewServeMux()
+	rootMux.HandleFunc("/healthz", Healthz)
+	rootMux.Handle("/metrics", promhttp.Handler())
+	handler := cors.AllowAll().Handler(rootMux)
+
+	klog.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", env.GetKubecostMetricsPort()), handler))
+}

configs/azure.json

@@ -9,6 +9,8 @@
     "zoneNetworkEgress": "0.01",
     "regionNetworkEgress": "0.01",
     "internetNetworkEgress": "0.0725",
+    "spotLabel": "kubernetes.azure.com/scalesetpriority",
+    "spotLabelValue": "spot",
     "azureSubscriptionID": "",
     "azureClientID": "" ,
     "azureClientSecret": "" ,

pkg/cloud/azureprovider.go

@@ -4,8 +4,11 @@ import (
 	"context"
 	"encoding/csv"
 	"fmt"
+	"github.com/kubecost/cost-model/pkg/log"
 	"io"
 	"io/ioutil"
+	"net/http"
+	"net/url"
 	"regexp"
 	"strconv"
 	"strings"
@@ -36,6 +39,8 @@ const (
 	AzureDiskPremiumSSDStorageClass  = "premium_ssd"
 	AzureDiskStandardSSDStorageClass = "standard_ssd"
 	AzureDiskStandardStorageClass    = "standard_hdd"
+	defaultSpotLabel                 = "kubernetes.azure.com/scalesetpriority"
+	defaultSpotLabelValue            = "spot"
 )
 
 var (
@@ -149,6 +154,72 @@ func getRegions(service string, subscriptionsClient subscriptions.Client, provid
 	}
 }
 
+func getRetailPrice(region string, skuName string, currencyCode string, spot bool) (string, error) {
+	pricingURL := "https://prices.azure.com/api/retail/prices?$skip=0"
+
+	if currencyCode != "" {
+		pricingURL += fmt.Sprintf("&currencyCode='%s'", currencyCode)
+	}
+
+	var filterParams []string
+
+	if region != "" {
+		regionParam := fmt.Sprintf("armRegionName eq '%s'",region)
+		filterParams = append(filterParams, regionParam)
+	}
+
+	if skuName != "" {
+		skuNameParam := fmt.Sprintf("armSkuName eq '%s'",skuName)
+		filterParams = append(filterParams, skuNameParam)
+	}
+
+	if len(filterParams) > 0 {
+		filterParamsEscaped := url.QueryEscape(strings.Join(filterParams[:], " and "))
+		pricingURL += fmt.Sprintf("&$filter=%s", filterParamsEscaped)
+	}
+
+	log.Infof("starting download retail price payload from \"%s\"", pricingURL)
+	resp, err := http.Get(pricingURL)
+
+	if err != nil {
+		return "", fmt.Errorf("bogus fetch of \"%s\": %v", pricingURL, err)
+	}
+
+	if resp.StatusCode < 200 && resp.StatusCode > 299 {
+		return "", fmt.Errorf("retail price responded with error status code %d", resp.StatusCode)
+	}
+
+	pricingPayload := AzureRetailPricing{}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("Error getting response: %v", err)
+	}
+
+	jsonErr := json.Unmarshal(body, &pricingPayload)
+	if jsonErr != nil {
+		return "", fmt.Errorf("Error unmarshalling data: %v", jsonErr)
+	}
+
+	retailPrice := ""
+	for _, item := range pricingPayload.Items {
+		if item.Type == "Consumption" && !strings.Contains(item.ProductName, "Windows") {
+			// if spot is true SkuName should contain "spot, if it is false it should not
+			if spot == strings.Contains(strings.ToLower(item.SkuName), " spot") {
+				retailPrice = fmt.Sprintf("%f", item.RetailPrice)
+			}
+		}
+	}
+
+	log.DedupedInfof(5, "done parsing retail price payload from \"%s\"\n", pricingURL)
+
+	if retailPrice == "" {
+		return retailPrice, fmt.Errorf("Couldn't find price for product \"%s\" in \"%s\" region", skuName, region)
+	}
+
+	return retailPrice, nil
+}
+
 func toRegionID(meterRegion string, regions map[string]string) (string, error) {
 	var rp regionParts = strings.Split(strings.ToLower(meterRegion), " ")
 	regionCode := regionCodeMappings[rp[0]]
@@ -182,6 +253,41 @@ func checkRegionID(regionID string, regions map[string]string) bool {
 	return false
 }
 
+// AzureRetailPricing struct for unmarshalling Azure Retail pricing api JSON response
+type AzureRetailPricing struct {
+	BillingCurrency    string                         `json:"BillingCurrency"`
+	CustomerEntityId   string                         `json:"CustomerEntityId"`
+	CustomerEntityType string                         `json:"CustomerEntityType"`
+	Items              []AzureRetailPricingAttributes `json:"Items"`
+	NextPageLink       string                         `json:"NextPageLink"`
+	Count              int                            `json:"Count"`
+}
+
+//AzureRetailPricingAttributes struct for unmarshalling Azure Retail pricing api JSON response
+type AzureRetailPricingAttributes struct {
+	CurrencyCode         string     `json:"currencyCode"`
+	TierMinimumUnits     float32    `json:"tierMinimumUnits"`
+	RetailPrice          float32    `json:"retailPrice"`
+	UnitPrice            float32    `json:"unitPrice"`
+	ArmRegionName        string     `json:"armRegionName"`
+	Location             string     `json:"location"`
+	EffectiveStartDate   *time.Time `json:"effectiveStartDate"`
+	EffectiveEndDate     *time.Time `json:"effectiveEndDate"`
+	MeterId              string     `json:"meterId"`
+	MeterName            string     `json:"meterName"`
+	ProductId            string     `json:"productId"`
+	SkuId                string     `json:"skuId"`
+	ProductName          string     `json:"productName"`
+	SkuName              string     `json:"skuName"`
+	ServiceName          string     `json:"serviceName"`
+	ServiceId            string     `json:"serviceId"`
+	ServiceFamily        string     `json:"serviceFamily"`
+	UnitOfMeasure        string     `json:"unitOfMeasure"`
+	Type                 string     `json:"type"`
+	IsPrimaryMeterRegion bool       `json:"isPrimaryMeterRegion"`
+	ArmSkuName           string     `json:"armSkuName"`
+}
+
 // AzurePricing either contains a Node or PV
 type AzurePricing struct {
 	Node *Node
@@ -703,6 +809,7 @@ func (az *Azure) DownloadPricingData() error {
 						Node: &Node{
 							Cost:         priceStr,
 							BaseCPUPrice: baseCPUPrice,
+							UsageType:    usageType,
 						},
 					}
 				}
@@ -729,6 +836,13 @@ func (az *Azure) DownloadPricingData() error {
 	return nil
 }
 
+func (az *Azure) addPricing(features string, azurePricing *AzurePricing) {
+	if az.Pricing == nil {
+		az.Pricing = map[string]*AzurePricing{}
+	}
+	az.Pricing[features] = azurePricing
+}
+
 // AllNodePricing returns the Azure pricing objects stored
 func (az *Azure) AllNodePricing() (interface{}, error) {
 	az.DownloadPricingDataLock.RLock()
@@ -740,10 +854,49 @@ func (az *Azure) AllNodePricing() (interface{}, error) {
 func (az *Azure) NodePricing(key Key) (*Node, error) {
 	az.DownloadPricingDataLock.RLock()
 	defer az.DownloadPricingDataLock.RUnlock()
+
 	azKey, ok := key.(*azureKey)
 	if !ok {
 		return nil, fmt.Errorf("azure: NodePricing: key is of type %T", key)
 	}
+	config, _ := az.GetConfig()
+	if slv, ok := azKey.Labels[config.SpotLabel];
+	ok && slv == config.SpotLabelValue && config.SpotLabel != "" && config.SpotLabelValue != "" {
+		features := strings.Split(azKey.Features(), ",")
+		region := features[0]
+		instance := features[1]
+		spotFeatures := fmt.Sprintf("%s,%s,%s", region, instance, "spot")
+		if n, ok := az.Pricing[spotFeatures]; ok {
+			log.DedupedInfof(5, "Returning pricing for node %s: %+v from key %s", azKey, n, spotFeatures)
+			if azKey.isValidGPUNode() {
+				n.Node.GPU = "1" // TODO: support multiple GPUs
+			}
+			return n.Node, nil
+		}
+		log.Infof("[Info] found spot instance, trying to get retail price for %s: %s, ", spotFeatures, azKey)
+
+		spotCost, err := getRetailPrice(region, instance, config.CurrencyCode, true)
+		if err != nil {
+			log.DedupedWarningf(5, "failed to retrieve spot retail pricing")
+		} else {
+			gpu := ""
+			if azKey.isValidGPUNode() {
+				gpu = "1"
+			}
+			spotNode := &Node{
+				Cost:      spotCost,
+				UsageType: "spot",
+				GPU:       gpu,
+			}
+
+			az.addPricing(spotFeatures, &AzurePricing{
+				Node: spotNode,
+			})
+
+			return spotNode, nil
+		}
+	}
+
 
 	if n, ok := az.Pricing[azKey.Features()]; ok {
 		klog.V(4).Infof("Returning pricing for node %s: %+v from key %s", azKey, n, azKey.Features())
@@ -961,6 +1114,12 @@ func (az *Azure) GetConfig() (*CustomPricing, error) {
 	if c.ShareTenancyCosts == "" {
 		c.ShareTenancyCosts = defaultShareTenancyCost
 	}
+	if c.SpotLabel == "" {
+		c.SpotLabel = defaultSpotLabel
+	}
+	if c.SpotLabelValue == "" {
+		c.SpotLabelValue = defaultSpotLabelValue
+	}
 	return c, nil
 }
 

pkg/clustercache/clustercache.go

@@ -7,6 +7,8 @@ import (
 	"k8s.io/klog"
 
 	appsv1 "k8s.io/api/apps/v1"
+	autoscaling "k8s.io/api/autoscaling/v2beta1"
+	batchv1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
 	stv1 "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/fields"
@@ -53,9 +55,18 @@ type ClusterCache interface {
 	// GetAllPersistentVolumes returns all the cached persistent volumes
 	GetAllPersistentVolumes() []*v1.PersistentVolume
 
+	// GetAllPersistentVolumeClaims returns all the cached persistent volume claims
+	GetAllPersistentVolumeClaims() []*v1.PersistentVolumeClaim
+
 	// GetAllStorageClasses returns all the cached storage classes
 	GetAllStorageClasses() []*stv1.StorageClass
 
+	// GetAllJobs returns all the cached jobs
+	GetAllJobs() []*batchv1.Job
+
+	// GetAllHorizontalPodAutoscalers() returns all cached horizontal pod autoscalers
+	GetAllHorizontalPodAutoscalers() []*autoscaling.HorizontalPodAutoscaler
+
 	// SetConfigMapUpdateFunc sets the configmap update function
 	SetConfigMapUpdateFunc(func(interface{}))
 }
@@ -74,7 +85,10 @@ type KubernetesClusterCache struct {
 	statefulsetWatch       WatchController
 	replicasetWatch        WatchController
 	pvWatch                WatchController
+	pvcWatch               WatchController
 	storageClassWatch      WatchController
+	jobsWatch              WatchController
+	hpaWatch               WatchController
 	stop                   chan struct{}
 }
 
@@ -87,6 +101,8 @@ func NewKubernetesClusterCache(client kubernetes.Interface) ClusterCache {
 	coreRestClient := client.CoreV1().RESTClient()
 	appsRestClient := client.AppsV1().RESTClient()
 	storageRestClient := client.StorageV1().RESTClient()
+	batchClient := client.BatchV1().RESTClient()
+	autoscalingClient := client.AutoscalingV2beta1().RESTClient()
 
 	kubecostNamespace := env.GetKubecostNamespace()
 	klog.Infof("NAMESPACE: %s", kubecostNamespace)
@@ -103,12 +119,15 @@ func NewKubernetesClusterCache(client kubernetes.Interface) ClusterCache {
 		statefulsetWatch:       NewCachingWatcher(appsRestClient, "statefulsets", &appsv1.StatefulSet{}, "", fields.Everything()),
 		replicasetWatch:        NewCachingWatcher(appsRestClient, "replicasets", &appsv1.ReplicaSet{}, "", fields.Everything()),
 		pvWatch:                NewCachingWatcher(coreRestClient, "persistentvolumes", &v1.PersistentVolume{}, "", fields.Everything()),
+		pvcWatch:               NewCachingWatcher(coreRestClient, "persistentvolumeclaims", &v1.PersistentVolumeClaim{}, "", fields.Everything()),
 		storageClassWatch:      NewCachingWatcher(storageRestClient, "storageclasses", &stv1.StorageClass{}, "", fields.Everything()),
+		jobsWatch:              NewCachingWatcher(batchClient, "jobs", &batchv1.Job{}, "", fields.Everything()),
+		hpaWatch:               NewCachingWatcher(autoscalingClient, "horizontalpodautoscalers", &autoscaling.HorizontalPodAutoscaler{}, "", fields.Everything()),
 	}
 
 	// Wait for each caching watcher to initialize
 	var wg sync.WaitGroup
-	wg.Add(11)
+	wg.Add(14)
 
 	cancel := make(chan struct{})
 
@@ -122,7 +141,10 @@ func NewKubernetesClusterCache(client kubernetes.Interface) ClusterCache {
 	go initializeCache(kcc.statefulsetWatch, &wg, cancel)
 	go initializeCache(kcc.replicasetWatch, &wg, cancel)
 	go initializeCache(kcc.pvWatch, &wg, cancel)
+	go initializeCache(kcc.pvcWatch, &wg, cancel)
 	go initializeCache(kcc.storageClassWatch, &wg, cancel)
+	go initializeCache(kcc.jobsWatch, &wg, cancel)
+	go initializeCache(kcc.hpaWatch, &wg, cancel)
 
 	wg.Wait()
 
@@ -145,7 +167,10 @@ func (kcc *KubernetesClusterCache) Run() {
 	go kcc.statefulsetWatch.Run(1, stopCh)
 	go kcc.replicasetWatch.Run(1, stopCh)
 	go kcc.pvWatch.Run(1, stopCh)
+	go kcc.pvcWatch.Run(1, stopCh)
 	go kcc.storageClassWatch.Run(1, stopCh)
+	go kcc.jobsWatch.Run(1, stopCh)
+	go kcc.hpaWatch.Run(1, stopCh)
 
 	kcc.stop = stopCh
 }
@@ -244,6 +269,15 @@ func (kcc *KubernetesClusterCache) GetAllPersistentVolumes() []*v1.PersistentVol
 	return pvs
 }
 
+func (kcc *KubernetesClusterCache) GetAllPersistentVolumeClaims() []*v1.PersistentVolumeClaim {
+	var pvcs []*v1.PersistentVolumeClaim
+	items := kcc.pvcWatch.GetAll()
+	for _, pvc := range items {
+		pvcs = append(pvcs, pvc.(*v1.PersistentVolumeClaim))
+	}
+	return pvcs
+}
+
 func (kcc *KubernetesClusterCache) GetAllStorageClasses() []*stv1.StorageClass {
 	var storageClasses []*stv1.StorageClass
 	items := kcc.storageClassWatch.GetAll()
@@ -253,6 +287,24 @@ func (kcc *KubernetesClusterCache) GetAllStorageClasses() []*stv1.StorageClass {
 	return storageClasses
 }
 
+func (kcc *KubernetesClusterCache) GetAllJobs() []*batchv1.Job {
+	var jobs []*batchv1.Job
+	items := kcc.jobsWatch.GetAll()
+	for _, job := range items {
+		jobs = append(jobs, job.(*batchv1.Job))
+	}
+	return jobs
+}
+
+func (kcc *KubernetesClusterCache) GetAllHorizontalPodAutoscalers() []*autoscaling.HorizontalPodAutoscaler {
+	var hpas []*autoscaling.HorizontalPodAutoscaler
+	items := kcc.hpaWatch.GetAll()
+	for _, hpa := range items {
+		hpas = append(hpas, hpa.(*autoscaling.HorizontalPodAutoscaler))
+	}
+	return hpas
+}
+
 func (kcc *KubernetesClusterCache) SetConfigMapUpdateFunc(f func(interface{})) {
 	kcc.kubecostConfigMapWatch.SetUpdateHandler(f)
 }

pkg/costmodel/allocation.go

@@ -37,6 +37,7 @@ const (
 	// https://prometheus.io/blog/2019/01/28/subquery-support/#examples
 	queryFmtCPUUsageMax           = `max(max_over_time(kubecost_savings_container_cpu_usage_seconds[%s]%s)) by (container_name, pod_name, namespace, instance, %s)`
 	queryFmtGPUsRequested         = `avg(avg_over_time(kube_pod_container_resource_requests{resource="nvidia_com_gpu", container!="",container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
+	queryFmtGPUsAllocated         = `avg(avg_over_time(container_gpu_allocation{container!="", container!="POD", node!=""}[%s]%s)) by (container, pod, namespace, node, %s)`
 	queryFmtNodeCostPerCPUHr      = `avg(avg_over_time(node_cpu_hourly_cost[%s]%s)) by (node, %s, instance_type, provider_id)`
 	queryFmtNodeCostPerRAMGiBHr   = `avg(avg_over_time(node_ram_hourly_cost[%s]%s)) by (node, %s, instance_type, provider_id)`
 	queryFmtNodeCostPerGPUHr      = `avg(avg_over_time(node_gpu_hourly_cost[%s]%s)) by (node, %s, instance_type, provider_id)`
@@ -156,6 +157,9 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	queryGPUsRequested := fmt.Sprintf(queryFmtGPUsRequested, durStr, offStr, env.GetPromClusterLabel())
 	resChGPUsRequested := ctx.Query(queryGPUsRequested)
 
+	queryGPUsAllocated := fmt.Sprintf(queryFmtGPUsAllocated, durStr, offStr, env.GetPromClusterLabel())
+	resChGPUsAllocated := ctx.Query(queryGPUsAllocated)
+
 	queryNodeCostPerCPUHr := fmt.Sprintf(queryFmtNodeCostPerCPUHr, durStr, offStr, env.GetPromClusterLabel())
 	resChNodeCostPerCPUHr := ctx.Query(queryNodeCostPerCPUHr)
 
@@ -249,6 +253,7 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	resRAMUsageAvg, _ := resChRAMUsageAvg.Await()
 	resRAMUsageMax, _ := resChRAMUsageMax.Await()
 	resGPUsRequested, _ := resChGPUsRequested.Await()
+	resGPUsAllocated, _ := resChGPUsAllocated.Await()
 
 	resNodeCostPerCPUHr, _ := resChNodeCostPerCPUHr.Await()
 	resNodeCostPerRAMGiBHr, _ := resChNodeCostPerRAMGiBHr.Await()
@@ -302,7 +307,7 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	applyRAMBytesRequested(podMap, resRAMRequests)
 	applyRAMBytesUsedAvg(podMap, resRAMUsageAvg)
 	applyRAMBytesUsedMax(podMap, resRAMUsageMax)
-	applyGPUsRequested(podMap, resGPUsRequested)
+	applyGPUsAllocated(podMap, resGPUsRequested, resGPUsAllocated)
 	applyNetworkTotals(podMap, resNetTransferBytes, resNetReceiveBytes)
 	applyNetworkAllocation(podMap, resNetZoneGiB, resNetZoneCostPerGiB)
 	applyNetworkAllocation(podMap, resNetRegionGiB, resNetRegionCostPerGiB)
@@ -895,7 +900,10 @@ func applyRAMBytesUsedMax(podMap map[podKey]*Pod, resRAMBytesUsedMax []*prom.Que
 	}
 }
 
-func applyGPUsRequested(podMap map[podKey]*Pod, resGPUsRequested []*prom.QueryResult) {
+func applyGPUsAllocated(podMap map[podKey]*Pod, resGPUsRequested []*prom.QueryResult, resGPUsAllocated []*prom.QueryResult) {
+	if len(resGPUsAllocated) > 0 { // Use the new query, when it's become available in a window
+		resGPUsRequested = resGPUsAllocated
+	}
 	for _, res := range resGPUsRequested {
 		key, err := resultPodKey(res, env.GetPromClusterLabel(), "namespace")
 		if err != nil {

pkg/costmodel/metrics.go

@@ -12,6 +12,7 @@ import (
 	"github.com/kubecost/cost-model/pkg/env"
 	"github.com/kubecost/cost-model/pkg/errors"
 	"github.com/kubecost/cost-model/pkg/log"
+	"github.com/kubecost/cost-model/pkg/metrics"
 	"github.com/kubecost/cost-model/pkg/prom"
 	"github.com/kubecost/cost-model/pkg/util"
 
@@ -19,446 +20,12 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	dto "github.com/prometheus/client_model/go"
 	v1 "k8s.io/api/core/v1"
+
 	"k8s.io/client-go/kubernetes"
 
 	"k8s.io/klog"
 )
 
-//--------------------------------------------------------------------------
-//  StatefulsetCollector
-//--------------------------------------------------------------------------
-
-// StatefulsetCollector is a prometheus collector that generates StatefulsetMetrics
-type StatefulsetCollector struct {
-	KubeClusterCache clustercache.ClusterCache
-}
-
-// Describe sends the super-set of all possible descriptors of metrics
-// collected by this Collector.
-func (sc StatefulsetCollector) Describe(ch chan<- *prometheus.Desc) {
-	ch <- prometheus.NewDesc("statefulSet_match_labels", "statfulSet match labels", []string{}, nil)
-}
-
-// Collect is called by the Prometheus registry when collecting metrics.
-func (sc StatefulsetCollector) Collect(ch chan<- prometheus.Metric) {
-	ds := sc.KubeClusterCache.GetAllStatefulSets()
-	for _, statefulset := range ds {
-		labels, values := prom.KubeLabelsToLabels(statefulset.Spec.Selector.MatchLabels)
-		if len(labels) > 0 {
-			m := newStatefulsetMetric(statefulset.GetName(), statefulset.GetNamespace(), "statefulSet_match_labels", labels, values)
-			ch <- m
-		}
-	}
-}
-
-//--------------------------------------------------------------------------
-//  StatefulsetMetric
-//--------------------------------------------------------------------------
-
-// StatefulsetMetric is a prometheus.Metric used to encode statefulset match labels
-type StatefulsetMetric struct {
-	fqName          string
-	help            string
-	labelNames      []string
-	labelValues     []string
-	statefulsetName string
-	namespace       string
-}
-
-// Creates a new StatefulsetMetric, implementation of prometheus.Metric
-func newStatefulsetMetric(name, namespace, fqname string, labelNames []string, labelvalues []string) StatefulsetMetric {
-	return StatefulsetMetric{
-		fqName:          fqname,
-		labelNames:      labelNames,
-		labelValues:     labelvalues,
-		help:            "statefulSet_match_labels StatefulSet Match Labels",
-		statefulsetName: name,
-		namespace:       namespace,
-	}
-}
-
-// Desc returns the descriptor for the Metric. This method idempotently
-// returns the same descriptor throughout the lifetime of the Metric.
-func (s StatefulsetMetric) Desc() *prometheus.Desc {
-	l := prometheus.Labels{"statefulSet": s.statefulsetName, "namespace": s.namespace}
-	return prometheus.NewDesc(s.fqName, s.help, s.labelNames, l)
-}
-
-// Write encodes the Metric into a "Metric" Protocol Buffer data
-// transmission object.
-func (s StatefulsetMetric) Write(m *dto.Metric) error {
-	h := float64(1)
-	m.Gauge = &dto.Gauge{
-		Value: &h,
-	}
-	var labels []*dto.LabelPair
-	for i := range s.labelNames {
-		labels = append(labels, &dto.LabelPair{
-			Name:  &s.labelNames[i],
-			Value: &s.labelValues[i],
-		})
-	}
-	n := "namespace"
-	labels = append(labels, &dto.LabelPair{
-		Name:  &n,
-		Value: &s.namespace,
-	})
-	r := "statefulSet"
-	labels = append(labels, &dto.LabelPair{
-		Name:  &r,
-		Value: &s.statefulsetName,
-	})
-	m.Label = labels
-	return nil
-}
-
-//--------------------------------------------------------------------------
-//  DeploymentCollector
-//--------------------------------------------------------------------------
-
-// DeploymentCollector is a prometheus collector that generates DeploymentMetrics
-type DeploymentCollector struct {
-	KubeClusterCache clustercache.ClusterCache
-}
-
-// Describe sends the super-set of all possible descriptors of metrics
-// collected by this Collector.
-func (sc DeploymentCollector) Describe(ch chan<- *prometheus.Desc) {
-	ch <- prometheus.NewDesc("deployment_match_labels", "deployment match labels", []string{}, nil)
-}
-
-// Collect is called by the Prometheus registry when collecting metrics.
-func (sc DeploymentCollector) Collect(ch chan<- prometheus.Metric) {
-	ds := sc.KubeClusterCache.GetAllDeployments()
-	for _, deployment := range ds {
-		labels, values := prom.KubeLabelsToLabels(deployment.Spec.Selector.MatchLabels)
-		if len(labels) > 0 {
-			m := newDeploymentMetric(deployment.GetName(), deployment.GetNamespace(), "deployment_match_labels", labels, values)
-			ch <- m
-		}
-	}
-}
-
-//--------------------------------------------------------------------------
-//  DeploymentMetric
-//--------------------------------------------------------------------------
-
-// DeploymentMetric is a prometheus.Metric used to encode deployment match labels
-type DeploymentMetric struct {
-	fqName         string
-	help           string
-	labelNames     []string
-	labelValues    []string
-	deploymentName string
-	namespace      string
-}
-
-// Creates a new DeploymentMetric, implementation of prometheus.Metric
-func newDeploymentMetric(name, namespace, fqname string, labelNames []string, labelvalues []string) DeploymentMetric {
-	return DeploymentMetric{
-		fqName:         fqname,
-		labelNames:     labelNames,
-		labelValues:    labelvalues,
-		help:           "deployment_match_labels Deployment Match Labels",
-		deploymentName: name,
-		namespace:      namespace,
-	}
-}
-
-// Desc returns the descriptor for the Metric. This method idempotently
-// returns the same descriptor throughout the lifetime of the Metric.
-func (s DeploymentMetric) Desc() *prometheus.Desc {
-	l := prometheus.Labels{"deployment": s.deploymentName, "namespace": s.namespace}
-	return prometheus.NewDesc(s.fqName, s.help, s.labelNames, l)
-}
-
-// Write encodes the Metric into a "Metric" Protocol Buffer data
-// transmission object.
-func (s DeploymentMetric) Write(m *dto.Metric) error {
-	h := float64(1)
-	m.Gauge = &dto.Gauge{
-		Value: &h,
-	}
-	var labels []*dto.LabelPair
-	for i := range s.labelNames {
-		labels = append(labels, &dto.LabelPair{
-			Name:  &s.labelNames[i],
-			Value: &s.labelValues[i],
-		})
-	}
-	n := "namespace"
-	labels = append(labels, &dto.LabelPair{
-		Name:  &n,
-		Value: &s.namespace,
-	})
-	r := "deployment"
-	labels = append(labels, &dto.LabelPair{
-		Name:  &r,
-		Value: &s.deploymentName,
-	})
-	m.Label = labels
-	return nil
-}
-
-//--------------------------------------------------------------------------
-//  ServiceCollector
-//--------------------------------------------------------------------------
-
-// ServiceCollector is a prometheus collector that generates ServiceMetrics
-type ServiceCollector struct {
-	KubeClusterCache clustercache.ClusterCache
-}
-
-// Describe sends the super-set of all possible descriptors of metrics
-// collected by this Collector.
-func (sc ServiceCollector) Describe(ch chan<- *prometheus.Desc) {
-	ch <- prometheus.NewDesc("service_selector_labels", "service selector labels", []string{}, nil)
-}
-
-// Collect is called by the Prometheus registry when collecting metrics.
-func (sc ServiceCollector) Collect(ch chan<- prometheus.Metric) {
-	svcs := sc.KubeClusterCache.GetAllServices()
-	for _, svc := range svcs {
-		labels, values := prom.KubeLabelsToLabels(svc.Spec.Selector)
-		if len(labels) > 0 {
-			m := newServiceMetric(svc.GetName(), svc.GetNamespace(), "service_selector_labels", labels, values)
-			ch <- m
-		}
-	}
-}
-
-//--------------------------------------------------------------------------
-//  ServiceMetric
-//--------------------------------------------------------------------------
-
-// ServiceMetric is a prometheus.Metric used to encode service selector labels
-type ServiceMetric struct {
-	fqName      string
-	help        string
-	labelNames  []string
-	labelValues []string
-	serviceName string
-	namespace   string
-}
-
-// Creates a new ServiceMetric, implementation of prometheus.Metric
-func newServiceMetric(name, namespace, fqname string, labelNames []string, labelvalues []string) ServiceMetric {
-	return ServiceMetric{
-		fqName:      fqname,
-		labelNames:  labelNames,
-		labelValues: labelvalues,
-		help:        "service_selector_labels Service Selector Labels",
-		serviceName: name,
-		namespace:   namespace,
-	}
-}
-
-// Desc returns the descriptor for the Metric. This method idempotently
-// returns the same descriptor throughout the lifetime of the Metric.
-func (s ServiceMetric) Desc() *prometheus.Desc {
-	l := prometheus.Labels{"service": s.serviceName, "namespace": s.namespace}
-	return prometheus.NewDesc(s.fqName, s.help, s.labelNames, l)
-}
-
-// Write encodes the Metric into a "Metric" Protocol Buffer data
-// transmission object.
-func (s ServiceMetric) Write(m *dto.Metric) error {
-	h := float64(1)
-	m.Gauge = &dto.Gauge{
-		Value: &h,
-	}
-	var labels []*dto.LabelPair
-	for i := range s.labelNames {
-		labels = append(labels, &dto.LabelPair{
-			Name:  &s.labelNames[i],
-			Value: &s.labelValues[i],
-		})
-	}
-	n := "namespace"
-	labels = append(labels, &dto.LabelPair{
-		Name:  &n,
-		Value: &s.namespace,
-	})
-	r := "service"
-	labels = append(labels, &dto.LabelPair{
-		Name:  &r,
-		Value: &s.serviceName,
-	})
-	m.Label = labels
-	return nil
-}
-
-//--------------------------------------------------------------------------
-//  NamespaceAnnotationCollector
-//--------------------------------------------------------------------------
-
-// NamespaceAnnotationCollector is a prometheus collector that generates NamespaceAnnotationMetrics
-type NamespaceAnnotationCollector struct {
-	KubeClusterCache clustercache.ClusterCache
-}
-
-// Describe sends the super-set of all possible descriptors of metrics
-// collected by this Collector.
-func (nsac NamespaceAnnotationCollector) Describe(ch chan<- *prometheus.Desc) {
-	ch <- prometheus.NewDesc("kube_namespace_annotations", "namespace annotations", []string{}, nil)
-}
-
-// Collect is called by the Prometheus registry when collecting metrics.
-func (nsac NamespaceAnnotationCollector) Collect(ch chan<- prometheus.Metric) {
-	namespaces := nsac.KubeClusterCache.GetAllNamespaces()
-	for _, namespace := range namespaces {
-		labels, values := prom.KubeAnnotationsToLabels(namespace.Annotations)
-		if len(labels) > 0 {
-			m := newNamespaceAnnotationsMetric(namespace.GetName(), "kube_namespace_annotations", labels, values)
-			ch <- m
-		}
-	}
-}
-
-//--------------------------------------------------------------------------
-//  NamespaceAnnotationsMetric
-//--------------------------------------------------------------------------
-
-// NamespaceAnnotationsMetric is a prometheus.Metric used to encode namespace annotations
-type NamespaceAnnotationsMetric struct {
-	fqName      string
-	help        string
-	labelNames  []string
-	labelValues []string
-	namespace   string
-}
-
-// Creates a new NamespaceAnnotationsMetric, implementation of prometheus.Metric
-func newNamespaceAnnotationsMetric(namespace, fqname string, labelNames []string, labelValues []string) NamespaceAnnotationsMetric {
-	return NamespaceAnnotationsMetric{
-		namespace:   namespace,
-		fqName:      fqname,
-		labelNames:  labelNames,
-		labelValues: labelValues,
-		help:        "kube_namespace_annotations Namespace Annotations",
-	}
-}
-
-// Desc returns the descriptor for the Metric. This method idempotently
-// returns the same descriptor throughout the lifetime of the Metric.
-func (nam NamespaceAnnotationsMetric) Desc() *prometheus.Desc {
-	l := prometheus.Labels{"namespace": nam.namespace}
-	return prometheus.NewDesc(nam.fqName, nam.help, nam.labelNames, l)
-}
-
-// Write encodes the Metric into a "Metric" Protocol Buffer data
-// transmission object.
-func (nam NamespaceAnnotationsMetric) Write(m *dto.Metric) error {
-	h := float64(1)
-	m.Gauge = &dto.Gauge{
-		Value: &h,
-	}
-
-	var labels []*dto.LabelPair
-	for i := range nam.labelNames {
-		labels = append(labels, &dto.LabelPair{
-			Name:  &nam.labelNames[i],
-			Value: &nam.labelValues[i],
-		})
-	}
-	n := "namespace"
-	labels = append(labels, &dto.LabelPair{
-		Name:  &n,
-		Value: &nam.namespace,
-	})
-	m.Label = labels
-	return nil
-}
-
-//--------------------------------------------------------------------------
-//  PodAnnotationCollector
-//--------------------------------------------------------------------------
-
-// PodAnnotationCollector is a prometheus collector that generates PodAnnotationMetrics
-type PodAnnotationCollector struct {
-	KubeClusterCache clustercache.ClusterCache
-}
-
-// Describe sends the super-set of all possible descriptors of metrics
-// collected by this Collector.
-func (pac PodAnnotationCollector) Describe(ch chan<- *prometheus.Desc) {
-	ch <- prometheus.NewDesc("kube_pod_annotations", "pod annotations", []string{}, nil)
-}
-
-// Collect is called by the Prometheus registry when collecting metrics.
-func (pac PodAnnotationCollector) Collect(ch chan<- prometheus.Metric) {
-	pods := pac.KubeClusterCache.GetAllPods()
-	for _, pod := range pods {
-		labels, values := prom.KubeAnnotationsToLabels(pod.Annotations)
-		if len(labels) > 0 {
-			m := newPodAnnotationMetric(pod.GetNamespace(), pod.GetName(), "kube_pod_annotations", labels, values)
-			ch <- m
-		}
-	}
-}
-
-//--------------------------------------------------------------------------
-//  PodAnnotationsMetric
-//--------------------------------------------------------------------------
-
-// PodAnnotationsMetric is a prometheus.Metric used to encode namespace annotations
-type PodAnnotationsMetric struct {
-	name        string
-	fqName      string
-	help        string
-	labelNames  []string
-	labelValues []string
-	namespace   string
-}
-
-// Creates a new PodAnnotationsMetric, implementation of prometheus.Metric
-func newPodAnnotationMetric(namespace, name, fqname string, labelNames []string, labelValues []string) PodAnnotationsMetric {
-	return PodAnnotationsMetric{
-		namespace:   namespace,
-		name:        name,
-		fqName:      fqname,
-		labelNames:  labelNames,
-		labelValues: labelValues,
-		help:        "kube_pod_annotations Pod Annotations",
-	}
-}
-
-// Desc returns the descriptor for the Metric. This method idempotently
-// returns the same descriptor throughout the lifetime of the Metric.
-func (pam PodAnnotationsMetric) Desc() *prometheus.Desc {
-	l := prometheus.Labels{"namespace": pam.namespace, "pod": pam.name}
-	return prometheus.NewDesc(pam.fqName, pam.help, pam.labelNames, l)
-}
-
-// Write encodes the Metric into a "Metric" Protocol Buffer data
-// transmission object.
-func (pam PodAnnotationsMetric) Write(m *dto.Metric) error {
-	h := float64(1)
-	m.Gauge = &dto.Gauge{
-		Value: &h,
-	}
-
-	var labels []*dto.LabelPair
-	for i := range pam.labelNames {
-		labels = append(labels, &dto.LabelPair{
-			Name:  &pam.labelNames[i],
-			Value: &pam.labelValues[i],
-		})
-	}
-	n := "namespace"
-	labels = append(labels, &dto.LabelPair{
-		Name:  &n,
-		Value: &pam.namespace,
-	})
-	r := "pod"
-	labels = append(labels, &dto.LabelPair{
-		Name:  &r,
-		Value: &pam.name,
-	})
-	m.Label = labels
-	return nil
-}
-
 //--------------------------------------------------------------------------
 //  ClusterInfoCollector
 //--------------------------------------------------------------------------
@@ -529,402 +96,8 @@ func (cim ClusterInfoMetric) Write(m *dto.Metric) error {
 	return nil
 }
 
-//--------------------------------------------------------------------------
-//  KubeNodeStatusCapacityMemoryBytesCollector
-//--------------------------------------------------------------------------
-
-// KubeNodeStatusCapacityMemoryBytesCollector is a prometheus collector that generates
-// KubeNodeStatusCapacityMemoryBytesMetrics
-type KubeNodeStatusCapacityMemoryBytesCollector struct {
-	KubeClusterCache clustercache.ClusterCache
-}
-
-// Describe sends the super-set of all possible descriptors of metrics
-// collected by this Collector.
-func (nsac KubeNodeStatusCapacityMemoryBytesCollector) Describe(ch chan<- *prometheus.Desc) {
-	ch <- prometheus.NewDesc("kube_node_status_capacity_memory_bytes", "node capacity memory bytes", []string{}, nil)
-}
-
-// Collect is called by the Prometheus registry when collecting metrics.
-func (nsac KubeNodeStatusCapacityMemoryBytesCollector) Collect(ch chan<- prometheus.Metric) {
-	nodes := nsac.KubeClusterCache.GetAllNodes()
-	for _, node := range nodes {
-		// k8s.io/apimachinery/pkg/api/resource/amount.go and
-		// k8s.io/apimachinery/pkg/api/resource/quantity.go for
-		// details on the "amount" API. See
-		// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-types
-		// for the units of memory and CPU.
-		memoryBytes := node.Status.Capacity.Memory().Value()
-
-		m := newKubeNodeStatusCapacityMemoryBytesMetric(node.GetName(), memoryBytes, "kube_node_status_capacity_memory_bytes", nil, nil)
-		ch <- m
-	}
-}
-
-//--------------------------------------------------------------------------
-//  KubeNodeStatusCapacityMemoryBytesMetric
-//--------------------------------------------------------------------------
-
-// KubeNodeStatusCapacityMemoryBytesMetric is a prometheus.Metric used to encode
-// a duplicate of the deprecated kube-state-metrics metric
-// kube_node_status_capacity_memory_bytes
-type KubeNodeStatusCapacityMemoryBytesMetric struct {
-	fqName      string
-	help        string
-	labelNames  []string
-	labelValues []string
-	bytes       int64
-	node        string
-}
-
-// Creates a new KubeNodeStatusCapacityMemoryBytesMetric, implementation of prometheus.Metric
-func newKubeNodeStatusCapacityMemoryBytesMetric(node string, bytes int64, fqname string, labelNames []string, labelValues []string) KubeNodeStatusCapacityMemoryBytesMetric {
-	return KubeNodeStatusCapacityMemoryBytesMetric{
-		fqName:      fqname,
-		labelNames:  labelNames,
-		labelValues: labelValues,
-		help:        "kube_node_status_capacity_memory_bytes Node Capacity Memory Bytes",
-		bytes:       bytes,
-		node:        node,
-	}
-}
-
-// Desc returns the descriptor for the Metric. This method idempotently
-// returns the same descriptor throughout the lifetime of the Metric.
-func (nam KubeNodeStatusCapacityMemoryBytesMetric) Desc() *prometheus.Desc {
-	l := prometheus.Labels{"node": nam.node}
-	return prometheus.NewDesc(nam.fqName, nam.help, nam.labelNames, l)
-}
-
-// Write encodes the Metric into a "Metric" Protocol Buffer data
-// transmission object.
-func (nam KubeNodeStatusCapacityMemoryBytesMetric) Write(m *dto.Metric) error {
-	h := float64(nam.bytes)
-	m.Gauge = &dto.Gauge{
-		Value: &h,
-	}
-
-	var labels []*dto.LabelPair
-	for i := range nam.labelNames {
-		labels = append(labels, &dto.LabelPair{
-			Name:  &nam.labelNames[i],
-			Value: &nam.labelValues[i],
-		})
-	}
-	n := "node"
-	labels = append(labels, &dto.LabelPair{
-		Name:  &n,
-		Value: &nam.node,
-	})
-	m.Label = labels
-	return nil
-}
-
-//--------------------------------------------------------------------------
-//  KubeNodeStatusCapacityCPUCoresCollector
-//--------------------------------------------------------------------------
-
-// KubeNodeStatusCapacityCPUCoresCollector is a prometheus collector that generates
-// KubeNodeStatusCapacityCPUCoresMetrics
-type KubeNodeStatusCapacityCPUCoresCollector struct {
-	KubeClusterCache clustercache.ClusterCache
-}
-
-// Describe sends the super-set of all possible descriptors of metrics
-// collected by this Collector.
-func (nsac KubeNodeStatusCapacityCPUCoresCollector) Describe(ch chan<- *prometheus.Desc) {
-	ch <- prometheus.NewDesc("kube_node_status_capacity_cpu_cores", "node capacity cpu cores", []string{}, nil)
-}
-
-// Collect is called by the Prometheus registry when collecting metrics.
-func (nsac KubeNodeStatusCapacityCPUCoresCollector) Collect(ch chan<- prometheus.Metric) {
-	nodes := nsac.KubeClusterCache.GetAllNodes()
-	for _, node := range nodes {
-		// k8s.io/apimachinery/pkg/api/resource/amount.go and
-		// k8s.io/apimachinery/pkg/api/resource/quantity.go for
-		// details on the "amount" API. See
-		// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-types
-		// for the units of memory and CPU.
-		cpuCores := float64(node.Status.Capacity.Cpu().MilliValue()) / 1000
-
-		m := newKubeNodeStatusCapacityCPUCoresMetric(node.GetName(), cpuCores, "kube_node_status_capacity_cpu_cores", nil, nil)
-		ch <- m
-	}
-}
-
-//--------------------------------------------------------------------------
-//  KubeNodeStatusCapacityCPUCoresMetric
-//--------------------------------------------------------------------------
-
-// KubeNodeStatusCapacityCPUCoresMetric is a prometheus.Metric used to encode
-// a duplicate of the deprecated kube-state-metrics metric
-// kube_node_status_capacity_memory_bytes
-type KubeNodeStatusCapacityCPUCoresMetric struct {
-	fqName      string
-	help        string
-	labelNames  []string
-	labelValues []string
-	cores       float64
-	node        string
-}
-
-// Creates a new KubeNodeStatusCapacityCPUCoresMetric, implementation of prometheus.Metric
-func newKubeNodeStatusCapacityCPUCoresMetric(node string, cores float64, fqname string, labelNames []string, labelValues []string) KubeNodeStatusCapacityCPUCoresMetric {
-	return KubeNodeStatusCapacityCPUCoresMetric{
-		fqName:      fqname,
-		labelNames:  labelNames,
-		labelValues: labelValues,
-		help:        "kube_node_status_capacity_cpu_cores Node Capacity CPU Cores",
-		cores:       cores,
-		node:        node,
-	}
-}
-
-// Desc returns the descriptor for the Metric. This method idempotently
-// returns the same descriptor throughout the lifetime of the Metric.
-func (nam KubeNodeStatusCapacityCPUCoresMetric) Desc() *prometheus.Desc {
-	l := prometheus.Labels{"node": nam.node}
-	return prometheus.NewDesc(nam.fqName, nam.help, nam.labelNames, l)
-}
-
-// Write encodes the Metric into a "Metric" Protocol Buffer data
-// transmission object.
-func (nam KubeNodeStatusCapacityCPUCoresMetric) Write(m *dto.Metric) error {
-	h := nam.cores
-	m.Gauge = &dto.Gauge{
-		Value: &h,
-	}
-
-	var labels []*dto.LabelPair
-	for i := range nam.labelNames {
-		labels = append(labels, &dto.LabelPair{
-			Name:  &nam.labelNames[i],
-			Value: &nam.labelValues[i],
-		})
-	}
-	n := "node"
-	labels = append(labels, &dto.LabelPair{
-		Name:  &n,
-		Value: &nam.node,
-	})
-	m.Label = labels
-	return nil
-}
-
-//--------------------------------------------------------------------------
-//  KubePodLabelsCollector
-//--------------------------------------------------------------------------
-//
-// We use this to emit kube_pod_labels with all of a pod's labels, regardless
-// of the whitelist setting introduced in KSM v2. See
-// https://github.com/kubernetes/kube-state-metrics/issues/1270#issuecomment-712986441
-
-// KubePodLabelsCollector is a prometheus collector that generates
-// KubePodLabelsMetrics
-type KubePodLabelsCollector struct {
-	KubeClusterCache clustercache.ClusterCache
-}
-
-// Describe sends the super-set of all possible descriptors of metrics
-// collected by this Collector.
-func (nsac KubePodLabelsCollector) Describe(ch chan<- *prometheus.Desc) {
-	ch <- prometheus.NewDesc("kube_pod_labels", "all labels for each pod prefixed with label_", []string{}, nil)
-}
-
-// Collect is called by the Prometheus registry when collecting metrics.
-func (nsac KubePodLabelsCollector) Collect(ch chan<- prometheus.Metric) {
-	pods := nsac.KubeClusterCache.GetAllPods()
-	for _, pod := range pods {
-
-		labelNames, labelValues := prom.KubePrependQualifierToLabels(pod.GetLabels(), "label_")
-
-		m := newKubePodLabelsMetric(
-			pod.GetName(),
-			pod.GetNamespace(),
-			string(pod.GetUID()),
-			"kube_pod_labels",
-			labelNames,
-			labelValues,
-		)
-		ch <- m
-	}
-}
-
-//--------------------------------------------------------------------------
-//  KubePodLabelsMetric
-//--------------------------------------------------------------------------
-
-// KubePodLabelsMetric is a prometheus.Metric used to encode
-// a duplicate of the deprecated kube-state-metrics metric
-// kube_pod_labels
-type KubePodLabelsMetric struct {
-	fqName      string
-	help        string
-	labelNames  []string
-	labelValues []string
-	pod         string
-	namespace   string
-	uid         string
-}
-
-// Creates a new KubePodLabelsMetric, implementation of prometheus.Metric
-func newKubePodLabelsMetric(pod string, namespace string, uid string, fqname string, labelNames []string, labelValues []string) KubePodLabelsMetric {
-	return KubePodLabelsMetric{
-		fqName:      fqname,
-		labelNames:  labelNames,
-		labelValues: labelValues,
-		help:        "kube_pod_labels all labels for each pod prefixed with label_",
-		pod:         pod,
-		namespace:   namespace,
-		uid:         uid,
-	}
-}
-
-// Desc returns the descriptor for the Metric. This method idempotently
-// returns the same descriptor throughout the lifetime of the Metric.
-func (nam KubePodLabelsMetric) Desc() *prometheus.Desc {
-	l := prometheus.Labels{
-		"pod":       nam.pod,
-		"namespace": nam.namespace,
-		"uid":       nam.uid,
-	}
-	return prometheus.NewDesc(nam.fqName, nam.help, nam.labelNames, l)
-}
-
-// Write encodes the Metric into a "Metric" Protocol Buffer data
-// transmission object.
-func (nam KubePodLabelsMetric) Write(m *dto.Metric) error {
-	h := float64(1)
-	m.Gauge = &dto.Gauge{
-		Value: &h,
-	}
-
-	var labels []*dto.LabelPair
-	for i := range nam.labelNames {
-		labels = append(labels, &dto.LabelPair{
-			Name:  &nam.labelNames[i],
-			Value: &nam.labelValues[i],
-		})
-	}
-
-	podString := "pod"
-	namespaceString := "namespace"
-	uidString := "uid"
-	labels = append(labels,
-		&dto.LabelPair{
-			Name:  &podString,
-			Value: &nam.pod,
-		},
-		&dto.LabelPair{
-			Name:  &namespaceString,
-			Value: &nam.namespace,
-		}, &dto.LabelPair{
-			Name:  &uidString,
-			Value: &nam.uid,
-		},
-	)
-	m.Label = labels
-	return nil
-}
-
-//--------------------------------------------------------------------------
-//  KubeNodeLabelsCollector
-//--------------------------------------------------------------------------
-//
-// We use this to emit kube_node_labels with all of a node's labels, regardless
-// of the whitelist setting introduced in KSM v2. See
-// https://github.com/kubernetes/kube-state-metrics/issues/1270#issuecomment-712986441
-
-// KubeNodeLabelsCollector is a prometheus collector that generates
-// KubeNodeLabelsMetrics
-type KubeNodeLabelsCollector struct {
-	KubeClusterCache clustercache.ClusterCache
-}
-
-// Describe sends the super-set of all possible descriptors of metrics
-// collected by this Collector.
-func (nsac KubeNodeLabelsCollector) Describe(ch chan<- *prometheus.Desc) {
-	ch <- prometheus.NewDesc("kube_node_labels", "all labels for each node prefixed with label_", []string{}, nil)
-}
-
-// Collect is called by the Prometheus registry when collecting metrics.
-func (nsac KubeNodeLabelsCollector) Collect(ch chan<- prometheus.Metric) {
-	nodes := nsac.KubeClusterCache.GetAllNodes()
-	for _, node := range nodes {
-
-		labelNames, labelValues := prom.KubePrependQualifierToLabels(node.GetLabels(), "label_")
-
-		m := newKubeNodeLabelsMetric(
-			node.GetName(),
-			"kube_node_labels",
-			labelNames,
-			labelValues,
-		)
-		ch <- m
-	}
-}
-
-//--------------------------------------------------------------------------
-//  KubeNodeLabelsMetric
-//--------------------------------------------------------------------------
-
-// KubeNodeLabelsMetric is a prometheus.Metric used to encode
-// a duplicate of the deprecated kube-state-metrics metric
-// kube_node_labels
-type KubeNodeLabelsMetric struct {
-	fqName      string
-	help        string
-	labelNames  []string
-	labelValues []string
-	node        string
-}
-
-// Creates a new KubeNodeLabelsMetric, implementation of prometheus.Metric
-func newKubeNodeLabelsMetric(node string, fqname string, labelNames []string, labelValues []string) KubeNodeLabelsMetric {
-	return KubeNodeLabelsMetric{
-		fqName:      fqname,
-		labelNames:  labelNames,
-		labelValues: labelValues,
-		help:        "kube_node_labels all labels for each node prefixed with label_",
-		node:        node,
-	}
-}
-
-// Desc returns the descriptor for the Metric. This method idempotently
-// returns the same descriptor throughout the lifetime of the Metric.
-func (nam KubeNodeLabelsMetric) Desc() *prometheus.Desc {
-	l := prometheus.Labels{
-		"node": nam.node,
-	}
-	return prometheus.NewDesc(nam.fqName, nam.help, nam.labelNames, l)
-}
-
-// Write encodes the Metric into a "Metric" Protocol Buffer data
-// transmission object.
-func (nam KubeNodeLabelsMetric) Write(m *dto.Metric) error {
-	h := float64(1)
-	m.Gauge = &dto.Gauge{
-		Value: &h,
-	}
-
-	var labels []*dto.LabelPair
-	for i := range nam.labelNames {
-		labels = append(labels, &dto.LabelPair{
-			Name:  &nam.labelNames[i],
-			Value: &nam.labelValues[i],
-		})
-	}
-
-	nodeString := "node"
-	labels = append(labels, &dto.LabelPair{Name: &nodeString, Value: &nam.node})
-	m.Label = labels
-	return nil
-}
-
-// toStringPtr is used to create a new string pointer from iteration vars
-func toStringPtr(s string) *string {
-	return &s
-}
+// returns a pointer to the string provided
+func toStringPtr(s string) *string { return &s }
 
 //--------------------------------------------------------------------------
 //  Cost Model Metrics Initialization
@@ -1042,46 +215,10 @@ func initCostModelMetrics(clusterCache clustercache.ClusterCache, provider cloud
 		prometheus.MustRegister(clusterManagementCostGv, lbCostGv)
 
 		// General Metric Collectors
-		prometheus.MustRegister(ServiceCollector{
-			KubeClusterCache: clusterCache,
-		})
-		prometheus.MustRegister(DeploymentCollector{
-			KubeClusterCache: clusterCache,
-		})
-		prometheus.MustRegister(StatefulsetCollector{
-			KubeClusterCache: clusterCache,
-		})
 		prometheus.MustRegister(ClusterInfoCollector{
 			KubeClientSet: clusterCache.GetClient(),
 			Cloud:         provider,
 		})
-
-		if env.IsEmitNamespaceAnnotationsMetric() {
-			prometheus.MustRegister(NamespaceAnnotationCollector{
-				KubeClusterCache: clusterCache,
-			})
-		}
-
-		if env.IsEmitPodAnnotationsMetric() {
-			prometheus.MustRegister(PodAnnotationCollector{
-				KubeClusterCache: clusterCache,
-			})
-		}
-
-		if env.IsEmitKsmV1Metrics() {
-			prometheus.MustRegister(KubeNodeStatusCapacityMemoryBytesCollector{
-				KubeClusterCache: clusterCache,
-			})
-			prometheus.MustRegister(KubeNodeStatusCapacityCPUCoresCollector{
-				KubeClusterCache: clusterCache,
-			})
-			prometheus.MustRegister(KubePodLabelsCollector{
-				KubeClusterCache: clusterCache,
-			})
-			prometheus.MustRegister(KubeNodeLabelsCollector{
-				KubeClusterCache: clusterCache,
-			})
-		}
 	})
 }
 
@@ -1126,6 +263,18 @@ func NewCostModelMetricsEmitter(promClient promclient.Client, clusterCache clust
 	// init will only actually execute once to register the custom gauges
 	initCostModelMetrics(clusterCache, provider)
 
+	// if the metrics pod is not enabled, we want to emit those metrics from this pod.
+	// NOTE: This is not optimal, as we calculate costs based on run times for other containers.
+	// NOTE: The metrics for run times should be emitted separate from cost-model
+	if !env.IsKubecostMetricsPodEnabled() {
+		metrics.InitKubeMetrics(clusterCache, &metrics.KubeMetricsOpts{
+			EmitKubecostControllerMetrics: true,
+			EmitNamespaceAnnotations:      env.IsEmitNamespaceAnnotationsMetric(),
+			EmitPodAnnotations:            env.IsEmitPodAnnotationsMetric(),
+			EmitKubeStateMetrics:          env.IsEmitKsmV1Metrics(),
+		})
+	}
+
 	return &CostModelMetricsEmitter{
 		PrometheusClient:              promClient,
 		KubeClusterCache:              clusterCache,
@@ -1381,6 +530,11 @@ func (cmme *CostModelMetricsEmitter) Start() bool {
 
 			pvs := cmme.KubeClusterCache.GetAllPersistentVolumes()
 			for _, pv := range pvs {
+				// Omit pv_hourly_cost if the volume status is failed
+				if pv.Status.Phase == v1.VolumeFailed {
+					continue
+				}
+
 				parameters, ok := storageClassMap[pv.Spec.StorageClassName]
 				if !ok {
 					klog.V(4).Infof("Unable to find parameters for storage class \"%s\". Does pv \"%s\" have a storageClassName?", pv.Spec.StorageClassName, pv.Name)

pkg/env/costmodelenv.go

@@ -76,7 +76,7 @@ const (
 // GetAWSAccessKeyID returns the environment variable value for AWSAccessKeyIDEnvVar which represents
 // the AWS access key for authentication
 func GetAppVersion() string {
-	return Get(AppVersionEnvVar, "1.82.2")
+	return Get(AppVersionEnvVar, "1.83.1")
 }
 
 // IsEmitNamespaceAnnotationsMetric returns true if cost-model is configured to emit the kube_namespace_annotations metric

pkg/env/kubemetricsenv.go

@@ -0,0 +1,15 @@
+package env
+
+const (
+	KubecostMetricsPodEnabledEnvVar = "KUBECOST_METRICS_POD_ENABLED"
+	KubecostMetricsPodPortEnvVar    = "KUBECOST_METRICS_PORT"
+)
+
+func GetKubecostMetricsPort() int {
+	return GetInt(KubecostMetricsPodPortEnvVar, 9005)
+}
+
+// IsKubecostMetricsPodEnabled returns true if the kubecost metrics pod is deployed
+func IsKubecostMetricsPodEnabled() bool {
+	return GetBool(KubecostMetricsPodEnabledEnvVar, false)
+}

pkg/kubecost/allocation.go

@@ -2405,3 +2405,67 @@ func (asr *AllocationSetRange) Window() Window {
 
 	return NewWindow(&start, &end)
 }
+
+// Start returns the earliest start of all Allocations in the AllocationSetRange.
+// It returns an error if there are no allocations.
+func (asr *AllocationSetRange) Start() (time.Time, error) {
+	start := time.Time{}
+	firstStartNotSet := true
+	asr.Each(func(i int, as *AllocationSet) {
+		as.Each(func(s string, a *Allocation) {
+			if firstStartNotSet {
+				start = a.Start
+				firstStartNotSet = false
+			}
+			if a.Start.Before(start) {
+				start = a.Start
+			}
+		})
+	})
+
+	if firstStartNotSet {
+		return start, fmt.Errorf("had no data to compute a start from")
+	}
+
+	return start, nil
+}
+
+// End returns the latest end of all Allocations in the AllocationSetRange.
+// It returns an error if there are no allocations.
+func (asr *AllocationSetRange) End() (time.Time, error) {
+	end := time.Time{}
+	firstEndNotSet := true
+	asr.Each(func(i int, as *AllocationSet) {
+		as.Each(func(s string, a *Allocation) {
+			if firstEndNotSet {
+				end = a.End
+				firstEndNotSet = false
+			}
+			if a.End.After(end) {
+				end = a.End
+			}
+		})
+	})
+
+	if firstEndNotSet {
+		return end, fmt.Errorf("had no data to compute an end from")
+	}
+
+	return end, nil
+}
+
+// Minutes returns the duration, in minutes, between the earliest start
+// and the latest end of all allocations in the AllocationSetRange.
+func (asr *AllocationSetRange) Minutes() float64 {
+	start, err := asr.Start()
+	if err != nil {
+		return 0
+	}
+	end, err := asr.End()
+	if err != nil {
+		return 0
+	}
+	duration := end.Sub(start)
+
+	return duration.Minutes()
+}

pkg/kubecost/allocation_test.go

@@ -2161,3 +2161,265 @@ func TestAllocationSetRange_InsertRange(t *testing.T) {
 
 // TODO niko/etl
 // func TestAllocationSetRange_Window(t *testing.T) {}
+
+func TestAllocationSetRange_Start(t *testing.T) {
+	tests := []struct {
+		name string
+		arg  *AllocationSetRange
+
+		expectError bool
+		expected    time.Time
+	}{
+		{
+			name: "Empty ASR",
+			arg:  nil,
+
+			expectError: true,
+		},
+		{
+			name: "Single allocation",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								Start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "Two allocations",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								Start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+							"b": &Allocation{
+								Start: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "Two AllocationSets",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								Start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"b": &Allocation{
+								Start: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+	}
+
+	for _, test := range tests {
+		result, err := test.arg.Start()
+		if test.expectError && err != nil {
+			continue
+		}
+
+		if test.expectError && err == nil {
+			t.Errorf("%s: expected error and got none", test.name)
+		} else if result != test.expected {
+			t.Errorf("%s: expected %s but got %s", test.name, test.expected, result)
+		}
+	}
+}
+
+func TestAllocationSetRange_End(t *testing.T) {
+	tests := []struct {
+		name string
+		arg  *AllocationSetRange
+
+		expectError bool
+		expected    time.Time
+	}{
+		{
+			name: "Empty ASR",
+			arg:  nil,
+
+			expectError: true,
+		},
+		{
+			name: "Single allocation",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								End: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "Two allocations",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								End: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+							"b": &Allocation{
+								End: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "Two AllocationSets",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								End: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"b": &Allocation{
+								End: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+		},
+	}
+
+	for _, test := range tests {
+		result, err := test.arg.End()
+		if test.expectError && err != nil {
+			continue
+		}
+
+		if test.expectError && err == nil {
+			t.Errorf("%s: expected error and got none", test.name)
+		} else if result != test.expected {
+			t.Errorf("%s: expected %s but got %s", test.name, test.expected, result)
+		}
+	}
+}
+
+func TestAllocationSetRange_Minutes(t *testing.T) {
+	tests := []struct {
+		name string
+		arg  *AllocationSetRange
+
+		expected float64
+	}{
+		{
+			name: "Empty ASR",
+			arg:  nil,
+
+			expected: 0,
+		},
+		{
+			name: "Single allocation",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								Start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+								End:   time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: 24 * 60,
+		},
+		{
+			name: "Two allocations",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								Start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+								End:   time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+							"b": &Allocation{
+								Start: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+								End:   time.Date(1970, 1, 3, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: 2 * 24 * 60,
+		},
+		{
+			name: "Two AllocationSets",
+			arg: &AllocationSetRange{
+				allocations: []*AllocationSet{
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"a": &Allocation{
+								Start: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC),
+								End:   time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+					&AllocationSet{
+						allocations: map[string]*Allocation{
+							"b": &Allocation{
+								Start: time.Date(1970, 1, 2, 0, 0, 0, 0, time.UTC),
+								End:   time.Date(1970, 1, 3, 0, 0, 0, 0, time.UTC),
+							},
+						},
+					},
+				},
+			},
+
+			expected: 2 * 24 * 60,
+		},
+	}
+
+	for _, test := range tests {
+		result := test.arg.Minutes()
+		if result != test.expected {
+			t.Errorf("%s: expected %f but got %f", test.name, test.expected, result)
+		}
+	}
+}

pkg/metrics/deploymentmetrics.go

@@ -0,0 +1,255 @@
+package metrics
+
+import (
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/prom"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+)
+
+//--------------------------------------------------------------------------
+//  KubecostDeploymentCollector
+//--------------------------------------------------------------------------
+
+// KubecostDeploymentCollector is a prometheus collector that generates kubecost
+// specific deployment metrics.
+type KubecostDeploymentCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (kdc KubecostDeploymentCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("deployment_match_labels", "deployment match labels", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kdc KubecostDeploymentCollector) Collect(ch chan<- prometheus.Metric) {
+	ds := kdc.KubeClusterCache.GetAllDeployments()
+
+	for _, deployment := range ds {
+		deploymentName := deployment.GetName()
+		deploymentNS := deployment.GetNamespace()
+
+		labels, values := prom.KubeLabelsToLabels(deployment.Spec.Selector.MatchLabels)
+		if len(labels) > 0 {
+			m := newDeploymentMatchLabelsMetric(deploymentName, deploymentNS, "deployment_match_labels", labels, values)
+			ch <- m
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  DeploymentMatchLabelsMetric
+//--------------------------------------------------------------------------
+
+// DeploymentMatchLabelsMetric is a prometheus.Metric used to encode deployment match labels
+type DeploymentMatchLabelsMetric struct {
+	fqName         string
+	help           string
+	labelNames     []string
+	labelValues    []string
+	deploymentName string
+	namespace      string
+}
+
+// Creates a new DeploymentMatchLabelsMetric, implementation of prometheus.Metric
+func newDeploymentMatchLabelsMetric(name, namespace, fqname string, labelNames, labelvalues []string) DeploymentMatchLabelsMetric {
+	return DeploymentMatchLabelsMetric{
+		fqName:         fqname,
+		labelNames:     labelNames,
+		labelValues:    labelvalues,
+		help:           "deployment_match_labels Deployment Match Labels",
+		deploymentName: name,
+		namespace:      namespace,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (dmlm DeploymentMatchLabelsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"deployment": dmlm.deploymentName,
+		"namespace":  dmlm.namespace,
+	}
+	return prometheus.NewDesc(dmlm.fqName, dmlm.help, dmlm.labelNames, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (dmlm DeploymentMatchLabelsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+	var labels []*dto.LabelPair
+	for i := range dmlm.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &dmlm.labelNames[i],
+			Value: &dmlm.labelValues[i],
+		})
+	}
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("namespace"),
+		Value: &dmlm.namespace,
+	})
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("deployment"),
+		Value: &dmlm.deploymentName,
+	})
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeDeploymentCollector
+//--------------------------------------------------------------------------
+
+// KubeDeploymentCollector is a prometheus collector that generates
+type KubeDeploymentCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (kdc KubeDeploymentCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_deployment_spec_replicas", "Number of desired pods for a deployment.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_deployment_status_replicas_available", "The number of available replicas per deployment.", []string{}, nil)
+
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kdc KubeDeploymentCollector) Collect(ch chan<- prometheus.Metric) {
+	deployments := kdc.KubeClusterCache.GetAllDeployments()
+
+	for _, deployment := range deployments {
+		deploymentName := deployment.GetName()
+		deploymentNS := deployment.GetNamespace()
+
+		// Replicas Defined
+		var replicas int32
+		if deployment.Spec.Replicas == nil {
+			replicas = 1 // defaults to 1, documented on the 'Replicas' field
+		} else {
+			replicas = *deployment.Spec.Replicas
+		}
+
+		ch <- newKubeDeploymentReplicasMetric("kube_deployment_spec_replicas", deploymentName, deploymentNS, replicas)
+
+		// Replicas Available
+		ch <- newKubeDeploymentStatusAvailableReplicasMetric(
+			"kube_deployment_status_replicas_available",
+			deploymentName,
+			deploymentNS,
+			deployment.Status.AvailableReplicas)
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubeDeploymentReplicasMetric
+//--------------------------------------------------------------------------
+
+// KubeDeploymentReplicasMetric is a prometheus.Metric used to encode deployment match labels
+type KubeDeploymentReplicasMetric struct {
+	fqName     string
+	help       string
+	deployment string
+	namespace  string
+	replicas   float64
+}
+
+// Creates a new DeploymentMatchLabelsMetric, implementation of prometheus.Metric
+func newKubeDeploymentReplicasMetric(fqname, deployment, namespace string, replicas int32) KubeDeploymentReplicasMetric {
+	return KubeDeploymentReplicasMetric{
+		fqName:     fqname,
+		help:       "kube_deployment_spec_replicas Number of desired pods for a deployment.",
+		deployment: deployment,
+		namespace:  namespace,
+		replicas:   float64(replicas),
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kdr KubeDeploymentReplicasMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"deployment": kdr.deployment,
+		"namespace":  kdr.namespace,
+	}
+	return prometheus.NewDesc(kdr.fqName, kdr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kdr KubeDeploymentReplicasMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kdr.replicas,
+	}
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kdr.namespace,
+		},
+		{
+			Name:  toStringPtr("deployment"),
+			Value: &kdr.deployment,
+		},
+	}
+
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeDeploymentStatusAvailableReplicasMetric
+//--------------------------------------------------------------------------
+
+// KubeDeploymentStatusAvailableReplicasMetric is a prometheus.Metric used to encode deployment match labels
+type KubeDeploymentStatusAvailableReplicasMetric struct {
+	fqName            string
+	help              string
+	deployment        string
+	namespace         string
+	replicasAvailable float64
+}
+
+// Creates a new DeploymentMatchLabelsMetric, implementation of prometheus.Metric
+func newKubeDeploymentStatusAvailableReplicasMetric(fqname, deployment, namespace string, replicasAvailable int32) KubeDeploymentStatusAvailableReplicasMetric {
+	return KubeDeploymentStatusAvailableReplicasMetric{
+		fqName:            fqname,
+		help:              "kube_deployment_status_replicas_available The number of available replicas per deployment.",
+		deployment:        deployment,
+		namespace:         namespace,
+		replicasAvailable: float64(replicasAvailable),
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kdr KubeDeploymentStatusAvailableReplicasMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"deployment": kdr.deployment,
+		"namespace":  kdr.namespace,
+	}
+	return prometheus.NewDesc(kdr.fqName, kdr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kdr KubeDeploymentStatusAvailableReplicasMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kdr.replicasAvailable,
+	}
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kdr.namespace,
+		},
+		{
+			Name:  toStringPtr("deployment"),
+			Value: &kdr.deployment,
+		},
+	}
+
+	return nil
+}

pkg/metrics/jobmetrics.go

@@ -0,0 +1,116 @@
+package metrics
+
+import (
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	batchv1 "k8s.io/api/batch/v1"
+)
+
+var (
+	jobFailureReasons = []string{"BackoffLimitExceeded", "DeadLineExceeded", "Evicted"}
+)
+
+//--------------------------------------------------------------------------
+//  KubeJobCollector
+//--------------------------------------------------------------------------
+
+// KubeJobCollector is a prometheus collector that generates job sourced metrics.
+type KubeJobCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (kjc KubeJobCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_job_status_failed", "The number of pods which reached Phase Failed and the reason for failure.", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kjc KubeJobCollector) Collect(ch chan<- prometheus.Metric) {
+	jobs := kjc.KubeClusterCache.GetAllJobs()
+	for _, job := range jobs {
+		jobName := job.GetName()
+		jobNS := job.GetNamespace()
+
+		if job.Status.Failed == 0 {
+			ch <- newKubeJobStatusFailedMetric(jobName, jobNS, "kube_job_status_failed", "", 0)
+		} else {
+			for _, condition := range job.Status.Conditions {
+				if condition.Type == batchv1.JobFailed {
+					reasonKnown := false
+					for _, reason := range jobFailureReasons {
+						reasonKnown = reasonKnown || failureReason(&condition, reason)
+
+						ch <- newKubeJobStatusFailedMetric(jobName, jobNS, "kube_job_status_failed", reason, boolFloat64(failureReason(&condition, reason)))
+					}
+
+					// for unknown reasons
+					if !reasonKnown {
+						ch <- newKubeJobStatusFailedMetric(jobName, jobNS, "kube_job_status_failed", "", float64(job.Status.Failed))
+					}
+				}
+			}
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubeJobStatusFailedMetric
+//--------------------------------------------------------------------------
+
+// KubeJobStatusFailedMetric
+type KubeJobStatusFailedMetric struct {
+	fqName    string
+	help      string
+	job       string
+	namespace string
+	reason    string
+	value     float64
+}
+
+// Creates a new KubeJobStatusFailedMetric, implementation of prometheus.Metric
+func newKubeJobStatusFailedMetric(job, namespace, fqName, reason string, value float64) KubeJobStatusFailedMetric {
+	return KubeJobStatusFailedMetric{
+		fqName:    fqName,
+		help:      "kube_job_status_failed Failed job",
+		job:       job,
+		namespace: namespace,
+		reason:    reason,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kjsf KubeJobStatusFailedMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"job_name":  kjsf.job,
+		"namespace": kjsf.namespace,
+		"reason":    kjsf.reason,
+	}
+	return prometheus.NewDesc(kjsf.fqName, kjsf.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kjsf KubeJobStatusFailedMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kjsf.value,
+	}
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("job_name"),
+			Value: &kjsf.job,
+		},
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kjsf.namespace,
+		},
+		{
+			Name:  toStringPtr("reason"),
+			Value: &kjsf.reason,
+		},
+	}
+	return nil
+}

pkg/metrics/kubemetrics.go

@@ -0,0 +1,211 @@
+package metrics
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/prom"
+
+	"github.com/prometheus/client_golang/prometheus"
+	batchv1 "k8s.io/api/batch/v1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/validation"
+)
+
+//--------------------------------------------------------------------------
+//  Kube Metric Registration
+//--------------------------------------------------------------------------
+
+// initializer
+var kubeMetricInit sync.Once
+
+// KubeMetricsOpts represents our Kubernetes metrics emission options.
+type KubeMetricsOpts struct {
+	EmitKubecostControllerMetrics bool
+	EmitNamespaceAnnotations      bool
+	EmitPodAnnotations            bool
+	EmitKubeStateMetrics          bool
+}
+
+// DefaultKubeMetricsOpts returns KubeMetricsOpts with default values set
+func DefaultKubeMetricsOpts() *KubeMetricsOpts {
+	return &KubeMetricsOpts{
+		EmitKubecostControllerMetrics: true,
+		EmitNamespaceAnnotations:      false,
+		EmitPodAnnotations:            false,
+		EmitKubeStateMetrics:          true,
+	}
+}
+
+// InitKubeMetrics initializes kubernetes metric emission using the provided options.
+func InitKubeMetrics(clusterCache clustercache.ClusterCache, opts *KubeMetricsOpts) {
+	if opts == nil {
+		opts = DefaultKubeMetricsOpts()
+	}
+
+	kubeMetricInit.Do(func() {
+		if opts.EmitKubecostControllerMetrics {
+			prometheus.MustRegister(KubecostServiceCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubecostDeploymentCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubecostStatefulsetCollector{
+				KubeClusterCache: clusterCache,
+			})
+		}
+
+		if opts.EmitPodAnnotations {
+			prometheus.MustRegister(KubecostPodCollector{
+				KubeClusterCache: clusterCache,
+			})
+		}
+
+		if opts.EmitNamespaceAnnotations {
+			prometheus.MustRegister(KubecostNamespaceCollector{
+				KubeClusterCache: clusterCache,
+			})
+		}
+
+		if opts.EmitKubeStateMetrics {
+			prometheus.MustRegister(KubeNodeCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubeNamespaceCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubeDeploymentCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubePodCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubePVCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubePVCCollector{
+				KubeClusterCache: clusterCache,
+			})
+			prometheus.MustRegister(KubeJobCollector{
+				KubeClusterCache: clusterCache,
+			})
+		}
+	})
+}
+
+//--------------------------------------------------------------------------
+//  Kube Metric Helpers
+//--------------------------------------------------------------------------
+
+// getPersistentVolumeClaimClass returns StorageClassName. If no storage class was
+// requested, it returns "".
+func getPersistentVolumeClaimClass(claim *v1.PersistentVolumeClaim) string {
+	// Use beta annotation first
+	if class, found := claim.Annotations[v1.BetaStorageClassAnnotation]; found {
+		return class
+	}
+
+	if claim.Spec.StorageClassName != nil {
+		return *claim.Spec.StorageClassName
+	}
+
+	// Special non-empty string to indicate absence of storage class.
+	return "<none>"
+}
+
+// toResourceUnitValue accepts a resource name and quantity and returns the sanitized resource, the unit, and the value in the units.
+// Returns an empty string for resource and unit if there was a failure.
+func toResourceUnitValue(resourceName v1.ResourceName, quantity resource.Quantity) (resource string, unit string, value float64) {
+	resource = prom.SanitizeLabelName(string(resourceName))
+
+	switch resourceName {
+	case v1.ResourceCPU:
+		unit = "core"
+		value = float64(quantity.MilliValue()) / 1000
+		return
+
+	case v1.ResourceStorage:
+		fallthrough
+	case v1.ResourceEphemeralStorage:
+		fallthrough
+	case v1.ResourceMemory:
+		unit = "byte"
+		value = float64(quantity.Value())
+		return
+	case v1.ResourcePods:
+		unit = "integer"
+		value = float64(quantity.Value())
+		return
+	default:
+		if isHugePageResourceName(resourceName) || isAttachableVolumeResourceName(resourceName) {
+			unit = "byte"
+			value = float64(quantity.Value())
+			return
+		}
+
+		if isExtendedResourceName(resourceName) {
+			unit = "integer"
+			value = float64(quantity.Value())
+			return
+		}
+	}
+
+	resource = ""
+	unit = ""
+	value = 0.0
+	return
+}
+
+// isHugePageResourceName checks for a huge page container resource name
+func isHugePageResourceName(name v1.ResourceName) bool {
+	return strings.HasPrefix(string(name), v1.ResourceHugePagesPrefix)
+}
+
+// isAttachableVolumeResourceName checks for attached volume container resource name
+func isAttachableVolumeResourceName(name v1.ResourceName) bool {
+	return strings.HasPrefix(string(name), v1.ResourceAttachableVolumesPrefix)
+}
+
+// isExtendedResourceName checks for extended container resource name
+func isExtendedResourceName(name v1.ResourceName) bool {
+	if isNativeResource(name) || strings.HasPrefix(string(name), v1.DefaultResourceRequestsPrefix) {
+		return false
+	}
+	// Ensure it satisfies the rules in IsQualifiedName() after converted into quota resource name
+	nameForQuota := fmt.Sprintf("%s%s", v1.DefaultResourceRequestsPrefix, string(name))
+	if errs := validation.IsQualifiedName(nameForQuota); len(errs) != 0 {
+		return false
+	}
+	return true
+}
+
+// isNativeResource checks for a kubernetes.io/ prefixed resource name
+func isNativeResource(name v1.ResourceName) bool {
+	return !strings.Contains(string(name), "/") || isPrefixedNativeResource(name)
+}
+
+func isPrefixedNativeResource(name v1.ResourceName) bool {
+	return strings.Contains(string(name), v1.ResourceDefaultNamespacePrefix)
+}
+
+func failureReason(jc *batchv1.JobCondition, reason string) bool {
+	if jc == nil {
+		return false
+	}
+	return jc.Reason == reason
+}
+
+// boolFloat64 converts a boolean input into a 1 or 0
+func boolFloat64(b bool) float64 {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+// toStringPtr is used to create a new string pointer from iteration vars
+func toStringPtr(s string) *string { return &s }

pkg/metrics/namespacemetrics.go

@@ -0,0 +1,178 @@
+package metrics
+
+import (
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/prom"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+)
+
+//--------------------------------------------------------------------------
+//  KubecostNamespaceCollector
+//--------------------------------------------------------------------------
+
+// KubecostNamespaceCollector is a prometheus collector that generates namespace sourced metrics
+type KubecostNamespaceCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (nsac KubecostNamespaceCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_namespace_annotations", "namespace annotations", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (nsac KubecostNamespaceCollector) Collect(ch chan<- prometheus.Metric) {
+	namespaces := nsac.KubeClusterCache.GetAllNamespaces()
+	for _, namespace := range namespaces {
+		nsName := namespace.GetName()
+
+		labels, values := prom.KubeAnnotationsToLabels(namespace.Annotations)
+		if len(labels) > 0 {
+			m := newNamespaceAnnotationsMetric("kube_namespace_annotations", nsName, labels, values)
+			ch <- m
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  NamespaceAnnotationsMetric
+//--------------------------------------------------------------------------
+
+// NamespaceAnnotationsMetric is a prometheus.Metric used to encode namespace annotations
+type NamespaceAnnotationsMetric struct {
+	fqName      string
+	help        string
+	namespace   string
+	labelNames  []string
+	labelValues []string
+}
+
+// Creates a new NamespaceAnnotationsMetric, implementation of prometheus.Metric
+func newNamespaceAnnotationsMetric(fqname, namespace string, labelNames []string, labelValues []string) NamespaceAnnotationsMetric {
+	return NamespaceAnnotationsMetric{
+		fqName:      fqname,
+		help:        "kube_namespace_annotations Namespace Annotations",
+		namespace:   namespace,
+		labelNames:  labelNames,
+		labelValues: labelValues,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam NamespaceAnnotationsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": nam.namespace,
+	}
+	return prometheus.NewDesc(nam.fqName, nam.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (nam NamespaceAnnotationsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	for i := range nam.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &nam.labelNames[i],
+			Value: &nam.labelValues[i],
+		})
+	}
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("namespace"),
+		Value: &nam.namespace,
+	})
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeNamespaceCollector
+//--------------------------------------------------------------------------
+
+// KubeNamespaceCollector is a prometheus collector that generates namespace sourced metrics
+type KubeNamespaceCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (nsac KubeNamespaceCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_namespace_labels", "namespace labels", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (nsac KubeNamespaceCollector) Collect(ch chan<- prometheus.Metric) {
+	namespaces := nsac.KubeClusterCache.GetAllNamespaces()
+	for _, namespace := range namespaces {
+		nsName := namespace.GetName()
+
+		labels, values := prom.KubeLabelsToLabels(namespace.Labels)
+		if len(labels) > 0 {
+			m := newNamespaceAnnotationsMetric("kube_namespace_labels", nsName, labels, values)
+			ch <- m
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  NamespaceAnnotationsMetric
+//--------------------------------------------------------------------------
+
+// NamespaceAnnotationsMetric is a prometheus.Metric used to encode namespace annotations
+type KubeNamespaceLabelsMetric struct {
+	fqName      string
+	help        string
+	namespace   string
+	labelNames  []string
+	labelValues []string
+}
+
+// Creates a new KubeNamespaceLabelsMetric, implementation of prometheus.Metric
+func newKubeNamespaceLabelsMetric(fqname, namespace string, labelNames []string, labelValues []string) KubeNamespaceLabelsMetric {
+	return KubeNamespaceLabelsMetric{
+		namespace:   namespace,
+		fqName:      fqname,
+		labelNames:  labelNames,
+		labelValues: labelValues,
+		help:        "kube_namespace_labels Namespace Labels",
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubeNamespaceLabelsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": nam.namespace,
+	}
+	return prometheus.NewDesc(nam.fqName, nam.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (nam KubeNamespaceLabelsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	for i := range nam.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &nam.labelNames[i],
+			Value: &nam.labelValues[i],
+		})
+	}
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("namespace"),
+		Value: &nam.namespace,
+	})
+	m.Label = labels
+	return nil
+}

pkg/metrics/nodemetrics.go

@@ -0,0 +1,558 @@
+package metrics
+
+import (
+	"strings"
+
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/log"
+	"github.com/kubecost/cost-model/pkg/prom"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	v1 "k8s.io/api/core/v1"
+)
+
+var (
+	conditionStatuses = []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionFalse, v1.ConditionUnknown}
+)
+
+//--------------------------------------------------------------------------
+//  KubeNodeCollector
+//--------------------------------------------------------------------------
+
+// KubeNodeCollector is a prometheus collector that generates node sourced metrics.
+type KubeNodeCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (nsac KubeNodeCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_node_status_capacity", "Node resource capacity.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_node_status_capacity_memory_bytes", "node capacity memory bytes", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_node_status_capacity_cpu_cores", "node capacity cpu cores", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_node_status_allocatable", "The allocatable for different resources of a node that are available for scheduling.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_node_status_allocatable_cpu_cores", "The allocatable cpu cores.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_node_status_allocatable_memory_bytes", "The allocatable memory in bytes.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_node_labels", "all labels for each node prefixed with label_", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_node_status_condition", "The condition of a cluster node.", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (nsac KubeNodeCollector) Collect(ch chan<- prometheus.Metric) {
+	nodes := nsac.KubeClusterCache.GetAllNodes()
+	for _, node := range nodes {
+		nodeName := node.GetName()
+
+		// Node Capacity
+		for resourceName, quantity := range node.Status.Capacity {
+			resource, unit, value := toResourceUnitValue(resourceName, quantity)
+
+			// failed to parse the resource type
+			if resource == "" {
+				log.DedupedWarningf(5, "Failed to parse resource units and quantity for resource: %s", resourceName)
+				continue
+			}
+
+			// KSM v1 Emission
+			if resource == "cpu" {
+				ch <- newKubeNodeStatusCapacityCPUCoresMetric("kube_node_status_capacity_cpu_cores", nodeName, value)
+
+			}
+			if resource == "memory" {
+				ch <- newKubeNodeStatusCapacityMemoryBytesMetric("kube_node_status_capacity_memory_bytes", nodeName, value)
+			}
+
+			ch <- newKubeNodeStatusCapacityMetric("kube_node_status_capacity", nodeName, resource, unit, value)
+		}
+
+		// Node Allocatable Resources
+		for resourceName, quantity := range node.Status.Allocatable {
+			resource, unit, value := toResourceUnitValue(resourceName, quantity)
+
+			// failed to parse the resource type
+			if resource == "" {
+				log.DedupedWarningf(5, "Failed to parse resource units and quantity for resource: %s", resourceName)
+				continue
+			}
+
+			// KSM v1 Emission
+			if resource == "cpu" {
+				ch <- newKubeNodeStatusAllocatableCPUCoresMetric("kube_node_status_allocatable_cpu_cores", nodeName, value)
+
+			}
+			if resource == "memory" {
+				ch <- newKubeNodeStatusAllocatableMemoryBytesMetric("kube_node_status_allocatable_memory_bytes", nodeName, value)
+			}
+
+			ch <- newKubeNodeStatusAllocatableMetric("kube_node_status_allocatable", nodeName, resource, unit, value)
+		}
+
+		// node labels
+		labelNames, labelValues := prom.KubePrependQualifierToLabels(node.GetLabels(), "label_")
+		ch <- newKubeNodeLabelsMetric(nodeName, "kube_node_labels", labelNames, labelValues)
+
+		// kube_node_status_condition
+		// Collect node conditions and while default to false.
+		for _, c := range node.Status.Conditions {
+			conditions := getConditions(c.Status)
+
+			for _, cond := range conditions {
+				ch <- newKubeNodeStatusConditionMetric(nodeName, "kube_node_status_condition", string(c.Type), cond.status, cond.value)
+			}
+		}
+
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusCapacityMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusCapacityMetric is a prometheus.Metric
+type KubeNodeStatusCapacityMetric struct {
+	fqName   string
+	help     string
+	resource string
+	unit     string
+	node     string
+	value    float64
+}
+
+// Creates a new KubeNodeStatusCapacityMetric, implementation of prometheus.Metric
+func newKubeNodeStatusCapacityMetric(fqname, node, resource, unit string, value float64) KubeNodeStatusCapacityMetric {
+	return KubeNodeStatusCapacityMetric{
+		fqName:   fqname,
+		help:     "kube_node_status_capacity node capacity",
+		node:     node,
+		resource: resource,
+		unit:     unit,
+		value:    value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubeNodeStatusCapacityMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"node":     kpcrr.node,
+		"resource": kpcrr.resource,
+		"unit":     kpcrr.unit,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (kpcrr KubeNodeStatusCapacityMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("node"),
+			Value: &kpcrr.node,
+		},
+		{
+			Name:  toStringPtr("resource"),
+			Value: &kpcrr.resource,
+		},
+		{
+			Name:  toStringPtr("unit"),
+			Value: &kpcrr.unit,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusCapacityMemoryBytesMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusCapacityMemoryBytesMetric is a prometheus.Metric used to encode
+// a duplicate of the deprecated kube-state-metrics metric
+// kube_node_status_capacity_memory_bytes
+type KubeNodeStatusCapacityMemoryBytesMetric struct {
+	fqName string
+	help   string
+	bytes  float64
+	node   string
+}
+
+// Creates a new KubeNodeStatusCapacityMemoryBytesMetric, implementation of prometheus.Metric
+func newKubeNodeStatusCapacityMemoryBytesMetric(fqname string, node string, bytes float64) KubeNodeStatusCapacityMemoryBytesMetric {
+	return KubeNodeStatusCapacityMemoryBytesMetric{
+		fqName: fqname,
+		help:   "kube_node_status_capacity_memory_bytes Node Capacity Memory Bytes",
+		node:   node,
+		bytes:  bytes,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubeNodeStatusCapacityMemoryBytesMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{"node": nam.node}
+	return prometheus.NewDesc(nam.fqName, nam.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (nam KubeNodeStatusCapacityMemoryBytesMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &nam.bytes,
+	}
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("node"),
+			Value: &nam.node,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusCapacityCPUCoresMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusCapacityCPUCoresMetric is a prometheus.Metric used to encode
+// a duplicate of the deprecated kube-state-metrics metric
+// kube_node_status_capacity_memory_bytes
+type KubeNodeStatusCapacityCPUCoresMetric struct {
+	fqName string
+	help   string
+	cores  float64
+	node   string
+}
+
+// Creates a new KubeNodeStatusCapacityCPUCoresMetric, implementation of prometheus.Metric
+func newKubeNodeStatusCapacityCPUCoresMetric(fqname string, node string, cores float64) KubeNodeStatusCapacityCPUCoresMetric {
+	return KubeNodeStatusCapacityCPUCoresMetric{
+		fqName: fqname,
+		help:   "kube_node_status_capacity_cpu_cores Node Capacity CPU Cores",
+		cores:  cores,
+		node:   node,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubeNodeStatusCapacityCPUCoresMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{"node": nam.node}
+	return prometheus.NewDesc(nam.fqName, nam.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (nam KubeNodeStatusCapacityCPUCoresMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &nam.cores,
+	}
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("node"),
+			Value: &nam.node,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeLabelsCollector
+//--------------------------------------------------------------------------
+//
+// We use this to emit kube_node_labels with all of a node's labels, regardless
+// of the whitelist setting introduced in KSM v2. See
+// https://github.com/kubernetes/kube-state-metrics/issues/1270#issuecomment-712986441
+
+//--------------------------------------------------------------------------
+//  KubeNodeLabelsMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeLabelsMetric is a prometheus.Metric used to encode
+// a duplicate of the deprecated kube-state-metrics metric
+// kube_node_labels
+type KubeNodeLabelsMetric struct {
+	fqName      string
+	help        string
+	labelNames  []string
+	labelValues []string
+	node        string
+}
+
+// Creates a new KubeNodeLabelsMetric, implementation of prometheus.Metric
+func newKubeNodeLabelsMetric(node string, fqname string, labelNames []string, labelValues []string) KubeNodeLabelsMetric {
+	return KubeNodeLabelsMetric{
+		fqName:      fqname,
+		labelNames:  labelNames,
+		labelValues: labelValues,
+		help:        "kube_node_labels all labels for each node prefixed with label_",
+		node:        node,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubeNodeLabelsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"node": nam.node,
+	}
+	return prometheus.NewDesc(nam.fqName, nam.help, nam.labelNames, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (nam KubeNodeLabelsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	for i := range nam.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &nam.labelNames[i],
+			Value: &nam.labelValues[i],
+		})
+	}
+
+	nodeString := "node"
+	labels = append(labels, &dto.LabelPair{Name: &nodeString, Value: &nam.node})
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusConditionMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusConditionMetric
+type KubeNodeStatusConditionMetric struct {
+	fqName    string
+	help      string
+	node      string
+	condition string
+	status    string
+	value     float64
+}
+
+// Creates a new KubeNodeStatusConditionMetric, implementation of prometheus.Metric
+func newKubeNodeStatusConditionMetric(node, fqname, condition, status string, value float64) KubeNodeStatusConditionMetric {
+	return KubeNodeStatusConditionMetric{
+		fqName:    fqname,
+		help:      "kube_node_status_condition condition status for nodes",
+		node:      node,
+		condition: condition,
+		status:    status,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubeNodeStatusConditionMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"node":      nam.node,
+		"condition": nam.condition,
+		"status":    nam.status,
+	}
+	return prometheus.NewDesc(nam.fqName, nam.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (nam KubeNodeStatusConditionMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &nam.value,
+	}
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("node"),
+			Value: &nam.node,
+		},
+		{
+			Name:  toStringPtr("condition"),
+			Value: &nam.condition,
+		},
+		{
+			Name:  toStringPtr("status"),
+			Value: &nam.status,
+		},
+	}
+	return nil
+}
+
+// helper type for status condition reporting and metric rollup
+type statusCondition struct {
+	status string
+	value  float64
+}
+
+// retrieves the total status conditions and the comparison to the provided condition
+func getConditions(cs v1.ConditionStatus) []*statusCondition {
+	ms := make([]*statusCondition, len(conditionStatuses))
+
+	for i, status := range conditionStatuses {
+		ms[i] = &statusCondition{
+			status: strings.ToLower(string(status)),
+			value:  boolFloat64(cs == status),
+		}
+	}
+
+	return ms
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusAllocatableMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusAllocatableMetric is a prometheus.Metric
+type KubeNodeStatusAllocatableMetric struct {
+	fqName   string
+	help     string
+	resource string
+	unit     string
+	node     string
+	value    float64
+}
+
+// Creates a new KubeNodeStatusAllocatableMetric, implementation of prometheus.Metric
+func newKubeNodeStatusAllocatableMetric(fqname, node, resource, unit string, value float64) KubeNodeStatusAllocatableMetric {
+	return KubeNodeStatusAllocatableMetric{
+		fqName:   fqname,
+		help:     "kube_node_status_allocatable node allocatable",
+		node:     node,
+		resource: resource,
+		unit:     unit,
+		value:    value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubeNodeStatusAllocatableMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"node":     kpcrr.node,
+		"resource": kpcrr.resource,
+		"unit":     kpcrr.unit,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (kpcrr KubeNodeStatusAllocatableMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("node"),
+			Value: &kpcrr.node,
+		},
+		{
+			Name:  toStringPtr("resource"),
+			Value: &kpcrr.resource,
+		},
+		{
+			Name:  toStringPtr("unit"),
+			Value: &kpcrr.unit,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusAllocatableCPUCoresMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusAllocatableCPUCoresMetric is a prometheus.Metric
+type KubeNodeStatusAllocatableCPUCoresMetric struct {
+	fqName   string
+	help     string
+	resource string
+	unit     string
+	node     string
+	value    float64
+}
+
+// Creates a new KubeNodeStatusAllocatableCPUCoresMetric, implementation of prometheus.Metric
+func newKubeNodeStatusAllocatableCPUCoresMetric(fqname, node string, value float64) KubeNodeStatusAllocatableCPUCoresMetric {
+	return KubeNodeStatusAllocatableCPUCoresMetric{
+		fqName: fqname,
+		help:   "kube_node_status_allocatable_cpu_cores node allocatable cpu cores",
+		node:   node,
+		value:  value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubeNodeStatusAllocatableCPUCoresMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"node": kpcrr.node,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (kpcrr KubeNodeStatusAllocatableCPUCoresMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("node"),
+			Value: &kpcrr.node,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubeNodeStatusAllocatableMemoryBytesMetric
+//--------------------------------------------------------------------------
+
+// KubeNodeStatusAllocatableMemoryBytesMetric is a prometheus.Metric
+type KubeNodeStatusAllocatableMemoryBytesMetric struct {
+	fqName   string
+	help     string
+	resource string
+	unit     string
+	node     string
+	value    float64
+}
+
+// Creates a new KubeNodeStatusAllocatableMemoryBytesMetric, implementation of prometheus.Metric
+func newKubeNodeStatusAllocatableMemoryBytesMetric(fqname, node string, value float64) KubeNodeStatusAllocatableMemoryBytesMetric {
+	return KubeNodeStatusAllocatableMemoryBytesMetric{
+		fqName: fqname,
+		help:   "kube_node_status_allocatable_memory_bytes node allocatable memory in bytes",
+		node:   node,
+		value:  value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubeNodeStatusAllocatableMemoryBytesMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"node": kpcrr.node,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (kpcrr KubeNodeStatusAllocatableMemoryBytesMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("node"),
+			Value: &kpcrr.node,
+		},
+	}
+	return nil
+}

pkg/metrics/podmetrics.go

@@ -0,0 +1,1016 @@
+package metrics
+
+import (
+	"fmt"
+
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/log"
+	"github.com/kubecost/cost-model/pkg/prom"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	v1 "k8s.io/api/core/v1"
+)
+
+//--------------------------------------------------------------------------
+//  KubecostPodCollector
+//--------------------------------------------------------------------------
+
+// KubecostPodCollector is a prometheus collector that emits pod metrics
+type KubecostPodCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (kpmc KubecostPodCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_pod_annotations", "All annotations for each pod prefix with annotation_", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kpmc KubecostPodCollector) Collect(ch chan<- prometheus.Metric) {
+	pods := kpmc.KubeClusterCache.GetAllPods()
+	for _, pod := range pods {
+		podName := pod.GetName()
+		podNS := pod.GetNamespace()
+
+		// Pod Annotations
+		labels, values := prom.KubeAnnotationsToLabels(pod.Annotations)
+		if len(labels) > 0 {
+			ch <- newPodAnnotationMetric("kube_pod_annotations", podNS, podName, labels, values)
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubePodCollector
+//--------------------------------------------------------------------------
+
+// KubePodMetricCollector is a prometheus collector that emits pod metrics
+type KubePodCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (kpmc KubePodCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_pod_labels", "All labels for each pod prefixed with label_", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_owner", "Information about the Pod's owner", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_container_status_running", "Describes whether the container is currently in running state", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_container_status_terminated_reason", "Describes the reason the container is currently in terminated state.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_container_status_restarts_total", "The number of container restarts per container.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_container_resource_requests", "The number of requested resource by a container", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_container_resource_limits", "The number of requested limit resource by a container.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_container_resource_limits_cpu_cores", "The number of requested limit cpu core resource by a container.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_container_resource_limits_memory_bytes", "The number of requested limit memory resource by a container.", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_pod_status_phase", "The pods current phase.", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kpmc KubePodCollector) Collect(ch chan<- prometheus.Metric) {
+	pods := kpmc.KubeClusterCache.GetAllPods()
+	for _, pod := range pods {
+		podName := pod.GetName()
+		podNS := pod.GetNamespace()
+		podUID := string(pod.GetUID())
+		node := pod.Spec.NodeName
+		phase := pod.Status.Phase
+
+		// Pod Status Phase
+		if phase != "" {
+			phases := []struct {
+				v bool
+				n string
+			}{
+				{phase == v1.PodPending, string(v1.PodPending)},
+				{phase == v1.PodSucceeded, string(v1.PodSucceeded)},
+				{phase == v1.PodFailed, string(v1.PodFailed)},
+				{phase == v1.PodUnknown, string(v1.PodUnknown)},
+				{phase == v1.PodRunning, string(v1.PodRunning)},
+			}
+
+			for _, p := range phases {
+				ch <- newKubePodStatusPhaseMetric("kube_pod_status_phase", podNS, podName, podUID, p.n, boolFloat64(p.v))
+			}
+		}
+
+		// Pod Labels
+		labelNames, labelValues := prom.KubePrependQualifierToLabels(pod.GetLabels(), "label_")
+		ch <- newKubePodLabelsMetric("kube_pod_labels", podNS, podName, podUID, labelNames, labelValues)
+
+		// Owner References
+		for _, owner := range pod.OwnerReferences {
+			ch <- newKubePodOwnerMetric("kube_pod_owner", podNS, podName, owner.Name, owner.Kind, owner.Controller != nil)
+		}
+
+		// Container Status
+		for _, status := range pod.Status.ContainerStatuses {
+			ch <- newKubePodContainerStatusRestartsTotalMetric("kube_pod_container_status_restarts_total", podNS, podName, podUID, status.Name, float64(status.RestartCount))
+			if status.State.Running != nil {
+				ch <- newKubePodContainerStatusRunningMetric("kube_pod_container_status_running", podNS, podName, podUID, status.Name)
+			}
+
+			if status.State.Terminated != nil {
+				ch <- newKubePodContainerStatusTerminatedReasonMetric(
+					"kube_pod_container_status_terminated_reason",
+					podNS,
+					podName,
+					podUID,
+					status.Name,
+					status.State.Terminated.Reason)
+			}
+		}
+
+		for _, container := range pod.Spec.Containers {
+			// Requests
+			for resourceName, quantity := range container.Resources.Requests {
+				resource, unit, value := toResourceUnitValue(resourceName, quantity)
+
+				// failed to parse the resource type
+				if resource == "" {
+					log.DedupedWarningf(5, "Failed to parse resource units and quantity for resource: %s", resourceName)
+					continue
+				}
+
+				ch <- newKubePodContainerResourceRequestsMetric(
+					"kube_pod_container_resource_requests",
+					podNS,
+					podName,
+					podUID,
+					container.Name,
+					node,
+					resource,
+					unit,
+					value)
+			}
+
+			// Limits
+			for resourceName, quantity := range container.Resources.Limits {
+				resource, unit, value := toResourceUnitValue(resourceName, quantity)
+
+				// failed to parse the resource type
+				if resource == "" {
+					log.DedupedWarningf(5, "Failed to parse resource units and quantity for resource: %s", resourceName)
+					continue
+				}
+
+				// KSM v1 Emission
+				if resource == "cpu" {
+					ch <- newKubePodContainerResourceLimitsCPUCoresMetric(
+						"kube_pod_container_resource_limits_cpu_cores",
+						podNS,
+						podName,
+						podUID,
+						container.Name,
+						node,
+						value)
+				}
+				if resource == "memory" {
+					ch <- newKubePodContainerResourceLimitsMemoryBytesMetric(
+						"kube_pod_container_resource_limits_memory_bytes",
+						podNS,
+						podName,
+						podUID,
+						container.Name,
+						node,
+						value)
+				}
+
+				ch <- newKubePodContainerResourceLimitsMetric(
+					"kube_pod_container_resource_limits",
+					podNS,
+					podName,
+					podUID,
+					container.Name,
+					node,
+					resource,
+					unit,
+					value)
+			}
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  PodAnnotationsMetric
+//--------------------------------------------------------------------------
+
+// PodAnnotationsMetric is a prometheus.Metric used to encode namespace annotations
+type PodAnnotationsMetric struct {
+	fqName      string
+	help        string
+	namespace   string
+	pod         string
+	labelNames  []string
+	labelValues []string
+}
+
+// Creates a new PodAnnotationsMetric, implementation of prometheus.Metric
+func newPodAnnotationMetric(fqname, namespace, pod string, labelNames, labelValues []string) PodAnnotationsMetric {
+	return PodAnnotationsMetric{
+		fqName:      fqname,
+		help:        "kube_pod_annotations Pod Annotations",
+		namespace:   namespace,
+		pod:         pod,
+		labelNames:  labelNames,
+		labelValues: labelValues,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (pam PodAnnotationsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": pam.namespace,
+		"pod":       pam.pod,
+	}
+	return prometheus.NewDesc(pam.fqName, pam.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (pam PodAnnotationsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	for i := range pam.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &pam.labelNames[i],
+			Value: &pam.labelValues[i],
+		})
+	}
+	labels = append(labels,
+		&dto.LabelPair{
+			Name:  toStringPtr("namespace"),
+			Value: &pam.namespace,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("pod"),
+			Value: &pam.pod,
+		})
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodLabelsMetric
+//--------------------------------------------------------------------------
+
+// KubePodLabelsMetric is a prometheus.Metric used to encode
+// a duplicate of the deprecated kube-state-metrics metric
+// kube_pod_labels
+type KubePodLabelsMetric struct {
+	fqName      string
+	help        string
+	pod         string
+	namespace   string
+	uid         string
+	labelNames  []string
+	labelValues []string
+}
+
+// Creates a new KubePodLabelsMetric, implementation of prometheus.Metric
+func newKubePodLabelsMetric(fqname, namespace, pod, uid string, labelNames []string, labelValues []string) KubePodLabelsMetric {
+	return KubePodLabelsMetric{
+		fqName:      fqname,
+		help:        "kube_pod_labels all labels for each pod prefixed with label_",
+		pod:         pod,
+		namespace:   namespace,
+		uid:         uid,
+		labelNames:  labelNames,
+		labelValues: labelValues,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubePodLabelsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": nam.namespace,
+		"pod":       nam.pod,
+		"uid":       nam.uid,
+	}
+	return prometheus.NewDesc(nam.fqName, nam.help, nam.labelNames, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (nam KubePodLabelsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	for i := range nam.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &nam.labelNames[i],
+			Value: &nam.labelValues[i],
+		})
+	}
+
+	labels = append(labels,
+		&dto.LabelPair{
+			Name:  toStringPtr("pod"),
+			Value: &nam.pod,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("namespace"),
+			Value: &nam.namespace,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("uid"),
+			Value: &nam.uid,
+		},
+	)
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodContainerStatusRestartsTotalMetric
+//--------------------------------------------------------------------------
+
+// KubePodContainerStatusRestartsTotalMetric is a prometheus.Metric emitting container restarts metrics.
+type KubePodContainerStatusRestartsTotalMetric struct {
+	fqName    string
+	help      string
+	pod       string
+	namespace string
+	container string
+	uid       string
+	value     float64
+}
+
+// Creates a new KubePodContainerStatusRestartsTotalMetric, implementation of prometheus.Metric
+func newKubePodContainerStatusRestartsTotalMetric(fqname, namespace, pod, uid, container string, value float64) KubePodContainerStatusRestartsTotalMetric {
+	return KubePodContainerStatusRestartsTotalMetric{
+		fqName:    fqname,
+		help:      "kube_pod_container_status_restarts_total total container restarts",
+		pod:       pod,
+		namespace: namespace,
+		uid:       uid,
+		container: container,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcs KubePodContainerStatusRestartsTotalMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": kpcs.namespace,
+		"pod":       kpcs.pod,
+		"uid":       kpcs.uid,
+		"container": kpcs.container,
+	}
+	return prometheus.NewDesc(kpcs.fqName, kpcs.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (kpcs KubePodContainerStatusRestartsTotalMetric) Write(m *dto.Metric) error {
+	m.Counter = &dto.Counter{
+		Value: &kpcs.value,
+	}
+
+	var labels []*dto.LabelPair
+	labels = append(labels,
+		&dto.LabelPair{
+			Name:  toStringPtr("namespace"),
+			Value: &kpcs.namespace,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("pod"),
+			Value: &kpcs.pod,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("container"),
+			Value: &kpcs.container,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("uid"),
+			Value: &kpcs.uid,
+		},
+	)
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodContainerStatusTerminatedReasonMetric
+//--------------------------------------------------------------------------
+
+// KubePodContainerStatusTerminatedReasonMetric is a prometheus.Metric emitting container termination reasons.
+type KubePodContainerStatusTerminatedReasonMetric struct {
+	fqName    string
+	help      string
+	pod       string
+	namespace string
+	container string
+	uid       string
+	reason    string
+}
+
+// Creates a new KubePodContainerStatusRestartsTotalMetric, implementation of prometheus.Metric
+func newKubePodContainerStatusTerminatedReasonMetric(fqname, namespace, pod, uid, container, reason string) KubePodContainerStatusTerminatedReasonMetric {
+	return KubePodContainerStatusTerminatedReasonMetric{
+		fqName:    fqname,
+		help:      "kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.",
+		pod:       pod,
+		namespace: namespace,
+		uid:       uid,
+		container: container,
+		reason:    reason,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcs KubePodContainerStatusTerminatedReasonMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": kpcs.namespace,
+		"pod":       kpcs.pod,
+		"uid":       kpcs.uid,
+		"container": kpcs.container,
+		"reason":    kpcs.reason,
+	}
+	return prometheus.NewDesc(kpcs.fqName, kpcs.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (kpcs KubePodContainerStatusTerminatedReasonMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	labels = append(labels,
+		&dto.LabelPair{
+			Name:  toStringPtr("namespace"),
+			Value: &kpcs.namespace,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("pod"),
+			Value: &kpcs.pod,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("container"),
+			Value: &kpcs.container,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("uid"),
+			Value: &kpcs.uid,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("reason"),
+			Value: &kpcs.reason,
+		},
+	)
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodStatusPhaseMetric
+//--------------------------------------------------------------------------
+
+// KubePodStatusPhaseMetric is a prometheus.Metric emitting all phases for a pod
+type KubePodStatusPhaseMetric struct {
+	fqName    string
+	help      string
+	pod       string
+	namespace string
+	uid       string
+	phase     string
+	value     float64
+}
+
+// Creates a new KubePodContainerStatusRestartsTotalMetric, implementation of prometheus.Metric
+func newKubePodStatusPhaseMetric(fqname, namespace, pod, uid, phase string, value float64) KubePodStatusPhaseMetric {
+	return KubePodStatusPhaseMetric{
+		fqName:    fqname,
+		help:      "kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.",
+		pod:       pod,
+		namespace: namespace,
+		uid:       uid,
+		phase:     phase,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcs KubePodStatusPhaseMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": kpcs.namespace,
+		"pod":       kpcs.pod,
+		"uid":       kpcs.uid,
+		"phase":     kpcs.phase,
+	}
+	return prometheus.NewDesc(kpcs.fqName, kpcs.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (kpcs KubePodStatusPhaseMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcs.value,
+	}
+
+	var labels []*dto.LabelPair
+	labels = append(labels,
+		&dto.LabelPair{
+			Name:  toStringPtr("namespace"),
+			Value: &kpcs.namespace,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("pod"),
+			Value: &kpcs.pod,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("uid"),
+			Value: &kpcs.uid,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("phase"),
+			Value: &kpcs.phase,
+		},
+	)
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodContainerStatusRunningMetric
+//--------------------------------------------------------------------------
+
+// KubePodLabelsMetric is a prometheus.Metric used to encode
+// a duplicate of the deprecated kube-state-metrics metric
+// kube_pod_labels
+type KubePodContainerStatusRunningMetric struct {
+	fqName    string
+	help      string
+	pod       string
+	namespace string
+	container string
+	uid       string
+}
+
+// Creates a new KubePodContainerStatusRunningMetric, implementation of prometheus.Metric
+func newKubePodContainerStatusRunningMetric(fqname, namespace, pod, uid, container string) KubePodContainerStatusRunningMetric {
+	return KubePodContainerStatusRunningMetric{
+		fqName:    fqname,
+		help:      "kube_pod_container_status_running pods container status",
+		pod:       pod,
+		namespace: namespace,
+		uid:       uid,
+		container: container,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcs KubePodContainerStatusRunningMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": kpcs.namespace,
+		"pod":       kpcs.pod,
+		"uid":       kpcs.uid,
+		"container": kpcs.container,
+	}
+	return prometheus.NewDesc(kpcs.fqName, kpcs.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpcs KubePodContainerStatusRunningMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	labels = append(labels,
+		&dto.LabelPair{
+			Name:  toStringPtr("namespace"),
+			Value: &kpcs.namespace,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("pod"),
+			Value: &kpcs.pod,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("container"),
+			Value: &kpcs.container,
+		},
+		&dto.LabelPair{
+			Name:  toStringPtr("uid"),
+			Value: &kpcs.uid,
+		},
+	)
+	m.Label = labels
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodContainerResourceRequestMetric
+//--------------------------------------------------------------------------
+
+// KubePodContainerResourceRequestsMetric is a prometheus.Metric
+type KubePodContainerResourceRequestsMetric struct {
+	fqName    string
+	help      string
+	pod       string
+	namespace string
+	container string
+	uid       string
+	resource  string
+	unit      string
+	node      string
+	value     float64
+}
+
+// Creates a new newKubePodContainerResourceRequestsMetric, implementation of prometheus.Metric
+func newKubePodContainerResourceRequestsMetric(fqname, namespace, pod, uid, container, node, resource, unit string, value float64) KubePodContainerResourceRequestsMetric {
+	return KubePodContainerResourceRequestsMetric{
+		fqName:    fqname,
+		help:      "kube_pod_container_resource_requests pods container resource requests",
+		pod:       pod,
+		namespace: namespace,
+		uid:       uid,
+		container: container,
+		node:      node,
+		resource:  resource,
+		unit:      unit,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubePodContainerResourceRequestsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": kpcrr.namespace,
+		"pod":       kpcrr.pod,
+		"uid":       kpcrr.uid,
+		"container": kpcrr.container,
+		"node":      kpcrr.node,
+		"resource":  kpcrr.resource,
+		"unit":      kpcrr.unit,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpcrr KubePodContainerResourceRequestsMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kpcrr.namespace,
+		},
+		{
+			Name:  toStringPtr("pod"),
+			Value: &kpcrr.pod,
+		},
+		{
+			Name:  toStringPtr("container"),
+			Value: &kpcrr.container,
+		},
+		{
+			Name:  toStringPtr("uid"),
+			Value: &kpcrr.uid,
+		},
+		{
+			Name:  toStringPtr("node"),
+			Value: &kpcrr.node,
+		},
+		{
+			Name:  toStringPtr("resource"),
+			Value: &kpcrr.resource,
+		},
+		{
+			Name:  toStringPtr("unit"),
+			Value: &kpcrr.unit,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodContainerResourceLimitsMetric
+//--------------------------------------------------------------------------
+
+// KubePodContainerResourceLimitsMetric is a prometheus.Metric
+type KubePodContainerResourceLimitsMetric struct {
+	fqName    string
+	help      string
+	pod       string
+	namespace string
+	container string
+	uid       string
+	resource  string
+	unit      string
+	node      string
+	value     float64
+}
+
+// Creates a new KubePodContainerResourceLimitsMetric, implementation of prometheus.Metric
+func newKubePodContainerResourceLimitsMetric(fqname, namespace, pod, uid, container, node, resource, unit string, value float64) KubePodContainerResourceLimitsMetric {
+	return KubePodContainerResourceLimitsMetric{
+		fqName:    fqname,
+		help:      "kube_pod_container_resource_limits pods container resource limits",
+		pod:       pod,
+		namespace: namespace,
+		uid:       uid,
+		container: container,
+		node:      node,
+		resource:  resource,
+		unit:      unit,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubePodContainerResourceLimitsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": kpcrr.namespace,
+		"pod":       kpcrr.pod,
+		"uid":       kpcrr.uid,
+		"container": kpcrr.container,
+		"node":      kpcrr.node,
+		"resource":  kpcrr.resource,
+		"unit":      kpcrr.unit,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpcrr KubePodContainerResourceLimitsMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kpcrr.namespace,
+		},
+		{
+			Name:  toStringPtr("pod"),
+			Value: &kpcrr.pod,
+		},
+		{
+			Name:  toStringPtr("container"),
+			Value: &kpcrr.container,
+		},
+		{
+			Name:  toStringPtr("uid"),
+			Value: &kpcrr.uid,
+		},
+		{
+			Name:  toStringPtr("node"),
+			Value: &kpcrr.node,
+		},
+		{
+			Name:  toStringPtr("resource"),
+			Value: &kpcrr.resource,
+		},
+		{
+			Name:  toStringPtr("unit"),
+			Value: &kpcrr.unit,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodContainerResourceLimitsCPUCoresMetric (KSM v1)
+//--------------------------------------------------------------------------
+
+// KubePodContainerResourceLimitsCPUCoresMetric is a prometheus.Metric
+type KubePodContainerResourceLimitsCPUCoresMetric struct {
+	fqName    string
+	help      string
+	pod       string
+	namespace string
+	container string
+	uid       string
+	node      string
+	value     float64
+}
+
+// Creates a new KubePodContainerResourceLimitsMetric, implementation of prometheus.Metric
+func newKubePodContainerResourceLimitsCPUCoresMetric(fqname, namespace, pod, uid, container, node string, value float64) KubePodContainerResourceLimitsCPUCoresMetric {
+	return KubePodContainerResourceLimitsCPUCoresMetric{
+		fqName:    fqname,
+		help:      "kube_pod_container_resource_limits_cpu_cores pods container cpu cores resource limits",
+		pod:       pod,
+		namespace: namespace,
+		uid:       uid,
+		container: container,
+		node:      node,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubePodContainerResourceLimitsCPUCoresMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": kpcrr.namespace,
+		"pod":       kpcrr.pod,
+		"uid":       kpcrr.uid,
+		"container": kpcrr.container,
+		"node":      kpcrr.node,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpcrr KubePodContainerResourceLimitsCPUCoresMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kpcrr.namespace,
+		},
+		{
+			Name:  toStringPtr("pod"),
+			Value: &kpcrr.pod,
+		},
+		{
+			Name:  toStringPtr("container"),
+			Value: &kpcrr.container,
+		},
+		{
+			Name:  toStringPtr("uid"),
+			Value: &kpcrr.uid,
+		},
+		{
+			Name:  toStringPtr("node"),
+			Value: &kpcrr.node,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodContainerResourceLimitsMemoryBytesMetric (KSM v1)
+//--------------------------------------------------------------------------
+
+// KubePodContainerResourceLimitsMemoryBytesMetric is a prometheus.Metric
+type KubePodContainerResourceLimitsMemoryBytesMetric struct {
+	fqName    string
+	help      string
+	pod       string
+	namespace string
+	container string
+	uid       string
+	node      string
+	value     float64
+}
+
+// Creates a new KubePodContainerResourceLimitsMemoryBytesMetric, implementation of prometheus.Metric
+func newKubePodContainerResourceLimitsMemoryBytesMetric(fqname, namespace, pod, uid, container, node string, value float64) KubePodContainerResourceLimitsMemoryBytesMetric {
+	return KubePodContainerResourceLimitsMemoryBytesMetric{
+		fqName:    fqname,
+		help:      "kube_pod_container_resource_limits_memory_bytes pods container memory bytes resource limits",
+		pod:       pod,
+		namespace: namespace,
+		uid:       uid,
+		container: container,
+		node:      node,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubePodContainerResourceLimitsMemoryBytesMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": kpcrr.namespace,
+		"pod":       kpcrr.pod,
+		"uid":       kpcrr.uid,
+		"container": kpcrr.container,
+		"node":      kpcrr.node,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpcrr KubePodContainerResourceLimitsMemoryBytesMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kpcrr.namespace,
+		},
+		{
+			Name:  toStringPtr("pod"),
+			Value: &kpcrr.pod,
+		},
+		{
+			Name:  toStringPtr("container"),
+			Value: &kpcrr.container,
+		},
+		{
+			Name:  toStringPtr("uid"),
+			Value: &kpcrr.uid,
+		},
+		{
+			Name:  toStringPtr("node"),
+			Value: &kpcrr.node,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePodOwnerMetric
+//--------------------------------------------------------------------------
+
+// KubePodOwnerMetric is a prometheus.Metric
+type KubePodOwnerMetric struct {
+	fqName            string
+	help              string
+	namespace         string
+	pod               string
+	ownerIsController bool
+	ownerName         string
+	ownerKind         string
+}
+
+// Creates a new KubePodOwnerMetric, implementation of prometheus.Metric
+func newKubePodOwnerMetric(fqname, namespace, pod, ownerName, ownerKind string, ownerIsController bool) KubePodOwnerMetric {
+	return KubePodOwnerMetric{
+		fqName:            fqname,
+		help:              "kube_pod_owner Information about the Pod's owner",
+		namespace:         namespace,
+		pod:               pod,
+		ownerName:         ownerName,
+		ownerKind:         ownerKind,
+		ownerIsController: ownerIsController,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpo KubePodOwnerMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace":           kpo.namespace,
+		"pod":                 kpo.pod,
+		"owner_name":          kpo.ownerName,
+		"owner_kind":          kpo.ownerKind,
+		"owner_is_controller": fmt.Sprintf("%t", kpo.ownerIsController),
+	}
+	return prometheus.NewDesc(kpo.fqName, kpo.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpo KubePodOwnerMetric) Write(m *dto.Metric) error {
+	v := float64(1.0)
+	m.Gauge = &dto.Gauge{
+		Value: &v,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kpo.namespace,
+		},
+		{
+			Name:  toStringPtr("pod"),
+			Value: &kpo.pod,
+		},
+		{
+			Name:  toStringPtr("owner_name"),
+			Value: &kpo.ownerName,
+		},
+		{
+			Name:  toStringPtr("owner_kind"),
+			Value: &kpo.ownerKind,
+		},
+		{
+			Name:  toStringPtr("owner_is_controller"),
+			Value: toStringPtr(fmt.Sprintf("%t", kpo.ownerIsController)),
+		},
+	}
+	return nil
+}

pkg/metrics/pvcmetrics.go

@@ -0,0 +1,159 @@
+package metrics
+
+import (
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	v1 "k8s.io/api/core/v1"
+)
+
+//--------------------------------------------------------------------------
+//  KubePVCCollector
+//--------------------------------------------------------------------------
+
+// KubePVCCollector is a prometheus collector that generates pvc sourced metrics
+type KubePVCCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics collected by this Collector.
+func (kpvc KubePVCCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_persistentvolumeclaim_resource_requests_storage_bytes", "The pvc storage resource requests in bytes", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_persistentvolumeclaim_info", "The pvc storage resource requests in bytes", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kpvc KubePVCCollector) Collect(ch chan<- prometheus.Metric) {
+	pvcs := kpvc.KubeClusterCache.GetAllPersistentVolumeClaims()
+	for _, pvc := range pvcs {
+		storageClass := getPersistentVolumeClaimClass(pvc)
+		volume := pvc.Spec.VolumeName
+
+		ch <- newKubePVCInfoMetric("kube_persistentvolumeclaim_info", pvc.Name, pvc.Namespace, volume, storageClass)
+
+		if storage, ok := pvc.Spec.Resources.Requests[v1.ResourceStorage]; ok {
+			ch <- newKubePVCResourceRequestsStorageBytesMetric("kube_persistentvolumeclaim_resource_requests_storage_bytes", pvc.Name, pvc.Namespace, float64(storage.Value()))
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubePVCResourceRequestsStorageBytesMetric
+//--------------------------------------------------------------------------
+
+// KubePVCResourceRequestsStorageBytesMetric is a prometheus.Metric
+type KubePVCResourceRequestsStorageBytesMetric struct {
+	fqName    string
+	help      string
+	namespace string
+	pvc       string
+	value     float64
+}
+
+// Creates a new KubePVCResourceRequestsStorageBytesMetric, implementation of prometheus.Metric
+func newKubePVCResourceRequestsStorageBytesMetric(fqname, pvc, namespace string, value float64) KubePVCResourceRequestsStorageBytesMetric {
+	return KubePVCResourceRequestsStorageBytesMetric{
+		fqName:    fqname,
+		help:      "kube_persistentvolumeclaim_resource_requests_storage_bytes pvc storage resource requests in bytes",
+		pvc:       pvc,
+		namespace: namespace,
+		value:     value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpvcrr KubePVCResourceRequestsStorageBytesMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"persistentvolumeclaim": kpvcrr.pvc,
+		"namespace":             kpvcrr.namespace,
+	}
+	return prometheus.NewDesc(kpvcrr.fqName, kpvcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpvcrr KubePVCResourceRequestsStorageBytesMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpvcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("persistentvolumeclaim"),
+			Value: &kpvcrr.pvc,
+		},
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kpvcrr.namespace,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePVCInfoMetric
+//--------------------------------------------------------------------------
+
+// KubePVCInfoMetric is a prometheus.Metric
+type KubePVCInfoMetric struct {
+	fqName       string
+	help         string
+	namespace    string
+	pvc          string
+	storageclass string
+	volume       string
+}
+
+// Creates a new KubePVCInfoMetric, implementation of prometheus.Metric
+func newKubePVCInfoMetric(fqname, pvc, namespace, storageclass, volume string) KubePVCInfoMetric {
+	return KubePVCInfoMetric{
+		fqName:       fqname,
+		help:         "kube_persistentvolumeclaim_info pvc storage resource requests in bytes",
+		pvc:          pvc,
+		namespace:    namespace,
+		storageclass: storageclass,
+		volume:       volume,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpvcrr KubePVCInfoMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"persistentvolumeclaim": kpvcrr.pvc,
+		"namespace":             kpvcrr.namespace,
+		"storageclass":          kpvcrr.storageclass,
+		"volumename":            kpvcrr.volume,
+	}
+	return prometheus.NewDesc(kpvcrr.fqName, kpvcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpvci KubePVCInfoMetric) Write(m *dto.Metric) error {
+	v := float64(1.0)
+	m.Gauge = &dto.Gauge{
+		Value: &v,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("namespace"),
+			Value: &kpvci.namespace,
+		},
+		{
+			Name:  toStringPtr("persistentvolumeclaim"),
+			Value: &kpvci.pvc,
+		},
+		{
+			Name:  toStringPtr("storageclass"),
+			Value: &kpvci.storageclass,
+		},
+		{
+			Name:  toStringPtr("volumename"),
+			Value: &kpvci.volume,
+		},
+	}
+	return nil
+}

pkg/metrics/pvmetrics.go

@@ -0,0 +1,154 @@
+package metrics
+
+import (
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	v1 "k8s.io/api/core/v1"
+)
+
+//--------------------------------------------------------------------------
+//  KubePVCollector
+//--------------------------------------------------------------------------
+
+// KubePVCollector is a prometheus collector that generates PV metrics
+type KubePVCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (kpvcb KubePVCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_persistentvolume_capacity_bytes", "The pv storage capacity in bytes", []string{}, nil)
+	ch <- prometheus.NewDesc("kube_persistentvolume_status_phase", "The phase indicates if a volume is available, bound to a claim, or released by a claim.", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kpvcb KubePVCollector) Collect(ch chan<- prometheus.Metric) {
+	pvs := kpvcb.KubeClusterCache.GetAllPersistentVolumes()
+	for _, pv := range pvs {
+		phase := pv.Status.Phase
+		if phase != "" {
+			phases := []struct {
+				v bool
+				n string
+			}{
+				{phase == v1.VolumePending, string(v1.VolumePending)},
+				{phase == v1.VolumeAvailable, string(v1.VolumeAvailable)},
+				{phase == v1.VolumeBound, string(v1.VolumeBound)},
+				{phase == v1.VolumeReleased, string(v1.VolumeReleased)},
+				{phase == v1.VolumeFailed, string(v1.VolumeFailed)},
+			}
+
+			for _, p := range phases {
+				ch <- newKubePVStatusPhaseMetric("kube_persistentvolume_status_phase", pv.Name, p.n, boolFloat64(p.v))
+			}
+		}
+
+		storage := pv.Spec.Capacity[v1.ResourceStorage]
+		m := newKubePVCapacityBytesMetric("kube_persistentvolume_capacity_bytes", pv.Name, float64(storage.Value()))
+
+		ch <- m
+	}
+}
+
+//--------------------------------------------------------------------------
+//  KubePVCapacityBytesMetric
+//--------------------------------------------------------------------------
+
+// KubePVCapacityBytesMetric is a prometheus.Metric
+type KubePVCapacityBytesMetric struct {
+	fqName string
+	help   string
+	pv     string
+	value  float64
+}
+
+// Creates a new KubePVCapacityBytesMetric, implementation of prometheus.Metric
+func newKubePVCapacityBytesMetric(fqname, pv string, value float64) KubePVCapacityBytesMetric {
+	return KubePVCapacityBytesMetric{
+		fqName: fqname,
+		help:   "kube_persistentvolume_capacity_bytes pv storage capacity in bytes",
+		pv:     pv,
+		value:  value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubePVCapacityBytesMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"persistentvolume": kpcrr.pv,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpcrr KubePVCapacityBytesMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("persistentvolume"),
+			Value: &kpcrr.pv,
+		},
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------
+//  KubePVStatusPhaseMetric
+//--------------------------------------------------------------------------
+
+// KubePVStatusPhaseMetric is a prometheus.Metric
+type KubePVStatusPhaseMetric struct {
+	fqName string
+	help   string
+	pv     string
+	phase  string
+	value  float64
+}
+
+// Creates a new KubePVCapacityBytesMetric, implementation of prometheus.Metric
+func newKubePVStatusPhaseMetric(fqname, pv, phase string, value float64) KubePVStatusPhaseMetric {
+	return KubePVStatusPhaseMetric{
+		fqName: fqname,
+		help:   "kube_persistentvolume_status_phase pv status phase",
+		pv:     pv,
+		phase:  phase,
+		value:  value,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (kpcrr KubePVStatusPhaseMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"persistentvolume": kpcrr.pv,
+		"phase":            kpcrr.phase,
+	}
+	return prometheus.NewDesc(kpcrr.fqName, kpcrr.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (kpcrr KubePVStatusPhaseMetric) Write(m *dto.Metric) error {
+	m.Gauge = &dto.Gauge{
+		Value: &kpcrr.value,
+	}
+
+	m.Label = []*dto.LabelPair{
+		{
+			Name:  toStringPtr("persistentvolume"),
+			Value: &kpcrr.pv,
+		},
+		{
+			Name:  toStringPtr("phase"),
+			Value: &kpcrr.phase,
+		},
+	}
+	return nil
+}

pkg/metrics/servicemetrics.go

@@ -0,0 +1,101 @@
+package metrics
+
+import (
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/prom"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+)
+
+//--------------------------------------------------------------------------
+//  KubecostServiceCollector
+//--------------------------------------------------------------------------
+
+// KubecostServiceCollector is a prometheus collector that generates service sourced metrics.
+type KubecostServiceCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (sc KubecostServiceCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("service_selector_labels", "service selector labels", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (sc KubecostServiceCollector) Collect(ch chan<- prometheus.Metric) {
+	svcs := sc.KubeClusterCache.GetAllServices()
+	for _, svc := range svcs {
+		serviceName := svc.GetName()
+		serviceNS := svc.GetNamespace()
+
+		labels, values := prom.KubeLabelsToLabels(svc.Spec.Selector)
+		if len(labels) > 0 {
+			m := newServiceSelectorLabelsMetric(serviceName, serviceNS, "service_selector_labels", labels, values)
+			ch <- m
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  ServiceSelectorLabelsMetric
+//--------------------------------------------------------------------------
+
+// ServiceSelectorLabelsMetric is a prometheus.Metric used to encode service selector labels
+type ServiceSelectorLabelsMetric struct {
+	fqName      string
+	help        string
+	labelNames  []string
+	labelValues []string
+	serviceName string
+	namespace   string
+}
+
+// Creates a new ServiceMetric, implementation of prometheus.Metric
+func newServiceSelectorLabelsMetric(name, namespace, fqname string, labelNames, labelvalues []string) ServiceSelectorLabelsMetric {
+	return ServiceSelectorLabelsMetric{
+		fqName:      fqname,
+		labelNames:  labelNames,
+		labelValues: labelvalues,
+		help:        "service_selector_labels Service Selector Labels",
+		serviceName: name,
+		namespace:   namespace,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (s ServiceSelectorLabelsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"service":   s.serviceName,
+		"namespace": s.namespace,
+	}
+	return prometheus.NewDesc(s.fqName, s.help, s.labelNames, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (s ServiceSelectorLabelsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+	var labels []*dto.LabelPair
+	for i := range s.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &s.labelNames[i],
+			Value: &s.labelValues[i],
+		})
+	}
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("namespace"),
+		Value: &s.namespace,
+	})
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("service"),
+		Value: &s.serviceName,
+	})
+	m.Label = labels
+	return nil
+}

pkg/metrics/statefulsetmetrics.go

@@ -0,0 +1,101 @@
+package metrics
+
+import (
+	"github.com/kubecost/cost-model/pkg/clustercache"
+	"github.com/kubecost/cost-model/pkg/prom"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+)
+
+//--------------------------------------------------------------------------
+//  KubecostStatefulsetCollector
+//--------------------------------------------------------------------------
+
+// StatefulsetCollector is a prometheus collector that generates StatefulsetMetrics
+type KubecostStatefulsetCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (sc KubecostStatefulsetCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("statefulSet_match_labels", "statfulSet match labels", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (sc KubecostStatefulsetCollector) Collect(ch chan<- prometheus.Metric) {
+	ds := sc.KubeClusterCache.GetAllStatefulSets()
+	for _, statefulset := range ds {
+		statefulsetName := statefulset.GetName()
+		statefulsetNS := statefulset.GetNamespace()
+
+		labels, values := prom.KubeLabelsToLabels(statefulset.Spec.Selector.MatchLabels)
+		if len(labels) > 0 {
+			m := newStatefulsetMatchLabelsMetric(statefulsetName, statefulsetNS, "statefulSet_match_labels", labels, values)
+			ch <- m
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  StatefulsetMatchLabelsMetric
+//--------------------------------------------------------------------------
+
+// StatefulsetMetric is a prometheus.Metric used to encode statefulset match labels
+type StatefulsetMatchLabelsMetric struct {
+	fqName          string
+	help            string
+	labelNames      []string
+	labelValues     []string
+	statefulsetName string
+	namespace       string
+}
+
+// Creates a new StatefulsetMetric, implementation of prometheus.Metric
+func newStatefulsetMatchLabelsMetric(name, namespace, fqname string, labelNames, labelvalues []string) StatefulsetMatchLabelsMetric {
+	return StatefulsetMatchLabelsMetric{
+		fqName:          fqname,
+		labelNames:      labelNames,
+		labelValues:     labelvalues,
+		help:            "statefulSet_match_labels StatefulSet Match Labels",
+		statefulsetName: name,
+		namespace:       namespace,
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (s StatefulsetMatchLabelsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"statefulSet": s.statefulsetName,
+		"namespace":   s.namespace,
+	}
+	return prometheus.NewDesc(s.fqName, s.help, s.labelNames, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data
+// transmission object.
+func (s StatefulsetMatchLabelsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+	var labels []*dto.LabelPair
+	for i := range s.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &s.labelNames[i],
+			Value: &s.labelValues[i],
+		})
+	}
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("namespace"),
+		Value: &s.namespace,
+	})
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("statefulSet"),
+		Value: &s.statefulsetName,
+	})
+	m.Label = labels
+	return nil
+}

pkg/prom/error.go

@@ -82,7 +82,7 @@ type ErrorsAndWarningStrings struct {
 }
 
 // QueryErrorCollector is used to collect prometheus query errors and warnings, and also meets the
-// Error
+// Error interface
 type QueryErrorCollector struct {
 	m        sync.RWMutex
 	errors   []*QueryError

pkg/prom/metrics_test.go

@@ -0,0 +1,95 @@
+package prom
+
+import (
+	"fmt"
+	"testing"
+)
+
+func checkSlice(s1, s2 []string) error {
+	if len(s1) != len(s2) {
+		return fmt.Errorf("len(s1) [%d] != len(s2) [%d]", len(s1), len(s2))
+	}
+
+	for i := 0; i < len(s1); i++ {
+		if s1[i] != s2[i] {
+			return fmt.Errorf("At Index: %d. Different Values %s (s1) != %s (s2)", i, s1[i], s2[i])
+		}
+	}
+	return nil
+}
+
+func TestEmptyKubeLabelsToPromLabels(t *testing.T) {
+	labels, values := KubeLabelsToLabels(nil)
+
+	if len(labels) != 0 {
+		t.Errorf("Labels length is non-zero\n")
+	}
+	if len(values) != 0 {
+		t.Errorf("Values length is non-zero\n")
+	}
+
+	labels, values = KubeLabelsToLabels(map[string]string{})
+
+	if len(labels) != 0 {
+		t.Errorf("Labels length is non-zero\n")
+	}
+	if len(values) != 0 {
+		t.Errorf("Values length is non-zero\n")
+	}
+}
+
+func TestKubeLabelsToPromLabels(t *testing.T) {
+	var expectedLabels []string = []string{
+		"label_app",
+		"label_chart",
+		"label_control_plane",
+		"label_gatekeeper_sh_operation",
+		"label_heritage",
+		"label_pod_template_hash",
+		"label_release",
+	}
+	var expectedValues []string = []string{
+		"gatekeeper",
+		"gatekeeper",
+		"audit-controller",
+		"audit",
+		"Helm",
+		"5599859cd4",
+		"gatekeeper",
+	}
+
+	kubeLabels := map[string]string{
+		"app":                     "gatekeeper",
+		"chart":                   "gatekeeper",
+		"control-plane":           "audit-controller",
+		"gatekeeper.sh/operation": "audit",
+		"heritage":                "Helm",
+		"pod-template-hash":       "5599859cd4",
+		"release":                 "gatekeeper",
+	}
+
+	labels, values := KubePrependQualifierToLabels(kubeLabels, "label_")
+	l2, v2 := KubeLabelsToLabels(kubeLabels)
+
+	// Check to make sure we get expected labels and values returned
+	err := checkSlice(labels, expectedLabels)
+	if err != nil {
+		t.Errorf("%s", err)
+	}
+	err = checkSlice(values, expectedValues)
+	if err != nil {
+		t.Errorf("%s", err)
+	}
+
+	// Check to make sure the helper function returns what the prependqualifier func
+	// returns
+	err = checkSlice(l2, labels)
+	if err != nil {
+		t.Errorf("%s", err)
+	}
+
+	err = checkSlice(v2, values)
+	if err != nil {
+		t.Errorf("%s", err)
+	}
+}

pkg/util/retry/retry_test.go

@@ -13,6 +13,7 @@ type Obj struct {
 }
 
 func TestPtrSliceRetry(t *testing.T) {
+	t.Parallel()
 	const Expected uint64 = 3
 
 	var count uint64 = 0
@@ -42,6 +43,7 @@ func TestPtrSliceRetry(t *testing.T) {
 }
 
 func TestSuccessRetry(t *testing.T) {
+	t.Parallel()
 	const Expected uint64 = 3
 
 	var count uint64 = 0
@@ -64,6 +66,7 @@ func TestSuccessRetry(t *testing.T) {
 }
 
 func TestFailRetry(t *testing.T) {
+	t.Parallel()
 	const Expected uint64 = 5
 
 	expectedError := fmt.Sprintf("Failed: %d", Expected)
@@ -86,6 +89,8 @@ func TestFailRetry(t *testing.T) {
 }
 
 func TestCancelRetry(t *testing.T) {
+	t.Parallel()
+
 	const Expected uint64 = 5
 
 	var count uint64 = 0