Normalized pod metric objects to make collapsing later more straight-forward. Added a unit test for KubeLabelsToLabels utility. Added kube_namespace_labels emission. Separated the pod annotation emission from the default ksm emission

pkg/metrics/kubemetrics.go

@@ -59,8 +59,14 @@ func InitKubeMetrics(clusterCache clustercache.ClusterCache, opts *KubeMetricsOp
 			})
 		}
 
+		if opts.EmitPodAnnotations {
+			prometheus.MustRegister(KubecostPodCollector{
+				KubeClusterCache: clusterCache,
+			})
+		}
+
 		if opts.EmitNamespaceAnnotations {
-			prometheus.MustRegister(KubeNamespaceCollector{
+			prometheus.MustRegister(KubecostNamespaceCollector{
 				KubeClusterCache: clusterCache,
 			})
 		}
@@ -69,12 +75,14 @@ func InitKubeMetrics(clusterCache clustercache.ClusterCache, opts *KubeMetricsOp
 			prometheus.MustRegister(KubeNodeCollector{
 				KubeClusterCache: clusterCache,
 			})
+			prometheus.MustRegister(KubeNamespaceCollector{
+				KubeClusterCache: clusterCache,
+			})
 			prometheus.MustRegister(KubeDeploymentCollector{
 				KubeClusterCache: clusterCache,
 			})
 			prometheus.MustRegister(KubePodCollector{
-				KubeClusterCache:   clusterCache,
-				emitPodAnnotations: opts.EmitPodAnnotations,
+				KubeClusterCache: clusterCache,
 			})
 			prometheus.MustRegister(KubePVCollector{
 				KubeClusterCache: clusterCache,

pkg/metrics/namespacemetrics.go

@@ -9,29 +9,29 @@ import (
 )
 
 //--------------------------------------------------------------------------
-//  KubeNamespaceCollector
+//  KubecostNamespaceCollector
 //--------------------------------------------------------------------------
 
-// KubeNamespaceCollector is a prometheus collector that generates namespace sourced metrics
-type KubeNamespaceCollector struct {
+// KubecostNamespaceCollector is a prometheus collector that generates namespace sourced metrics
+type KubecostNamespaceCollector struct {
 	KubeClusterCache clustercache.ClusterCache
 }
 
 // Describe sends the super-set of all possible descriptors of metrics
 // collected by this Collector.
-func (nsac KubeNamespaceCollector) Describe(ch chan<- *prometheus.Desc) {
+func (nsac KubecostNamespaceCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- prometheus.NewDesc("kube_namespace_annotations", "namespace annotations", []string{}, nil)
 }
 
 // Collect is called by the Prometheus registry when collecting metrics.
-func (nsac KubeNamespaceCollector) Collect(ch chan<- prometheus.Metric) {
+func (nsac KubecostNamespaceCollector) Collect(ch chan<- prometheus.Metric) {
 	namespaces := nsac.KubeClusterCache.GetAllNamespaces()
 	for _, namespace := range namespaces {
 		nsName := namespace.GetName()
 
 		labels, values := prom.KubeAnnotationsToLabels(namespace.Annotations)
 		if len(labels) > 0 {
-			m := newNamespaceAnnotationsMetric(nsName, "kube_namespace_annotations", labels, values)
+			m := newNamespaceAnnotationsMetric("kube_namespace_annotations", nsName, labels, values)
 			ch <- m
 		}
 	}
@@ -45,27 +45,29 @@ func (nsac KubeNamespaceCollector) Collect(ch chan<- prometheus.Metric) {
 type NamespaceAnnotationsMetric struct {
 	fqName      string
 	help        string
+	namespace   string
 	labelNames  []string
 	labelValues []string
-	namespace   string
 }
 
 // Creates a new NamespaceAnnotationsMetric, implementation of prometheus.Metric
-func newNamespaceAnnotationsMetric(namespace, fqname string, labelNames []string, labelValues []string) NamespaceAnnotationsMetric {
+func newNamespaceAnnotationsMetric(fqname, namespace string, labelNames []string, labelValues []string) NamespaceAnnotationsMetric {
 	return NamespaceAnnotationsMetric{
-		namespace:   namespace,
 		fqName:      fqname,
+		help:        "kube_namespace_annotations Namespace Annotations",
+		namespace:   namespace,
 		labelNames:  labelNames,
 		labelValues: labelValues,
-		help:        "kube_namespace_annotations Namespace Annotations",
 	}
 }
 
 // Desc returns the descriptor for the Metric. This method idempotently
 // returns the same descriptor throughout the lifetime of the Metric.
 func (nam NamespaceAnnotationsMetric) Desc() *prometheus.Desc {
-	l := prometheus.Labels{"namespace": nam.namespace}
-	return prometheus.NewDesc(nam.fqName, nam.help, nam.labelNames, l)
+	l := prometheus.Labels{
+		"namespace": nam.namespace,
+	}
+	return prometheus.NewDesc(nam.fqName, nam.help, []string{}, l)
 }
 
 // Write encodes the Metric into a "Metric" Protocol Buffer data
@@ -90,3 +92,87 @@ func (nam NamespaceAnnotationsMetric) Write(m *dto.Metric) error {
 	m.Label = labels
 	return nil
 }
+
+//--------------------------------------------------------------------------
+//  KubeNamespaceCollector
+//--------------------------------------------------------------------------
+
+// KubeNamespaceCollector is a prometheus collector that generates namespace sourced metrics
+type KubeNamespaceCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (nsac KubeNamespaceCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_namespace_labels", "namespace labels", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (nsac KubeNamespaceCollector) Collect(ch chan<- prometheus.Metric) {
+	namespaces := nsac.KubeClusterCache.GetAllNamespaces()
+	for _, namespace := range namespaces {
+		nsName := namespace.GetName()
+
+		labels, values := prom.KubeLabelsToLabels(namespace.Labels)
+		if len(labels) > 0 {
+			m := newNamespaceAnnotationsMetric("kube_namespace_labels", nsName, labels, values)
+			ch <- m
+		}
+	}
+}
+
+//--------------------------------------------------------------------------
+//  NamespaceAnnotationsMetric
+//--------------------------------------------------------------------------
+
+// NamespaceAnnotationsMetric is a prometheus.Metric used to encode namespace annotations
+type KubeNamespaceLabelsMetric struct {
+	fqName      string
+	help        string
+	namespace   string
+	labelNames  []string
+	labelValues []string
+}
+
+// Creates a new KubeNamespaceLabelsMetric, implementation of prometheus.Metric
+func newKubeNamespaceLabelsMetric(fqname, namespace string, labelNames []string, labelValues []string) KubeNamespaceLabelsMetric {
+	return KubeNamespaceLabelsMetric{
+		namespace:   namespace,
+		fqName:      fqname,
+		labelNames:  labelNames,
+		labelValues: labelValues,
+		help:        "kube_namespace_labels Namespace Labels",
+	}
+}
+
+// Desc returns the descriptor for the Metric. This method idempotently
+// returns the same descriptor throughout the lifetime of the Metric.
+func (nam KubeNamespaceLabelsMetric) Desc() *prometheus.Desc {
+	l := prometheus.Labels{
+		"namespace": nam.namespace,
+	}
+	return prometheus.NewDesc(nam.fqName, nam.help, []string{}, l)
+}
+
+// Write encodes the Metric into a "Metric" Protocol Buffer data transmission object.
+func (nam KubeNamespaceLabelsMetric) Write(m *dto.Metric) error {
+	h := float64(1)
+	m.Gauge = &dto.Gauge{
+		Value: &h,
+	}
+
+	var labels []*dto.LabelPair
+	for i := range nam.labelNames {
+		labels = append(labels, &dto.LabelPair{
+			Name:  &nam.labelNames[i],
+			Value: &nam.labelValues[i],
+		})
+	}
+	labels = append(labels, &dto.LabelPair{
+		Name:  toStringPtr("namespace"),
+		Value: &nam.namespace,
+	})
+	m.Label = labels
+	return nil
+}

pkg/metrics/podmetrics.go

@@ -11,23 +11,49 @@ import (
 	v1 "k8s.io/api/core/v1"
 )
 
+//--------------------------------------------------------------------------
+//  KubecostPodCollector
+//--------------------------------------------------------------------------
+
+// KubecostPodCollector is a prometheus collector that emits pod metrics
+type KubecostPodCollector struct {
+	KubeClusterCache clustercache.ClusterCache
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+// collected by this Collector.
+func (kpmc KubecostPodCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- prometheus.NewDesc("kube_pod_annotations", "All annotations for each pod prefix with annotation_", []string{}, nil)
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (kpmc KubecostPodCollector) Collect(ch chan<- prometheus.Metric) {
+	pods := kpmc.KubeClusterCache.GetAllPods()
+	for _, pod := range pods {
+		podName := pod.GetName()
+		podNS := pod.GetNamespace()
+
+		// Pod Annotations
+		labels, values := prom.KubeAnnotationsToLabels(pod.Annotations)
+		if len(labels) > 0 {
+			ch <- newPodAnnotationMetric("kube_pod_annotations", podNS, podName, labels, values)
+		}
+	}
+}
+
 //--------------------------------------------------------------------------
 //  KubePodCollector
 //--------------------------------------------------------------------------
 
 // KubePodMetricCollector is a prometheus collector that emits pod metrics
 type KubePodCollector struct {
-	KubeClusterCache   clustercache.ClusterCache
-	emitPodAnnotations bool
+	KubeClusterCache clustercache.ClusterCache
 }
 
 // Describe sends the super-set of all possible descriptors of metrics
 // collected by this Collector.
 func (kpmc KubePodCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- prometheus.NewDesc("kube_pod_labels", "All labels for each pod prefixed with label_", []string{}, nil)
-	if kpmc.emitPodAnnotations {
-		ch <- prometheus.NewDesc("kube_pod_annotations", "All annotations for each pod prefix with annotation_", []string{}, nil)
-	}
 	ch <- prometheus.NewDesc("kube_pod_owner", "Information about the Pod's owner", []string{}, nil)
 	ch <- prometheus.NewDesc("kube_pod_container_status_running", "Describes whether the container is currently in running state", []string{}, nil)
 	ch <- prometheus.NewDesc("kube_pod_container_status_terminated_reason", "Describes the reason the container is currently in terminated state.", []string{}, nil)
@@ -63,22 +89,13 @@ func (kpmc KubePodCollector) Collect(ch chan<- prometheus.Metric) {
 			}
 
 			for _, p := range phases {
-				ch <- newKubePodStatusPhaseMetric("kube_pod_status_phase", podName, podNS, podUID, p.n, boolFloat64(p.v))
+				ch <- newKubePodStatusPhaseMetric("kube_pod_status_phase", podNS, podName, podUID, p.n, boolFloat64(p.v))
 			}
 		}
 
 		// Pod Labels
 		labelNames, labelValues := prom.KubePrependQualifierToLabels(pod.GetLabels(), "label_")
-		ch <- newKubePodLabelsMetric(podName, podNS, podUID, "kube_pod_labels", labelNames, labelValues)
-
-		// Pod Annotations
-		if kpmc.emitPodAnnotations {
-			labels, values := prom.KubeAnnotationsToLabels(pod.Annotations)
-
-			if len(labels) > 0 {
-				ch <- newPodAnnotationMetric(podNS, podName, "kube_pod_annotations", labels, values)
-			}
-		}
+		ch <- newKubePodLabelsMetric("kube_pod_labels", podNS, podName, podUID, labelNames, labelValues)
 
 		// Owner References
 		for _, owner := range pod.OwnerReferences {
@@ -87,16 +104,16 @@ func (kpmc KubePodCollector) Collect(ch chan<- prometheus.Metric) {
 
 		// Container Status
 		for _, status := range pod.Status.ContainerStatuses {
-			ch <- newKubePodContainerStatusRestartsTotalMetric("kube_pod_container_status_restarts_total", podName, podNS, podUID, status.Name, float64(status.RestartCount))
+			ch <- newKubePodContainerStatusRestartsTotalMetric("kube_pod_container_status_restarts_total", podNS, podName, podUID, status.Name, float64(status.RestartCount))
 			if status.State.Running != nil {
-				ch <- newKubePodContainerStatusRunningMetric("kube_pod_container_status_running", podName, podNS, podUID, status.Name)
+				ch <- newKubePodContainerStatusRunningMetric("kube_pod_container_status_running", podNS, podName, podUID, status.Name)
 			}
 
 			if status.State.Terminated != nil {
 				ch <- newKubePodContainerStatusTerminatedReasonMetric(
 					"kube_pod_container_status_terminated_reason",
-					podName,
 					podNS,
+					podName,
 					podUID,
 					status.Name,
 					status.State.Terminated.Reason)
@@ -116,8 +133,8 @@ func (kpmc KubePodCollector) Collect(ch chan<- prometheus.Metric) {
 
 				ch <- newKubePodContainerResourceRequestsMetric(
 					"kube_pod_container_resource_requests",
-					podName,
 					podNS,
+					podName,
 					podUID,
 					container.Name,
 					node,
@@ -140,8 +157,8 @@ func (kpmc KubePodCollector) Collect(ch chan<- prometheus.Metric) {
 				if resource == "cpu" {
 					ch <- newKubePodContainerResourceLimitsCPUCoresMetric(
 						"kube_pod_container_resource_limits_cpu_cores",
-						podName,
 						podNS,
+						podName,
 						podUID,
 						container.Name,
 						node,
@@ -150,8 +167,8 @@ func (kpmc KubePodCollector) Collect(ch chan<- prometheus.Metric) {
 				if resource == "memory" {
 					ch <- newKubePodContainerResourceLimitsMemoryBytesMetric(
 						"kube_pod_container_resource_limits_memory_bytes",
-						podName,
 						podNS,
+						podName,
 						podUID,
 						container.Name,
 						node,
@@ -160,8 +177,8 @@ func (kpmc KubePodCollector) Collect(ch chan<- prometheus.Metric) {
 
 				ch <- newKubePodContainerResourceLimitsMetric(
 					"kube_pod_container_resource_limits",
-					podName,
 					podNS,
+					podName,
 					podUID,
 					container.Name,
 					node,
@@ -179,31 +196,34 @@ func (kpmc KubePodCollector) Collect(ch chan<- prometheus.Metric) {
 
 // PodAnnotationsMetric is a prometheus.Metric used to encode namespace annotations
 type PodAnnotationsMetric struct {
-	name        string
 	fqName      string
 	help        string
+	namespace   string
+	pod         string
 	labelNames  []string
 	labelValues []string
-	namespace   string
 }
 
 // Creates a new PodAnnotationsMetric, implementation of prometheus.Metric
-func newPodAnnotationMetric(namespace, name, fqname string, labelNames []string, labelValues []string) PodAnnotationsMetric {
+func newPodAnnotationMetric(fqname, namespace, pod string, labelNames, labelValues []string) PodAnnotationsMetric {
 	return PodAnnotationsMetric{
-		namespace:   namespace,
-		name:        name,
 		fqName:      fqname,
+		help:        "kube_pod_annotations Pod Annotations",
+		namespace:   namespace,
+		pod:         pod,
 		labelNames:  labelNames,
 		labelValues: labelValues,
-		help:        "kube_pod_annotations Pod Annotations",
 	}
 }
 
 // Desc returns the descriptor for the Metric. This method idempotently
 // returns the same descriptor throughout the lifetime of the Metric.
 func (pam PodAnnotationsMetric) Desc() *prometheus.Desc {
-	l := prometheus.Labels{"namespace": pam.namespace, "pod": pam.name}
-	return prometheus.NewDesc(pam.fqName, pam.help, pam.labelNames, l)
+	l := prometheus.Labels{
+		"namespace": pam.namespace,
+		"pod":       pam.pod,
+	}
+	return prometheus.NewDesc(pam.fqName, pam.help, []string{}, l)
 }
 
 // Write encodes the Metric into a "Metric" Protocol Buffer data
@@ -228,7 +248,7 @@ func (pam PodAnnotationsMetric) Write(m *dto.Metric) error {
 		},
 		&dto.LabelPair{
 			Name:  toStringPtr("pod"),
-			Value: &pam.name,
+			Value: &pam.pod,
 		})
 	m.Label = labels
 	return nil
@@ -244,23 +264,23 @@ func (pam PodAnnotationsMetric) Write(m *dto.Metric) error {
 type KubePodLabelsMetric struct {
 	fqName      string
 	help        string
-	labelNames  []string
-	labelValues []string
 	pod         string
 	namespace   string
 	uid         string
+	labelNames  []string
+	labelValues []string
 }
 
 // Creates a new KubePodLabelsMetric, implementation of prometheus.Metric
-func newKubePodLabelsMetric(pod string, namespace string, uid string, fqname string, labelNames []string, labelValues []string) KubePodLabelsMetric {
+func newKubePodLabelsMetric(fqname, namespace, pod, uid string, labelNames []string, labelValues []string) KubePodLabelsMetric {
 	return KubePodLabelsMetric{
 		fqName:      fqname,
-		labelNames:  labelNames,
-		labelValues: labelValues,
 		help:        "kube_pod_labels all labels for each pod prefixed with label_",
 		pod:         pod,
 		namespace:   namespace,
 		uid:         uid,
+		labelNames:  labelNames,
+		labelValues: labelValues,
 	}
 }
 
@@ -268,8 +288,8 @@ func newKubePodLabelsMetric(pod string, namespace string, uid string, fqname str
 // returns the same descriptor throughout the lifetime of the Metric.
 func (nam KubePodLabelsMetric) Desc() *prometheus.Desc {
 	l := prometheus.Labels{
-		"pod":       nam.pod,
 		"namespace": nam.namespace,
+		"pod":       nam.pod,
 		"uid":       nam.uid,
 	}
 	return prometheus.NewDesc(nam.fqName, nam.help, nam.labelNames, l)
@@ -325,7 +345,7 @@ type KubePodContainerStatusRestartsTotalMetric struct {
 }
 
 // Creates a new KubePodContainerStatusRestartsTotalMetric, implementation of prometheus.Metric
-func newKubePodContainerStatusRestartsTotalMetric(fqname, pod, namespace, uid, container string, value float64) KubePodContainerStatusRestartsTotalMetric {
+func newKubePodContainerStatusRestartsTotalMetric(fqname, namespace, pod, uid, container string, value float64) KubePodContainerStatusRestartsTotalMetric {
 	return KubePodContainerStatusRestartsTotalMetric{
 		fqName:    fqname,
 		help:      "kube_pod_container_status_restarts_total total container restarts",
@@ -341,8 +361,8 @@ func newKubePodContainerStatusRestartsTotalMetric(fqname, pod, namespace, uid, c
 // returns the same descriptor throughout the lifetime of the Metric.
 func (kpcs KubePodContainerStatusRestartsTotalMetric) Desc() *prometheus.Desc {
 	l := prometheus.Labels{
-		"pod":       kpcs.pod,
 		"namespace": kpcs.namespace,
+		"pod":       kpcs.pod,
 		"uid":       kpcs.uid,
 		"container": kpcs.container,
 	}
@@ -357,14 +377,14 @@ func (kpcs KubePodContainerStatusRestartsTotalMetric) Write(m *dto.Metric) error
 
 	var labels []*dto.LabelPair
 	labels = append(labels,
-		&dto.LabelPair{
-			Name:  toStringPtr("pod"),
-			Value: &kpcs.pod,
-		},
 		&dto.LabelPair{
 			Name:  toStringPtr("namespace"),
 			Value: &kpcs.namespace,
 		},
+		&dto.LabelPair{
+			Name:  toStringPtr("pod"),
+			Value: &kpcs.pod,
+		},
 		&dto.LabelPair{
 			Name:  toStringPtr("container"),
 			Value: &kpcs.container,
@@ -394,7 +414,7 @@ type KubePodContainerStatusTerminatedReasonMetric struct {
 }
 
 // Creates a new KubePodContainerStatusRestartsTotalMetric, implementation of prometheus.Metric
-func newKubePodContainerStatusTerminatedReasonMetric(fqname, pod, namespace, uid, container, reason string) KubePodContainerStatusTerminatedReasonMetric {
+func newKubePodContainerStatusTerminatedReasonMetric(fqname, namespace, pod, uid, container, reason string) KubePodContainerStatusTerminatedReasonMetric {
 	return KubePodContainerStatusTerminatedReasonMetric{
 		fqName:    fqname,
 		help:      "kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.",
@@ -410,8 +430,8 @@ func newKubePodContainerStatusTerminatedReasonMetric(fqname, pod, namespace, uid
 // returns the same descriptor throughout the lifetime of the Metric.
 func (kpcs KubePodContainerStatusTerminatedReasonMetric) Desc() *prometheus.Desc {
 	l := prometheus.Labels{
-		"pod":       kpcs.pod,
 		"namespace": kpcs.namespace,
+		"pod":       kpcs.pod,
 		"uid":       kpcs.uid,
 		"container": kpcs.container,
 		"reason":    kpcs.reason,
@@ -428,14 +448,14 @@ func (kpcs KubePodContainerStatusTerminatedReasonMetric) Write(m *dto.Metric) er
 
 	var labels []*dto.LabelPair
 	labels = append(labels,
-		&dto.LabelPair{
-			Name:  toStringPtr("pod"),
-			Value: &kpcs.pod,
-		},
 		&dto.LabelPair{
 			Name:  toStringPtr("namespace"),
 			Value: &kpcs.namespace,
 		},
+		&dto.LabelPair{
+			Name:  toStringPtr("pod"),
+			Value: &kpcs.pod,
+		},
 		&dto.LabelPair{
 			Name:  toStringPtr("container"),
 			Value: &kpcs.container,
@@ -469,7 +489,7 @@ type KubePodStatusPhaseMetric struct {
 }
 
 // Creates a new KubePodContainerStatusRestartsTotalMetric, implementation of prometheus.Metric
-func newKubePodStatusPhaseMetric(fqname, pod, namespace, uid, phase string, value float64) KubePodStatusPhaseMetric {
+func newKubePodStatusPhaseMetric(fqname, namespace, pod, uid, phase string, value float64) KubePodStatusPhaseMetric {
 	return KubePodStatusPhaseMetric{
 		fqName:    fqname,
 		help:      "kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.",
@@ -485,8 +505,8 @@ func newKubePodStatusPhaseMetric(fqname, pod, namespace, uid, phase string, valu
 // returns the same descriptor throughout the lifetime of the Metric.
 func (kpcs KubePodStatusPhaseMetric) Desc() *prometheus.Desc {
 	l := prometheus.Labels{
-		"pod":       kpcs.pod,
 		"namespace": kpcs.namespace,
+		"pod":       kpcs.pod,
 		"uid":       kpcs.uid,
 		"phase":     kpcs.phase,
 	}
@@ -501,14 +521,14 @@ func (kpcs KubePodStatusPhaseMetric) Write(m *dto.Metric) error {
 
 	var labels []*dto.LabelPair
 	labels = append(labels,
-		&dto.LabelPair{
-			Name:  toStringPtr("pod"),
-			Value: &kpcs.pod,
-		},
 		&dto.LabelPair{
 			Name:  toStringPtr("namespace"),
 			Value: &kpcs.namespace,
 		},
+		&dto.LabelPair{
+			Name:  toStringPtr("pod"),
+			Value: &kpcs.pod,
+		},
 		&dto.LabelPair{
 			Name:  toStringPtr("uid"),
 			Value: &kpcs.uid,
@@ -539,7 +559,7 @@ type KubePodContainerStatusRunningMetric struct {
 }
 
 // Creates a new KubePodContainerStatusRunningMetric, implementation of prometheus.Metric
-func newKubePodContainerStatusRunningMetric(fqname string, pod string, namespace string, uid string, container string) KubePodContainerStatusRunningMetric {
+func newKubePodContainerStatusRunningMetric(fqname, namespace, pod, uid, container string) KubePodContainerStatusRunningMetric {
 	return KubePodContainerStatusRunningMetric{
 		fqName:    fqname,
 		help:      "kube_pod_container_status_running pods container status",
@@ -554,8 +574,8 @@ func newKubePodContainerStatusRunningMetric(fqname string, pod string, namespace
 // returns the same descriptor throughout the lifetime of the Metric.
 func (kpcs KubePodContainerStatusRunningMetric) Desc() *prometheus.Desc {
 	l := prometheus.Labels{
-		"pod":       kpcs.pod,
 		"namespace": kpcs.namespace,
+		"pod":       kpcs.pod,
 		"uid":       kpcs.uid,
 		"container": kpcs.container,
 	}
@@ -572,14 +592,14 @@ func (kpcs KubePodContainerStatusRunningMetric) Write(m *dto.Metric) error {
 
 	var labels []*dto.LabelPair
 	labels = append(labels,
-		&dto.LabelPair{
-			Name:  toStringPtr("pod"),
-			Value: &kpcs.pod,
-		},
 		&dto.LabelPair{
 			Name:  toStringPtr("namespace"),
 			Value: &kpcs.namespace,
 		},
+		&dto.LabelPair{
+			Name:  toStringPtr("pod"),
+			Value: &kpcs.pod,
+		},
 		&dto.LabelPair{
 			Name:  toStringPtr("container"),
 			Value: &kpcs.container,
@@ -612,7 +632,7 @@ type KubePodContainerResourceRequestsMetric struct {
 }
 
 // Creates a new newKubePodContainerResourceRequestsMetric, implementation of prometheus.Metric
-func newKubePodContainerResourceRequestsMetric(fqname, pod, namespace, uid, container, node, resource, unit string, value float64) KubePodContainerResourceRequestsMetric {
+func newKubePodContainerResourceRequestsMetric(fqname, namespace, pod, uid, container, node, resource, unit string, value float64) KubePodContainerResourceRequestsMetric {
 	return KubePodContainerResourceRequestsMetric{
 		fqName:    fqname,
 		help:      "kube_pod_container_resource_requests pods container resource requests",
@@ -631,8 +651,8 @@ func newKubePodContainerResourceRequestsMetric(fqname, pod, namespace, uid, cont
 // returns the same descriptor throughout the lifetime of the Metric.
 func (kpcrr KubePodContainerResourceRequestsMetric) Desc() *prometheus.Desc {
 	l := prometheus.Labels{
-		"pod":       kpcrr.pod,
 		"namespace": kpcrr.namespace,
+		"pod":       kpcrr.pod,
 		"uid":       kpcrr.uid,
 		"container": kpcrr.container,
 		"node":      kpcrr.node,
@@ -650,14 +670,14 @@ func (kpcrr KubePodContainerResourceRequestsMetric) Write(m *dto.Metric) error {
 	}
 
 	m.Label = []*dto.LabelPair{
-		{
-			Name:  toStringPtr("pod"),
-			Value: &kpcrr.pod,
-		},
 		{
 			Name:  toStringPtr("namespace"),
 			Value: &kpcrr.namespace,
 		},
+		{
+			Name:  toStringPtr("pod"),
+			Value: &kpcrr.pod,
+		},
 		{
 			Name:  toStringPtr("container"),
 			Value: &kpcrr.container,
@@ -701,7 +721,7 @@ type KubePodContainerResourceLimitsMetric struct {
 }
 
 // Creates a new KubePodContainerResourceLimitsMetric, implementation of prometheus.Metric
-func newKubePodContainerResourceLimitsMetric(fqname, pod, namespace, uid, container, node, resource, unit string, value float64) KubePodContainerResourceLimitsMetric {
+func newKubePodContainerResourceLimitsMetric(fqname, namespace, pod, uid, container, node, resource, unit string, value float64) KubePodContainerResourceLimitsMetric {
 	return KubePodContainerResourceLimitsMetric{
 		fqName:    fqname,
 		help:      "kube_pod_container_resource_limits pods container resource limits",
@@ -720,8 +740,8 @@ func newKubePodContainerResourceLimitsMetric(fqname, pod, namespace, uid, contai
 // returns the same descriptor throughout the lifetime of the Metric.
 func (kpcrr KubePodContainerResourceLimitsMetric) Desc() *prometheus.Desc {
 	l := prometheus.Labels{
-		"pod":       kpcrr.pod,
 		"namespace": kpcrr.namespace,
+		"pod":       kpcrr.pod,
 		"uid":       kpcrr.uid,
 		"container": kpcrr.container,
 		"node":      kpcrr.node,
@@ -739,14 +759,14 @@ func (kpcrr KubePodContainerResourceLimitsMetric) Write(m *dto.Metric) error {
 	}
 
 	m.Label = []*dto.LabelPair{
-		{
-			Name:  toStringPtr("pod"),
-			Value: &kpcrr.pod,
-		},
 		{
 			Name:  toStringPtr("namespace"),
 			Value: &kpcrr.namespace,
 		},
+		{
+			Name:  toStringPtr("pod"),
+			Value: &kpcrr.pod,
+		},
 		{
 			Name:  toStringPtr("container"),
 			Value: &kpcrr.container,
@@ -788,7 +808,7 @@ type KubePodContainerResourceLimitsCPUCoresMetric struct {
 }
 
 // Creates a new KubePodContainerResourceLimitsMetric, implementation of prometheus.Metric
-func newKubePodContainerResourceLimitsCPUCoresMetric(fqname, pod, namespace, uid, container, node string, value float64) KubePodContainerResourceLimitsCPUCoresMetric {
+func newKubePodContainerResourceLimitsCPUCoresMetric(fqname, namespace, pod, uid, container, node string, value float64) KubePodContainerResourceLimitsCPUCoresMetric {
 	return KubePodContainerResourceLimitsCPUCoresMetric{
 		fqName:    fqname,
 		help:      "kube_pod_container_resource_limits_cpu_cores pods container cpu cores resource limits",
@@ -805,8 +825,8 @@ func newKubePodContainerResourceLimitsCPUCoresMetric(fqname, pod, namespace, uid
 // returns the same descriptor throughout the lifetime of the Metric.
 func (kpcrr KubePodContainerResourceLimitsCPUCoresMetric) Desc() *prometheus.Desc {
 	l := prometheus.Labels{
-		"pod":       kpcrr.pod,
 		"namespace": kpcrr.namespace,
+		"pod":       kpcrr.pod,
 		"uid":       kpcrr.uid,
 		"container": kpcrr.container,
 		"node":      kpcrr.node,
@@ -822,14 +842,14 @@ func (kpcrr KubePodContainerResourceLimitsCPUCoresMetric) Write(m *dto.Metric) e
 	}
 
 	m.Label = []*dto.LabelPair{
-		{
-			Name:  toStringPtr("pod"),
-			Value: &kpcrr.pod,
-		},
 		{
 			Name:  toStringPtr("namespace"),
 			Value: &kpcrr.namespace,
 		},
+		{
+			Name:  toStringPtr("pod"),
+			Value: &kpcrr.pod,
+		},
 		{
 			Name:  toStringPtr("container"),
 			Value: &kpcrr.container,
@@ -863,7 +883,7 @@ type KubePodContainerResourceLimitsMemoryBytesMetric struct {
 }
 
 // Creates a new KubePodContainerResourceLimitsMemoryBytesMetric, implementation of prometheus.Metric
-func newKubePodContainerResourceLimitsMemoryBytesMetric(fqname, pod, namespace, uid, container, node string, value float64) KubePodContainerResourceLimitsMemoryBytesMetric {
+func newKubePodContainerResourceLimitsMemoryBytesMetric(fqname, namespace, pod, uid, container, node string, value float64) KubePodContainerResourceLimitsMemoryBytesMetric {
 	return KubePodContainerResourceLimitsMemoryBytesMetric{
 		fqName:    fqname,
 		help:      "kube_pod_container_resource_limits_memory_bytes pods container memory bytes resource limits",
@@ -880,8 +900,8 @@ func newKubePodContainerResourceLimitsMemoryBytesMetric(fqname, pod, namespace,
 // returns the same descriptor throughout the lifetime of the Metric.
 func (kpcrr KubePodContainerResourceLimitsMemoryBytesMetric) Desc() *prometheus.Desc {
 	l := prometheus.Labels{
-		"pod":       kpcrr.pod,
 		"namespace": kpcrr.namespace,
+		"pod":       kpcrr.pod,
 		"uid":       kpcrr.uid,
 		"container": kpcrr.container,
 		"node":      kpcrr.node,
@@ -897,14 +917,14 @@ func (kpcrr KubePodContainerResourceLimitsMemoryBytesMetric) Write(m *dto.Metric
 	}
 
 	m.Label = []*dto.LabelPair{
-		{
-			Name:  toStringPtr("pod"),
-			Value: &kpcrr.pod,
-		},
 		{
 			Name:  toStringPtr("namespace"),
 			Value: &kpcrr.namespace,
 		},
+		{
+			Name:  toStringPtr("pod"),
+			Value: &kpcrr.pod,
+		},
 		{
 			Name:  toStringPtr("container"),
 			Value: &kpcrr.container,

pkg/prom/metrics_test.go

@@ -0,0 +1,95 @@
+package prom
+
+import (
+	"fmt"
+	"testing"
+)
+
+func checkSlice(s1, s2 []string) error {
+	if len(s1) != len(s2) {
+		return fmt.Errorf("len(s1) [%d] != len(s2) [%d]", len(s1), len(s2))
+	}
+
+	for i := 0; i < len(s1); i++ {
+		if s1[i] != s2[i] {
+			return fmt.Errorf("At Index: %d. Different Values %s (s1) != %s (s2)", i, s1[i], s2[i])
+		}
+	}
+	return nil
+}
+
+func TestEmptyKubeLabelsToPromLabels(t *testing.T) {
+	labels, values := KubeLabelsToLabels(nil)
+
+	if len(labels) != 0 {
+		t.Errorf("Labels length is non-zero\n")
+	}
+	if len(values) != 0 {
+		t.Errorf("Values length is non-zero\n")
+	}
+
+	labels, values = KubeLabelsToLabels(map[string]string{})
+
+	if len(labels) != 0 {
+		t.Errorf("Labels length is non-zero\n")
+	}
+	if len(values) != 0 {
+		t.Errorf("Values length is non-zero\n")
+	}
+}
+
+func TestKubeLabelsToPromLabels(t *testing.T) {
+	var expectedLabels []string = []string{
+		"label_app",
+		"label_chart",
+		"label_control_plane",
+		"label_gatekeeper_sh_operation",
+		"label_heritage",
+		"label_pod_template_hash",
+		"label_release",
+	}
+	var expectedValues []string = []string{
+		"gatekeeper",
+		"gatekeeper",
+		"audit-controller",
+		"audit",
+		"Helm",
+		"5599859cd4",
+		"gatekeeper",
+	}
+
+	kubeLabels := map[string]string{
+		"app":                     "gatekeeper",
+		"chart":                   "gatekeeper",
+		"control-plane":           "audit-controller",
+		"gatekeeper.sh/operation": "audit",
+		"heritage":                "Helm",
+		"pod-template-hash":       "5599859cd4",
+		"release":                 "gatekeeper",
+	}
+
+	labels, values := KubePrependQualifierToLabels(kubeLabels, "label_")
+	l2, v2 := KubeLabelsToLabels(kubeLabels)
+
+	// Check to make sure we get expected labels and values returned
+	err := checkSlice(labels, expectedLabels)
+	if err != nil {
+		t.Errorf("%s", err)
+	}
+	err = checkSlice(values, expectedValues)
+	if err != nil {
+		t.Errorf("%s", err)
+	}
+
+	// Check to make sure the helper function returns what the prependqualifier func
+	// returns
+	err = checkSlice(l2, labels)
+	if err != nil {
+		t.Errorf("%s", err)
+	}
+
+	err = checkSlice(v2, values)
+	if err != nil {
+		t.Errorf("%s", err)
+	}
+}