KCM-3828 - Prevent duplicate Prometheus metric labels (#3114)

Signed-off-by: Nik Willwerth <nwillwerth@kubecost.com>

core/pkg/util/promutil/promutil.go

@@ -75,16 +75,22 @@ func LabelNamesFrom(labels map[string]string) []string {
 
 // Prepends a qualifier string to the keys provided in the m map and returns the new keys and values.
 func KubePrependQualifierToLabels(m map[string]string, qualifier string) ([]string, []string) {
-	keys := make([]string, 0, len(m))
-	for k := range m {
+	// sanitize the keys in m to prevent duplicate output keys
+	sanitizedM := make(map[string]string)
+	for k, v := range m {
+		sanitizedM[SanitizeLabelName(k)] = v
+	}
+
+	keys := make([]string, 0, len(sanitizedM))
+	for k := range sanitizedM {
 		keys = append(keys, k)
 	}
 	sort.Strings(keys)
 
-	values := make([]string, 0, len(m))
+	values := make([]string, 0, len(sanitizedM))
 	for i, k := range keys {
-		keys[i] = qualifier + SanitizeLabelName(k)
-		values = append(values, m[k])
+		keys[i] = qualifier + k
+		values = append(values, sanitizedM[k])
 	}
 
 	return keys, values

core/pkg/util/promutil/promutil_test.go

@@ -95,6 +95,53 @@ func TestKubeLabelsToPromLabels(t *testing.T) {
 	}
 }
 
+func TestKubePrependQualifierToLabelsDuplicates(t *testing.T) {
+	// 7 expected labels/values
+	expectedLabels := []string{
+		"label_app_",
+		"label_chart",
+		"label_control_plane",
+		"label_gatekeeper_sh_operation",
+		"label_heritage",
+		"label_pod_template_hash",
+		"label_release",
+	}
+	expectedValues := []string{
+		"gatekeeper",
+		"gatekeeper",
+		"audit-controller",
+		"audit",
+		"Helm",
+		"5599859cd4",
+		"gatekeeper",
+	}
+
+	// 8 input labels/values, with one duplicate label
+	kubeLabels := map[string]string{
+		// app- will be sanitized to app_
+		"app-":                    "gatekeeper",
+		"app_":                    "gatekeeper",
+		"chart":                   "gatekeeper",
+		"control-plane":           "audit-controller",
+		"gatekeeper.sh/operation": "audit",
+		"heritage":                "Helm",
+		"pod-template-hash":       "5599859cd4",
+		"release":                 "gatekeeper",
+	}
+
+	labels, values := KubePrependQualifierToLabels(kubeLabels, "label_")
+
+	// Check to make sure we get expected labels and values returned
+	err := checkSlice(labels, expectedLabels)
+	if err != nil {
+		t.Errorf("%s", err)
+	}
+	err = checkSlice(values, expectedValues)
+	if err != nil {
+		t.Errorf("%s", err)
+	}
+}
+
 func TestSanitizeLabels(t *testing.T) {
 	type testCase struct {
 		in  map[string]string