KCM-5392: Configmap External label implementation and connect it to Default Collector (#3887)

Signed-off-by: Alan Rodrigues <alanrodrigues@Alans-MacBook-Pro.local>
Signed-off-by: Nik Willwerth <nwillwerth@kubecost.com>
Signed-off-by: simanadler <sima@il.ibm.com>
Signed-off-by: Claude <noreply@anthropic.com>
Signed-off-by: Tushar Verma <tusharmyself06@gmail.com>
Signed-off-by: York Chen <york.chen@nutanix.com>
Signed-off-by: Kush Agarwal <agrawalkush783@gmail.com>
Signed-off-by: Sawyer Ward <104373596+sawyerward@users.noreply.github.com>
Signed-off-by: thomasvn <thomasvn.dev@gmail.com>
Signed-off-by: Thomas Nguyen <thomasvn.dev@gmail.com>
Signed-off-by: thomasvn <thomasnguyen96@gmail.com>
Co-authored-by: Alan Rodrigues <alanrodrigues@Alans-MacBook-Pro.local>
Co-authored-by: nik-kc <127428785+nik-kc@users.noreply.github.com>
Co-authored-by: simanadler <sima@il.ibm.com>
Co-authored-by: Alex Meijer <ameijer@users.noreply.github.com>
Co-authored-by: Warwick <warwick.peatey@ibm.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Matt Bolt <mbolt35@gmail.com>
Co-authored-by: Tushar-Verma <tusharmyself06@gmail.com>
Co-authored-by: York Chen <york.chen@nutanix.com>
Co-authored-by: Kush Agarwal <145124726+Kush172005@users.noreply.github.com>
Co-authored-by: Sawyer Ward <104373596+sawyerward@users.noreply.github.com>
Co-authored-by: Thomas Nguyen <thomasvn.dev@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

core/go.mod

@@ -33,6 +33,7 @@ require (
 	google.golang.org/grpc v1.79.3
 	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.36.0
 	k8s.io/apimachinery v0.36.0
 	k8s.io/client-go v0.36.0
@@ -151,7 +152,6 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260316180232-0b37fe3546d5 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/klog/v2 v2.140.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect
 	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect

core/pkg/external/configmapsource.go

@@ -0,0 +1,230 @@
+package external
+
+import (
+	"fmt"
+	"maps"
+	"regexp"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+// Kubernetes label key/value constraints.
+// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set
+var (
+	// nameSegment: 1–63 chars, alphanumeric start/end, [-_.] allowed between.
+	reNameSegment = regexp.MustCompile(`^[a-zA-Z0-9]([a-zA-Z0-9._-]{0,61}[a-zA-Z0-9])?$`)
+
+	// DNS label: 1–63 chars, alphanumeric start/end, hyphens allowed between.
+	reDNSLabel = regexp.MustCompile(`^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$`)
+
+	// label value: empty OR 1–63 chars with same rules as name segment.
+	reLabelValue = regexp.MustCompile(`^[a-zA-Z0-9]([a-zA-Z0-9._-]{0,61}[a-zA-Z0-9])?$`)
+)
+
+// ConfigMapSource implements LabelSource for Kubernetes ConfigMaps.
+type ConfigMapSource struct {
+	cfg *Config
+}
+
+func (cms *ConfigMapSource) ExtractNodeLabels(data map[string]string) (map[string]string, error) {
+	if cms.cfg == nil {
+		return nil, fmt.Errorf("nil config")
+	}
+
+	nlCfg := cms.cfg.NodeLabelConfig()
+	if nlCfg == nil {
+		return nil, fmt.Errorf("no node label config")
+	}
+
+	cm := nlCfg.ConfigMapName()
+	key := nlCfg.Key()
+	route := nlCfg.Route()
+	// Traditional ConfigMap — labels live directly in data.
+	if key == "" && route == "" {
+		return maps.Clone(data), nil
+	}
+
+	// route is optional for block scalar. A root yaml node can be the map of node labels.
+	if key == "" && route != "" {
+		return nil, fmt.Errorf("key must be set for block scalar configMap")
+	}
+
+	// Block-scalar ConfigMap — extract the YAML document from data[Key].
+	raw, ok := data[key]
+	if !ok {
+		return nil, fmt.Errorf("key %q not found in ConfigMap %s", key, cm)
+	}
+
+	labels, err := parse(raw, route)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing the yaml: %w", err)
+	}
+
+	// Drop any keys or values that don't satisfy the Kubernetes label spec.
+	return filterValidLabels(labels), nil
+}
+
+// filterValidLabels removes entries from labels whose key or value does not
+// satisfy the Kubernetes label syntax rules. The map is mutated in place.
+func filterValidLabels(labels map[string]string) map[string]string {
+	for k, v := range labels {
+		if validateLabelKey(k) != nil || validateLabelValue(v) != nil {
+			delete(labels, k)
+		}
+	}
+	return labels
+}
+
+// validateLabelKey checks the optional-prefix/name structure of a label key.
+func validateLabelKey(key string) error {
+	if key == "" {
+		return fmt.Errorf("label key must not be empty")
+	}
+
+	prefix, name, hasSep := strings.Cut(key, "/")
+
+	// No Prefix such as
+	// app.kubernetes.io/name
+	if !hasSep {
+		// No prefix — the whole key is the name segment.
+		if err := validateNameSegment(key); err != nil {
+			return fmt.Errorf("invalid label key %q: %w", key, err)
+		}
+		return nil
+	}
+
+	if err := validateDNSSubdomain(prefix); err != nil {
+		return fmt.Errorf("invalid label key %q: prefix is not a valid DNS subdomain: %w", key, err)
+	}
+	if err := validateNameSegment(name); err != nil {
+		return fmt.Errorf("invalid label key %q: name segment: %w", key, err)
+	}
+	return nil
+}
+
+// validateNameSegment checks the name part of a label key (up to 63 chars).
+func validateNameSegment(name string) error {
+	if name == "" {
+		return fmt.Errorf("name segment must not be empty")
+	}
+	if len(name) > 63 {
+		return fmt.Errorf("name segment %q exceeds 63 characters", name)
+	}
+	if !reNameSegment.MatchString(name) {
+		return fmt.Errorf("name segment %q must begin and end with an alphanumeric character and may only contain [-_.]", name)
+	}
+	return nil
+}
+
+// validateDNSSubdomain checks that s is a valid DNS subdomain (≤253 chars,
+// dot-separated DNS labels each ≤63 chars).
+func validateDNSSubdomain(s string) error {
+	if s == "" {
+		return fmt.Errorf("DNS subdomain must not be empty")
+	}
+	if len(s) > 253 {
+		return fmt.Errorf("DNS subdomain %q exceeds 253 characters", s)
+	}
+	for _, label := range strings.Split(s, ".") {
+		if label == "" {
+			return fmt.Errorf("DNS subdomain %q contains an empty label (consecutive or trailing dots)", s)
+		}
+		if len(label) > 63 {
+			return fmt.Errorf("DNS subdomain %q: label %q exceeds 63 characters", s, label)
+		}
+		if !reDNSLabel.MatchString(label) {
+			return fmt.Errorf("DNS subdomain %q: label %q must begin and end with an alphanumeric character and may only contain hyphens", s, label)
+		}
+	}
+	return nil
+}
+
+// validateLabelValue checks a label value (empty is allowed; otherwise ≤63 chars).
+func validateLabelValue(value string) error {
+	if value == "" {
+		return nil
+	}
+	if len(value) > 63 {
+		return fmt.Errorf("label value %q exceeds 63 characters", value)
+	}
+	if !reLabelValue.MatchString(value) {
+		return fmt.Errorf("label value %q must begin and end with an alphanumeric character and may only contain [-_.]", value)
+	}
+	return nil
+}
+
+func parseNormally(input []byte) (map[string]string, error) {
+	var m map[string]string
+	err := yaml.Unmarshal(input, &m)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse yaml: %w", err)
+	}
+
+	return m, nil
+}
+
+func parseRoute(input []byte, routes []string) (map[string]string, error) {
+	// 1. parse as map[string]any
+	var m map[string]any
+	err := yaml.Unmarshal(input, &m)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse root yaml: %w", err)
+	}
+
+	// 2. traverse the yaml based on the route. error if any of the routes don't exist
+	for _, route := range routes {
+		value, ok := m[route]
+		if !ok {
+			return nil, fmt.Errorf("failed to locate route: %s within yaml", route)
+		}
+
+		innerMap, ok := value.(map[string]any)
+		if !ok {
+			return nil, fmt.Errorf("route at %s is not a map", route)
+		}
+
+		m = innerMap
+	}
+
+	// 3. Now that we've traversed the route, our `m` value can be marshalled back into yaml,
+	// and then unmarshalled regularly
+	targetBytes, err := yaml.Marshal(m)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal route yaml block: %w", err)
+	}
+
+	return parseNormally(targetBytes)
+}
+
+func parse(yamlData string, routeStr string) (map[string]string, error) {
+	// do all the validation stuff ...
+
+	input := []byte(yamlData)
+
+	routeStr = strings.TrimSpace(routeStr)
+	if routeStr == "" {
+		// No route provided; parse the root YAML as the labels map.
+		return parseNormally(input)
+	}
+
+	// Split routes and drop any empty segments (e.g. leading/trailing dots).
+	routes := strings.Split(routeStr, ".")
+
+	// no routes, just parse yaml as is
+	if len(routes) == 0 {
+		return parseNormally(input)
+	}
+
+	// when there are empty segments error out
+	// Eg: external..labels
+	for _, r := range routes {
+		if r == "" {
+			return nil, fmt.Errorf("invalid route %q: empty segment found", routeStr)
+		}
+	}
+
+	// parse with routes
+	return parseRoute(input, routes)
+
+}

core/pkg/external/configmapsource_test.go

@@ -0,0 +1,424 @@
+package external
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// newSource returns a ConfigMapSource for the given config.
+func newSource(cfg *Config) *ConfigMapSource {
+	return &ConfigMapSource{cfg: cfg}
+}
+
+func TestConfigMapSource_NilConfig(t *testing.T) {
+	s := newSource(nil)
+
+	res, err := s.ExtractNodeLabels(map[string]string{})
+
+	require.Error(t, err)
+	assert.Nil(t, res)
+	assert.EqualError(t, err, "nil config")
+}
+
+// --- traditional ConfigMap (Key and Route both empty) ---
+
+func TestConfigMapSource_Traditional_PassesDataDirectly(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("my-cm", "", "", "")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	res, err := s.ExtractNodeLabels(map[string]string{"env": "prod", "region": "us-east-1"})
+	require.NoError(t, err)
+	assert.Equal(t, map[string]string{"env": "prod", "region": "us-east-1"}, res)
+}
+
+func TestConfigMapSource_Traditional_EmptyData(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("my-cm", "", "", "")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	res, err := s.ExtractNodeLabels(map[string]string{})
+	require.NoError(t, err)
+	assert.Empty(t, res)
+}
+
+// --- block-scalar ConfigMap ---
+
+func TestConfigMapSource_BlockScalar_EmptyLabelsMap(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("my-cm", "", "config.yaml", "labels")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	yamlData := `
+labels: {}
+`
+
+	res, err := s.ExtractNodeLabels(map[string]string{
+		"config.yaml": yamlData,
+	})
+
+	require.NoError(t, err)
+	assert.Empty(t, res)
+	assert.NotNil(t, res)
+}
+
+const prometheusConfig = `
+prometheusK8s:
+  externalLabels:
+    cluster: prod-cluster
+    region: eu-west-1
+`
+
+func TestConfigMapSource_BlockScalar_SingleLevel(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("prometheus-cm", "", "config.yaml", "prometheusK8s.externalLabels")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	res, err := s.ExtractNodeLabels(map[string]string{"config.yaml": prometheusConfig})
+	require.NoError(t, err)
+	assert.Equal(t, map[string]string{
+		"cluster": "prod-cluster",
+		"region":  "eu-west-1",
+	}, res)
+}
+
+func TestConfigMapSource_BlockScalar_TopLevelRoute(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("my-cm", "", "data", "labels")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	yaml := `
+labels:
+  team: platform
+  env: staging
+`
+	res, err := s.ExtractNodeLabels(map[string]string{"data": yaml})
+	require.NoError(t, err)
+	assert.Equal(t, map[string]string{"team": "platform", "env": "staging"}, res)
+}
+
+// --- error cases ---
+
+func TestConfigMapSource_BlockScalar_EmptyMapDataInConfigYaml(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("my-cm", "", "config.yaml", "labels")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	res, err := s.ExtractNodeLabels(map[string]string{
+		"config.yaml": "",
+	})
+
+	require.Error(t, err)
+	assert.Nil(t, res)
+	assert.Contains(t, err.Error(), "failed to locate route")
+}
+
+func TestConfigMapSource_BlockScalar_MissingKey(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("my-cm", "", "config.yaml", "externalLabels")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	_, err := s.ExtractNodeLabels(map[string]string{"other-key": "value"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), `key "config.yaml" not found`)
+}
+
+func TestConfigMapSource_BlockScalar_InvalidYAML(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("my-cm", "", "config.yaml", "labels")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	_, err := s.ExtractNodeLabels(map[string]string{"config.yaml": ":\tinvalid: yaml: {"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "found character that cannot start any token")
+}
+
+func TestConfigMapSource_BlockScalar_RouteSegmentNotFound(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("my-cm", "", "config.yaml", "does.not.exist")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	_, err := s.ExtractNodeLabels(map[string]string{"config.yaml": "foo: bar\n"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), `failed to locate route`)
+}
+
+func TestConfigMapSource_BlockScalar_RouteSegmentNotANodeSequenceType(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("my-cm", "", "config.yaml", "labels.nested")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	// labels is a string, not a map — traversing into it should fail
+	_, err := s.ExtractNodeLabels(map[string]string{"config.yaml": "labels: just-a-string\n"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "is not a map")
+}
+
+func TestConfigMapSource_BlockScalar_RoutePointsToNodeScalarType(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("my-cm", "", "config.yaml", "labels")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	// labels is a string, not a map — final conversion should fail
+	_, err := s.ExtractNodeLabels(map[string]string{"config.yaml": "labels: just-a-string\n"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "is not a map")
+}
+
+func TestConfigMapSource_BlockScalar_BoolValuesConvertedToString(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("my-cm", "", "config.yaml", "labels")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	yaml := `
+labels:
+  active: true
+  deprecated: false
+`
+	res, err := s.ExtractNodeLabels(map[string]string{"config.yaml": yaml})
+	require.NoError(t, err)
+	assert.Equal(t, "true", res["active"])
+	assert.Equal(t, "false", res["deprecated"])
+}
+
+func TestConfigMapSource_BlockScalar_IntValuesConvertedToString(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("my-cm", "", "config.yaml", "labels")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	yaml := `
+labels:
+  priority: 42
+  replicas: 3
+`
+	res, err := s.ExtractNodeLabels(map[string]string{"config.yaml": yaml})
+	require.NoError(t, err)
+	assert.Equal(t, "42", res["priority"])
+	assert.Equal(t, "3", res["replicas"])
+}
+
+func TestConfigMapSource_BlockScalar_SequenceValueRejected(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("my-cm", "", "config.yaml", "labels")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	yamlData := `
+labels:
+  environments:
+    - dev
+    - staging
+    - prod
+`
+
+	res, err := s.ExtractNodeLabels(map[string]string{
+		"config.yaml": yamlData,
+	})
+
+	require.Error(t, err)
+	assert.Nil(t, res)
+	assert.Contains(t, err.Error(), `cannot unmarshal !!seq into string`)
+}
+
+func TestConfigMapSource_BlockScalar_NestedMapValueRejected(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("my-cm", "", "config.yaml", "labels")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	yamlData := `
+labels:
+  inner-map:
+    my-key: value
+    other-key: value
+`
+
+	res, err := s.ExtractNodeLabels(map[string]string{
+		"config.yaml": yamlData,
+	})
+
+	require.Error(t, err)
+	assert.Nil(t, res)
+	assert.Contains(t, err.Error(), `unmarshal !!map into string`)
+}
+
+func TestConfigMapSource_BlockScalar_NullValues(t *testing.T) {
+	tests := []struct {
+		name      string
+		nullValue string
+	}{
+		{
+			name:      "explicit null",
+			nullValue: "null",
+		},
+		{
+			name:      "tilde",
+			nullValue: "~",
+		},
+		{
+			name:      "empty value",
+			nullValue: "",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			nlCfg := NewNodeLabelConfig("my-cm", "", "config.yaml", "labels")
+			cfg := NewConfig(nlCfg)
+			s := newSource(cfg)
+			yamlData := fmt.Sprintf(`
+labels:
+  priority: %s
+`, tt.nullValue)
+
+			res, err := s.ExtractNodeLabels(map[string]string{
+				"config.yaml": yamlData,
+			})
+
+			require.NoError(t, err)
+			assert.Equal(t, "", res["priority"])
+		})
+	}
+}
+
+func TestConfigMapSource_BlockScalar_AliasValue(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("my-cm", "", "config.yaml", "labels")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	yamlData := `
+defaultEnvironment: &defaultEnvironment production
+
+labels:
+  environment: *defaultEnvironment
+`
+
+	res, err := s.ExtractNodeLabels(map[string]string{
+		"config.yaml": yamlData,
+	})
+
+	require.NoError(t, err)
+	assert.Equal(t, "production", res["environment"])
+}
+
+func TestConfigMapSource_BlockScalar_ComplexYamlWithTwoUnsupportedNodeType(t *testing.T) {
+	nlCfg := NewNodeLabelConfig("my-cm", "", "config.yaml", "labels")
+	cfg := NewConfig(nlCfg)
+	s := newSource(cfg)
+
+	yaml := `
+labels:
+  name: complex
+  priority: 42
+  replicas: 3
+  other:
+    - "hello"
+    - "this"
+    - "is"
+    - "a"
+    - "test"
+  inner-map:
+    my-key: "value"
+    other-key: "value"
+`
+
+	res, err := s.ExtractNodeLabels(map[string]string{"config.yaml": yaml})
+
+	require.Error(t, err)
+	assert.Nil(t, res)
+}
+
+// --- filterValidLabels ---
+
+func TestFilterValidLabels_AllValid_ReturnedUnchanged(t *testing.T) {
+	in := map[string]string{
+		"env":                     "prod",
+		"region":                  "us-east-1",
+		"app.kubernetes.io/name":  "opencost",
+		"my-key":                  "my-value",
+		"a":                       "b",
+	}
+	out := filterValidLabels(in)
+	assert.Equal(t, in, out)
+}
+
+func TestFilterValidLabels_EmptyInput_ReturnsEmptyMap(t *testing.T) {
+	out := filterValidLabels(map[string]string{})
+	assert.NotNil(t, out)
+	assert.Empty(t, out)
+}
+
+func TestFilterValidLabels_InvalidKey_EntryDropped(t *testing.T) {
+	in := map[string]string{
+		"valid-key":  "value",
+		"":           "empty-key-dropped",
+		"-bad-start": "dropped",
+	}
+	out := filterValidLabels(in)
+	assert.Equal(t, map[string]string{"valid-key": "value"}, out)
+}
+
+func TestFilterValidLabels_InvalidValue_EntryDropped(t *testing.T) {
+	in := map[string]string{
+		"good-key":  "good-value",
+		"bad-value": "-starts-with-dash",
+	}
+	out := filterValidLabels(in)
+	assert.Equal(t, map[string]string{"good-key": "good-value"}, out)
+}
+
+func TestFilterValidLabels_ValueTooLong_EntryDropped(t *testing.T) {
+	longValue := string(make([]byte, 64))
+	for i := range longValue {
+		longValue = longValue[:i] + "a" + longValue[i+1:]
+	}
+	in := map[string]string{
+		"ok":      "fine",
+		"toolong": longValue,
+	}
+	out := filterValidLabels(in)
+	assert.Equal(t, map[string]string{"ok": "fine"}, out)
+}
+
+func TestFilterValidLabels_KeyTooLong_EntryDropped(t *testing.T) {
+	longKey := string(make([]byte, 64))
+	for i := range longKey {
+		longKey = longKey[:i] + "a" + longKey[i+1:]
+	}
+	in := map[string]string{
+		"ok":    "fine",
+		longKey: "value",
+	}
+	out := filterValidLabels(in)
+	assert.Equal(t, map[string]string{"ok": "fine"}, out)
+}
+
+func TestFilterValidLabels_PrefixedKey_Valid(t *testing.T) {
+	in := map[string]string{
+		"app.kubernetes.io/name":      "opencost",
+		"example.com/env":             "staging",
+	}
+	out := filterValidLabels(in)
+	assert.Equal(t, in, out)
+}
+
+func TestFilterValidLabels_PrefixedKey_InvalidPrefix_EntryDropped(t *testing.T) {
+	in := map[string]string{
+		"valid":          "kept",
+		"-bad.prefix/k": "dropped",
+	}
+	out := filterValidLabels(in)
+	assert.Equal(t, map[string]string{"valid": "kept"}, out)
+}
+
+func TestFilterValidLabels_MixedValidAndInvalid_OnlyValidReturned(t *testing.T) {
+	in := map[string]string{
+		"cluster":    "prod",
+		"":           "no-key",
+		"bad-value":  "-oops",
+		"region":     "eu-west-1",
+	}
+	out := filterValidLabels(in)
+	assert.Equal(t, map[string]string{"cluster": "prod", "region": "eu-west-1"}, out)
+}

core/pkg/external/configs.go

@@ -0,0 +1,73 @@
+package external
+
+// Config stores the configuration for external labels.
+// Currently, only NodeLabelConfig is supported, and ConfigMaps
+// are the only supported source of external node labels.
+type Config struct {
+	nodeLabelConfig *NodeLabelConfig
+}
+
+func NewConfig(nodeCfg *NodeLabelConfig) *Config {
+	return &Config{
+		nodeLabelConfig: nodeCfg,
+	}
+}
+
+// NodeLabelConfig returns the node label configuration.
+// It returns nil if node label configuration is not provided.
+func (c *Config) NodeLabelConfig() *NodeLabelConfig {
+	if c == nil {
+		return nil
+	}
+	return c.nodeLabelConfig
+}
+
+// HasNodeLabelConfig reports whether node labels are configured.
+func (c *Config) HasNodeLabelConfig() bool {
+	return c != nil && c.nodeLabelConfig != nil
+}
+
+// NodeLabelConfig identifies a ConfigMap to watch and describes how to extract
+// labels from its data. ConfigMapName is required; all other fields are optional.
+// Set Key and Route only when labels are embedded inside a YAML document
+// stored as a block-scalar value; leave both empty for a flat key/value ConfigMap.
+type NodeLabelConfig struct {
+	// configMapName is the name of the ConfigMap to watch.
+	configMapName string
+	// namespace is the namespace of the ConfigMap. Defaults to the agent's own namespace when empty.
+	namespace string
+	// key is the ConfigMap data key that holds the YAML document (block-scalar ConfigMaps only).
+	key string
+	// route is the dot-separated path to the labels map within the parsed YAML document.
+	route string
+}
+
+func NewNodeLabelConfig(
+	configMapName string,
+	namespace string,
+	key string,
+	route string,
+) *NodeLabelConfig {
+	return &NodeLabelConfig{
+		configMapName: configMapName,
+		namespace:     namespace,
+		key:           key,
+		route:         route,
+	}
+}
+
+func (nlc *NodeLabelConfig) ConfigMapName() string {
+	return nlc.configMapName
+}
+
+func (nlc *NodeLabelConfig) Namespace() string {
+	return nlc.namespace
+}
+
+func (nlc *NodeLabelConfig) Key() string {
+	return nlc.key
+}
+
+func (nlc *NodeLabelConfig) Route() string {
+	return nlc.route
+}

core/pkg/external/nodelabelprovider.go

@@ -0,0 +1,40 @@
+package external
+
+import (
+	"maps"
+	"sync"
+
+	"github.com/opencost/opencost/core/pkg/log"
+)
+
+// NodeLabelProvider maintains a key/value map of external labels sourced from any
+// watcher function. It is intended to be wired up to a WatchFunc such as ConfigMapWatcher
+// the caller is responsible for registering it.
+type NodeLabelProvider struct {
+	mu     sync.RWMutex
+	labels map[string]string
+}
+
+// NewNodeLabelProvider creates a NodeLabelProvider with an empty label cache.
+func NewNodeLabelProvider() *NodeLabelProvider {
+	return &NodeLabelProvider{
+		labels: make(map[string]string),
+	}
+}
+
+// Update replaces the cached labels with the full contents of any source of data.
+func (nlp *NodeLabelProvider) Update(name string, data map[string]string) error {
+	nlp.mu.Lock()
+	defer nlp.mu.Unlock()
+	// Clone to avoid retaining a reference to a map that may be mutated by the caller.
+	nlp.labels = maps.Clone(data)
+	log.Debugf("External: NodeLabelProvider: updated %d label(s) %s", len(data), name)
+	return nil
+}
+
+// Labels returns a copy of the currently cached external labels.
+func (nlp *NodeLabelProvider) Labels() (map[string]string, error) {
+	nlp.mu.RLock()
+	defer nlp.mu.RUnlock()
+	return maps.Clone(nlp.labels), nil
+}

core/pkg/external/nodelabelprovider_test.go

@@ -0,0 +1,44 @@
+package external
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNodeLabelProvider_Labels(t *testing.T) {
+	p := NewNodeLabelProvider()
+
+	require.NoError(t, p.Update("external-labels", map[string]string{
+		"cluster": "1de3e77b-266d-48c1-91cb-ec5e22902af7",
+		"env":     "dev",
+		"region":  "nam",
+	}))
+
+	labels, err := p.Labels()
+	require.NoError(t, err)
+	assert.Equal(t, "1de3e77b-266d-48c1-91cb-ec5e22902af7", labels["cluster"])
+	assert.Equal(t, "dev", labels["env"])
+	assert.Equal(t, "nam", labels["region"])
+}
+
+func TestNodeLabelProvider_EmptyOnNoUpdates(t *testing.T) {
+	p := NewNodeLabelProvider()
+
+	labels, err := p.Labels()
+	require.NoError(t, err)
+	assert.Empty(t, labels)
+}
+
+func TestNodeLabelProvider_UpdateDropsRemovedKeys(t *testing.T) {
+	p := NewNodeLabelProvider()
+
+	require.NoError(t, p.Update("cm", map[string]string{"a": "1", "b": "2"}))
+	// second update removes "b" — the whole map is replaced
+	require.NoError(t, p.Update("cm", map[string]string{"a": "1"}))
+
+	labels, err := p.Labels()
+	require.NoError(t, err)
+	assert.Equal(t, map[string]string{"a": "1"}, labels)
+}

core/pkg/external/provider.go

@@ -0,0 +1,6 @@
+package external
+
+type LabelProvider interface {
+	Update(name string, data map[string]string) error
+	Labels() (map[string]string, error)
+}

core/pkg/external/source.go

@@ -0,0 +1,50 @@
+package external
+
+import "fmt"
+
+type LabelSource interface {
+	ExtractNodeLabels(map[string]string) (map[string]string, error)
+}
+
+func NewLabelSource(cfg *Config) (LabelSource, error) {
+	if cfg == nil {
+		return nil, fmt.Errorf("nil config")
+	}
+
+	if !cfg.HasNodeLabelConfig() {
+		return nil, fmt.Errorf("no supported external label config")
+	}
+
+	nlConfig := cfg.NodeLabelConfig()
+
+	if nlConfig.ConfigMapName() != "" {
+		return &ConfigMapSource{
+			cfg: cfg,
+		}, nil
+	}
+
+	return nil, fmt.Errorf("no label source configured")
+}
+
+// WatchFunc bridges a LabelSource and a LabelProvider as a watcher callback.
+// It returns a func(string, map[string]string) error that passes the raw source
+// data through src.ExtractNodeLabels and forwards the resulting labels to provider.Update.
+func WatchFunc(src LabelSource, provider LabelProvider) func(string, map[string]string) error {
+	if src == nil {
+		return func(string, map[string]string) error {
+			return fmt.Errorf("nil LabelSource")
+		}
+	}
+	if provider == nil {
+		return func(string, map[string]string) error {
+			return fmt.Errorf("nil LabelProvider")
+		}
+	}
+	return func(name string, data map[string]string) error {
+		labels, err := src.ExtractNodeLabels(data)
+		if err != nil {
+			return err
+		}
+		return provider.Update(name, labels)
+	}
+}

core/pkg/external/source_test.go

@@ -0,0 +1,122 @@
+package external
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// --- test doubles ---
+
+// stubLabelSource is a LabelSource whose behaviour is controlled by the test.
+type stubLabelSource struct {
+	labels map[string]string
+	err    error
+}
+
+func (s *stubLabelSource) ExtractNodeLabels(_ map[string]string) (map[string]string, error) {
+	return s.labels, s.err
+}
+
+// stubLabelProvider records the last Update call.
+type stubLabelProvider struct {
+	name   string
+	labels map[string]string
+	err    error
+}
+
+func (p *stubLabelProvider) Update(name string, data map[string]string) error {
+	p.name = name
+	p.labels = data
+	return p.err
+}
+
+func (p *stubLabelProvider) Labels() (map[string]string, error) {
+	return p.labels, nil
+}
+
+// --- WatchFunc tests ---
+
+func TestWatchFunc_NilSource_ReturnsError(t *testing.T) {
+	provider := &stubLabelProvider{}
+	fn := WatchFunc(nil, provider)
+
+	err := fn("cm", map[string]string{"k": "v"})
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "nil LabelSource")
+}
+
+func TestWatchFunc_NilProvider_ReturnsError(t *testing.T) {
+	src := &stubLabelSource{labels: map[string]string{"k": "v"}}
+	fn := WatchFunc(src, nil)
+
+	err := fn("cm", map[string]string{"k": "v"})
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "nil LabelProvider")
+}
+
+func TestWatchFunc_HappyPath_LabelsForwardedToProvider(t *testing.T) {
+	src := &stubLabelSource{labels: map[string]string{"cluster": "prod", "region": "us-east-1"}}
+	provider := &stubLabelProvider{}
+
+	fn := WatchFunc(src, provider)
+	err := fn("my-cm", map[string]string{"raw": "data"})
+
+	require.NoError(t, err)
+	assert.Equal(t, "my-cm", provider.name)
+	assert.Equal(t, map[string]string{"cluster": "prod", "region": "us-east-1"}, provider.labels)
+}
+
+func TestWatchFunc_SourceExtractError_PropagatesError(t *testing.T) {
+	src := &stubLabelSource{err: fmt.Errorf("extract failed")}
+	provider := &stubLabelProvider{}
+
+	fn := WatchFunc(src, provider)
+	err := fn("cm", map[string]string{})
+
+	require.Error(t, err)
+	assert.EqualError(t, err, "extract failed")
+	// provider.Update must not have been called
+	assert.Nil(t, provider.labels)
+}
+
+func TestWatchFunc_ProviderUpdateError_PropagatesError(t *testing.T) {
+	src := &stubLabelSource{labels: map[string]string{"env": "dev"}}
+	provider := &stubLabelProvider{err: fmt.Errorf("update failed")}
+
+	fn := WatchFunc(src, provider)
+	err := fn("cm", map[string]string{})
+
+	require.Error(t, err)
+	assert.EqualError(t, err, "update failed")
+}
+
+func TestWatchFunc_EmptyLabels_ForwardedToProvider(t *testing.T) {
+	src := &stubLabelSource{labels: map[string]string{}}
+	provider := &stubLabelProvider{}
+
+	fn := WatchFunc(src, provider)
+	err := fn("cm", map[string]string{})
+
+	require.NoError(t, err)
+	assert.Empty(t, provider.labels)
+}
+
+func TestWatchFunc_ReturnedFuncCalledMultipleTimes_ProviderUpdatedEachTime(t *testing.T) {
+	src := &stubLabelSource{}
+	provider := &stubLabelProvider{}
+
+	fn := WatchFunc(src, provider)
+
+	src.labels = map[string]string{"a": "1"}
+	require.NoError(t, fn("cm", map[string]string{}))
+	assert.Equal(t, map[string]string{"a": "1"}, provider.labels)
+
+	src.labels = map[string]string{"b": "2"}
+	require.NoError(t, fn("cm", map[string]string{}))
+	assert.Equal(t, map[string]string{"b": "2"}, provider.labels)
+}

core/pkg/util/maputil/maputil.go

@@ -1,6 +1,8 @@
 package maputil
 
-import "iter"
+import (
+	"iter"
+)
 
 // Map applies a transformation function to each value within a map to get a new map containing the
 // transformed values.

core/pkg/util/promutil/promutil.go

@@ -96,6 +96,49 @@ func KubePrependQualifierToLabels(m map[string]string, qualifier string) ([]stri
 	return keys, values
 }
 
+// Prepends a qualifier string to the keys provided in the m map and returns a new map with the new
+// keys and values
+func KubePrependQualifierToLabelsMap(labels map[string]string, qualifier string) map[string]string {
+	result := make(map[string]string, len(labels))
+	for k, v := range labels {
+		result[qualifier+SanitizeLabelName(k)] = v
+	}
+	return result
+}
+
+// Prepends a qualifier string to the keys provided in the m1 and m2 maps and returns a new map with the new
+// keys and values merged. Any overlapping keys will be replaced by the second map parameter.
+func KubePrependQualifierToLabelsAndMerge(m1, m2 map[string]string, qualifier string) map[string]string {
+	size := len(m1) + len(m2)
+	if size == 0 {
+		return map[string]string{}
+	}
+
+	result := make(map[string]string, size)
+	for k, v := range m1 {
+		result[qualifier+SanitizeLabelName(k)] = v
+	}
+	for k, v := range m2 {
+		result[qualifier+SanitizeLabelName(k)] = v
+	}
+	return result
+}
+
+// Converts two sources of labels into a single map of prometheus labels
+func KubeLabelsToLabelsMerge(m1, m2 map[string]string) map[string]string {
+	return KubePrependQualifierToLabelsAndMerge(m1, m2, "label_")
+}
+
+// Converts kubernetes labels into a map of prometheus labels
+func KubeLabelsToLabelsMap(labels map[string]string) map[string]string {
+	return KubePrependQualifierToLabelsMap(labels, "label_")
+}
+
+// Converts kubernetes labels into a map of prometheus labels
+func KubeAnnotationsToLabelsMap(labels map[string]string) map[string]string {
+	return KubePrependQualifierToLabelsMap(labels, "annotation_")
+}
+
 // Converts kubernetes labels into prometheus labels.
 func KubeLabelsToLabels(labels map[string]string) ([]string, []string) {
 	return KubePrependQualifierToLabels(labels, "label_")

core/pkg/util/promutil/promutil_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/opencost/opencost/core/pkg/util/json"
+	"github.com/stretchr/testify/assert"
 )
 
 func checkSlice(s1, s2 []string) error {
@@ -227,3 +228,123 @@ func TestClusterInfoLabels(t *testing.T) {
 		}
 	}
 }
+
+func TestPrependQualifierAndMerge(t *testing.T) {
+	m := map[string]string{
+		"a-a":     "A",
+		"b-b.c.d": "B",
+		"cfg-c":   "C",
+	}
+
+	exLabels := map[string]string{
+		"node-type":  "m1.large",
+		"cluster.id": "cluster-a",
+		"some-value": "524.2",
+	}
+
+	expected := map[string]string{
+		"label_a_a":        "A",
+		"label_b_b_c_d":    "B",
+		"label_cfg_c":      "C",
+		"label_cluster_id": "cluster-a",
+		"label_node_type":  "m1.large",
+		"label_some_value": "524.2",
+	}
+
+	result := KubePrependQualifierToLabelsAndMerge(m, exLabels, "label_")
+	for k, v := range expected {
+		val, ok := result[k]
+		if !ok {
+			t.Fatalf("Expected key: %s in result map, but was not found.", k)
+		}
+		if val != v {
+			t.Fatalf("Expected value: %s for key: %s in result map. Got: %s", v, k, val)
+		}
+	}
+}
+
+func TestPrependQualifierToMap(t *testing.T) {
+	m := map[string]string{
+		"a-a":     "A",
+		"b-b.c.d": "B",
+		"cfg-c":   "C",
+	}
+
+	expected := map[string]string{
+		"label_a_a":     "A",
+		"label_b_b_c_d": "B",
+		"label_cfg_c":   "C",
+	}
+
+	result := KubePrependQualifierToLabelsMap(m, "label_")
+	for k, v := range expected {
+		val, ok := result[k]
+		if !ok {
+			t.Fatalf("Expected key: %s in result map, but was not found.", k)
+		}
+		if val != v {
+			t.Fatalf("Expected value: %s for key: %s in result map. Got: %s", v, k, val)
+		}
+	}
+}
+
+func TestPrependQualifierAndMerge_ExternalAddedToBase(t *testing.T) {
+	base := map[string]string{"node": "worker-1"}
+	external := map[string]string{"region": "us-east-1"}
+
+	got := KubePrependQualifierToLabelsAndMerge(base, external, "label_")
+
+	assert.Equal(t, map[string]string{"label_node": "worker-1", "label_region": "us-east-1"}, got)
+}
+
+func TestMerge_BaseWinsOnConflict(t *testing.T) {
+	base := map[string]string{"region": "from-node"}
+	external := map[string]string{"region": "from-configmap"}
+
+	got := KubePrependQualifierToLabelsAndMerge(external, base, "label_")
+
+	assert.Equal(t, "from-node", got["label_region"])
+}
+
+func TestMerge_EmptyExternal(t *testing.T) {
+	base := map[string]string{"node": "worker-1"}
+	baseWithLabelPrefix := map[string]string{"label_node": "worker-1"}
+	got := KubePrependQualifierToLabelsAndMerge(map[string]string{}, base, "label_")
+
+	assert.Equal(t, baseWithLabelPrefix, got)
+}
+
+func TestMerge_NilExternal(t *testing.T) {
+	base := map[string]string{"node": "worker-1"}
+	baseWithLabelPrefix := map[string]string{"label_node": "worker-1"}
+	var external map[string]string
+	got := KubePrependQualifierToLabelsAndMerge(external, base, "label_")
+
+	assert.Equal(t, baseWithLabelPrefix, got)
+}
+
+func TestMerge_EmptyBase(t *testing.T) {
+	external := map[string]string{"region": "us-east-1"}
+	externalWithLabelPrefix := map[string]string{"label_region": "us-east-1"}
+	got := KubePrependQualifierToLabelsAndMerge(external, map[string]string{}, "label_")
+
+	assert.Equal(t, externalWithLabelPrefix, got)
+}
+
+func TestMerge_BothEmpty(t *testing.T) {
+	got := KubePrependQualifierToLabelsAndMerge(map[string]string{}, map[string]string{}, "label_")
+
+	assert.Empty(t, got)
+}
+
+// TestKubePrependQualifierToLabelsAndMerge proves the original base map is not modified.
+// A naive implementation using `out := base` copies the map header only,
+// so writes to out also mutate the caller's map.
+func TestMerge_DoesNotMutateBase(t *testing.T) {
+	base := map[string]string{"node": "worker-1"}
+	external := map[string]string{"region": "us-east-1"}
+
+	_ = KubePrependQualifierToLabelsAndMerge(external, base, "label_")
+
+	assert.Equal(t, map[string]string{"node": "worker-1"}, base, "Merge must not mutate the base map")
+}

docs/swagger.json

@@ -325,6 +325,170 @@
           }
         }
       }
+    },
+    "/inferenceCost/total": {
+      "get": {
+        "summary": "query for aggregated AI inference costs",
+        "description": "Returns a single aggregated InferenceCostSet covering the full requested window. Costs are broken down per model/namespace with blended and differentiated (input/output) cost-per-million-token rates under the chosen cost basis. Requires `INFERENCE_COST_ENABLED=true`.",
+        "parameters": [
+          {
+            "name": "window",
+            "in": "query",
+            "required": true,
+            "description": "Duration of time over which to query. Accepts durations like `7d`, `24h`, or RFC3339 date pairs like `2024-01-01T00:00:00Z,2024-01-02T00:00:00Z`.",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "7days": {
+                "value": "7d"
+              },
+              "24hours": {
+                "value": "24h"
+              },
+              "range": {
+                "value": "2024-01-01T00:00:00Z,2024-01-08T00:00:00Z"
+              }
+            }
+          },
+          {
+            "name": "costBasis",
+            "in": "query",
+            "description": "`allocation` (default): max(request,usage) × price + idle + shared infra; reconciles to the infrastructure bill. `usage`: actual consumption only; idle and shared infra costs excluded; does not reconcile to the bill.",
+            "schema": {
+              "type": "string",
+              "enum": ["allocation", "usage"],
+              "default": "allocation"
+            }
+          },
+          {
+            "name": "aggregate",
+            "in": "query",
+            "description": "Comma-separated dimensions to aggregate by. Supported values: `model_name`, `model_version`, `namespace`, `cluster`, `pod`, `controller`, `controller_kind`, `container`, `workload_type`.",
+            "schema": {
+              "type": "string"
+            },
+            "example": "model_name"
+          },
+          {
+            "name": "accumulate",
+            "in": "query",
+            "description": "Step size used internally before accumulating into the total. Accepted values: `hour`, `day`, `week`, `month`. Optional for this endpoint.",
+            "schema": {
+              "type": "string",
+              "enum": ["hour", "day", "week", "month"]
+            }
+          },
+          {
+            "name": "filter",
+            "in": "query",
+            "description": "Filter results by property values. Format: `prop:value` for a single filter, `prop:value+prop:value` for AND logic. Supported properties: `model_name`, `model_version`, `namespace`, `cluster`, `pod`, `controller`, `controller_kind`, `container`, `workload_type`.",
+            "schema": {
+              "type": "string"
+            },
+            "example": "namespace:llm-d-prod+model_name:Qwen/Qwen3-32B"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Success",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/InferenceCostSetResponse"
+                }
+              }
+            }
+          },
+          "400": {
+            "description": "Bad request — missing or invalid parameters"
+          },
+          "501": {
+            "description": "Inference cost tracking is not enabled (`INFERENCE_COST_ENABLED` is not set to `true`)"
+          }
+        }
+      }
+    },
+    "/inferenceCost/timeseries": {
+      "get": {
+        "summary": "query for AI inference costs as a time series",
+        "description": "Returns one InferenceCostSet per time step within the requested window. The `accumulate` parameter is required and defines the step size. All other parameters are identical to `/inferenceCost/total`. Requires `INFERENCE_COST_ENABLED=true`.",
+        "parameters": [
+          {
+            "name": "window",
+            "in": "query",
+            "required": true,
+            "description": "Duration of time over which to query. Accepts durations like `7d`, `24h`, or RFC3339 date pairs.",
+            "schema": {
+              "type": "string"
+            },
+            "examples": {
+              "7days": {
+                "value": "7d"
+              },
+              "range": {
+                "value": "2024-01-01T00:00:00Z,2024-01-08T00:00:00Z"
+              }
+            }
+          },
+          {
+            "name": "accumulate",
+            "in": "query",
+            "required": true,
+            "description": "Step size for each time-series data point. Required for this endpoint.",
+            "schema": {
+              "type": "string",
+              "enum": ["hour", "day", "week", "month"]
+            },
+            "example": "day"
+          },
+          {
+            "name": "costBasis",
+            "in": "query",
+            "description": "`allocation` (default) or `usage`. See `/inferenceCost/total` for details.",
+            "schema": {
+              "type": "string",
+              "enum": ["allocation", "usage"],
+              "default": "allocation"
+            }
+          },
+          {
+            "name": "aggregate",
+            "in": "query",
+            "description": "Comma-separated dimensions to aggregate by: `model_name`, `model_version`, `namespace`, `cluster`, `pod`, `controller`, `controller_kind`, `container`, `workload_type`.",
+            "schema": {
+              "type": "string"
+            },
+            "example": "model_name"
+          },
+          {
+            "name": "filter",
+            "in": "query",
+            "description": "Filter by property values. Format: `prop:value+prop:value` (AND logic). Supported properties: `model_name`, `model_version`, `namespace`, `cluster`, `pod`, `controller`, `controller_kind`, `container`, `workload_type`.",
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Success",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/InferenceCostSetRangeResponse"
+                }
+              }
+            }
+          },
+          "400": {
+            "description": "Bad request — missing or invalid parameters (including missing `accumulate`)"
+          },
+          "501": {
+            "description": "Inference cost tracking is not enabled (`INFERENCE_COST_ENABLED` is not set to `true`)"
+          }
+        }
+      }
     }
   },
   "components": {

modules/collector-source/pkg/collector/datasource.go

@@ -8,6 +8,7 @@ import (
 	"github.com/opencost/opencost/core/pkg/clustercache"
 	"github.com/opencost/opencost/core/pkg/clusters"
 	"github.com/opencost/opencost/core/pkg/diagnostics"
+	"github.com/opencost/opencost/core/pkg/external"
 	"github.com/opencost/opencost/core/pkg/log"
 	"github.com/opencost/opencost/core/pkg/nodestats"
 	"github.com/opencost/opencost/core/pkg/source"
@@ -32,6 +33,7 @@ func NewDefaultCollectorDataSource(
 	clusterInfoProvider clusters.ClusterInfoProvider,
 	clusterCache clustercache.ClusterCache,
 	statSummaryClient nodestats.StatSummaryClient,
+	externalLabelProvider external.LabelProvider,
 ) source.OpenCostDataSource {
 	config := NewOpenCostCollectorConfigFromEnv(clusterUID)
 	return NewCollectorDataSource(
@@ -40,6 +42,7 @@ func NewDefaultCollectorDataSource(
 		clusterInfoProvider,
 		clusterCache,
 		statSummaryClient,
+		externalLabelProvider,
 	)
 }
 
@@ -49,6 +52,7 @@ func NewCollectorDataSource(
 	clusterInfoProvider clusters.ClusterInfoProvider,
 	clusterCache clustercache.ClusterCache,
 	statSummaryClient nodestats.StatSummaryClient,
+	externalLabelProvider external.LabelProvider,
 ) source.OpenCostDataSource {
 	var resolutions []*util.Resolution
 	for _, resconf := range config.Resolutions {
@@ -100,6 +104,7 @@ func NewCollectorDataSource(
 		clusterInfoProvider,
 		clusterCache,
 		statSummaryClient,
+		externalLabelProvider,
 	)
 	scrapeController.Start()
 

modules/collector-source/pkg/scrape/clustercache.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/kubecost/events"
 	"github.com/opencost/opencost/core/pkg/clustercache"
+	"github.com/opencost/opencost/core/pkg/external"
 	"github.com/opencost/opencost/core/pkg/log"
 	"github.com/opencost/opencost/core/pkg/source"
 	coreutil "github.com/opencost/opencost/core/pkg/util"
@@ -25,12 +26,14 @@ import (
 const unmountedPVsContainer = "unmounted-pvs"
 
 type ClusterCacheScraper struct {
-	clusterCache clustercache.ClusterCache
+	clusterCache          clustercache.ClusterCache
+	externalLabelProvider external.LabelProvider
 }
 
-func newClusterCacheScraper(clusterCache clustercache.ClusterCache) Scraper {
+func newClusterCacheScraper(clusterCache clustercache.ClusterCache, externalLabelProvider external.LabelProvider) Scraper {
 	return &ClusterCacheScraper{
-		clusterCache: clusterCache,
+		clusterCache:          clusterCache,
+		externalLabelProvider: externalLabelProvider,
 	}
 }
 
@@ -84,6 +87,15 @@ func (ccs *ClusterCacheScraper) GetScrapeNodes(nodes []*clustercache.Node) Scrap
 func (ccs *ClusterCacheScraper) scrapeNodes(nodes []*clustercache.Node) []metric.Update {
 	var scrapeResults []metric.Update
 
+	// get external labels
+	var externalLabels map[string]string
+	var err error
+	if ccs.externalLabelProvider != nil {
+		externalLabels, err = ccs.externalLabelProvider.Labels()
+		if err != nil {
+			log.Errorf("failed to get external labels to nodes: %s", err)
+		}
+	}
 	for _, node := range nodes {
 		nodeInfo := map[string]string{
 			source.NodeLabel:       node.Name,
@@ -157,9 +169,13 @@ func (ccs *ClusterCacheScraper) scrapeNodes(nodes []*clustercache.Node) []metric
 			}
 		}
 
-		// node labels
-		labelNames, labelValues := promutil.KubeLabelsToLabels(node.Labels)
-		nodeLabels := util.ToMap(labelNames, labelValues)
+		var nodeLabels map[string]string
+		// Merge external labels into node labels; node labels win on conflict.\
+		if len(externalLabels) > 0 {
+			nodeLabels = promutil.KubeLabelsToLabelsMerge(node.Labels, externalLabels)
+		} else {
+			nodeLabels = promutil.KubeLabelsToLabelsMap(node.Labels)
+		}
 
 		scrapeResults = append(scrapeResults, metric.Update{
 			Name:           metric.KubeNodeLabels,

modules/collector-source/pkg/scrape/clustercache_test.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/opencost/opencost/core/pkg/clustercache"
+	"github.com/opencost/opencost/core/pkg/external"
 	"github.com/opencost/opencost/core/pkg/source"
 	"github.com/opencost/opencost/modules/collector-source/pkg/metric"
 	"github.com/opencost/opencost/modules/collector-source/pkg/util"
@@ -200,6 +201,202 @@ func Test_kubernetesScraper_scrapeNodes(t *testing.T) {
 	}
 }
 
+func Test_kubernetesScraper_scrapeNodesWithExternalLabels(t *testing.T) {
+	start1, _ := time.Parse(time.RFC3339, Start1Str)
+
+	const (
+		testSource             = "mock"
+		testExternalLabelKey   = "externalLabelKey"
+		testExternalLabelValue = "externalLabelValue"
+	)
+	mockLabelProvider := external.NewNodeLabelProvider()
+	err := mockLabelProvider.Update(testSource, map[string]string{testExternalLabelKey: testExternalLabelValue})
+	if err != nil {
+		t.Fatalf("failed to get test node labels: %s", err)
+	}
+	type scrape struct {
+		Nodes     []*clustercache.Node
+		Timestamp time.Time
+	}
+	tests := []struct {
+		name     string
+		scrapes  []scrape
+		expected []metric.Update
+	}{
+		{
+			name: "simple",
+			scrapes: []scrape{
+				{
+					Nodes: []*clustercache.Node{
+						{
+							Name:           "node1",
+							UID:            "uuid1",
+							SpecProviderID: "i-1",
+							Status: v1.NodeStatus{
+								Capacity: v1.ResourceList{
+									v1.ResourceCPU:    resource.MustParse("2"),
+									v1.ResourceMemory: resource.MustParse("2048"),
+								},
+								Allocatable: v1.ResourceList{
+									v1.ResourceCPU:    resource.MustParse("1"),
+									v1.ResourceMemory: resource.MustParse("1024"),
+								},
+							},
+							Labels: map[string]string{
+								"test1": "blah",
+								"test2": "blah2",
+							},
+						},
+					},
+					Timestamp: start1,
+				},
+			},
+			expected: []metric.Update{
+				{
+					Name: metric.NodeInfo,
+					Labels: map[string]string{
+						source.NodeLabel:       "node1",
+						source.ProviderIDLabel: "i-1",
+						source.UIDLabel:        "uuid1",
+					},
+					Value: 0,
+					AdditionalInfo: map[string]string{
+						source.NodeLabel:       "node1",
+						source.ProviderIDLabel: "i-1",
+						source.UIDLabel:        "uuid1",
+					},
+				},
+				{
+					Name: metric.NodeResourceCapacities,
+					Labels: map[string]string{
+						source.NodeLabel:       "node1",
+						source.ProviderIDLabel: "i-1",
+						source.UIDLabel:        "uuid1",
+						source.ResourceLabel:   "cpu",
+						source.UnitLabel:       "core",
+					},
+					Value:          2.0,
+					AdditionalInfo: nil,
+				},
+				{
+					Name: metric.NodeResourceCapacities,
+					Labels: map[string]string{
+						source.NodeLabel:       "node1",
+						source.ProviderIDLabel: "i-1",
+						source.UIDLabel:        "uuid1",
+						source.ResourceLabel:   "memory",
+						source.UnitLabel:       "byte",
+					},
+					Value:          2048.0,
+					AdditionalInfo: nil,
+				},
+				{
+					Name: metric.KubeNodeStatusCapacityCPUCores,
+					Labels: map[string]string{
+						source.NodeLabel:       "node1",
+						source.ProviderIDLabel: "i-1",
+						source.UIDLabel:        "uuid1",
+					},
+					Value:          2.0,
+					AdditionalInfo: nil,
+				},
+				{
+					Name: metric.KubeNodeStatusCapacityMemoryBytes,
+					Labels: map[string]string{
+						source.NodeLabel:       "node1",
+						source.ProviderIDLabel: "i-1",
+						source.UIDLabel:        "uuid1",
+					},
+					Value:          2048.0,
+					AdditionalInfo: nil,
+				},
+				{
+					Name: metric.NodeResourcesAllocatable,
+					Labels: map[string]string{
+						source.NodeLabel:       "node1",
+						source.ProviderIDLabel: "i-1",
+						source.UIDLabel:        "uuid1",
+						source.ResourceLabel:   "cpu",
+						source.UnitLabel:       "core",
+					},
+					Value:          1.0,
+					AdditionalInfo: nil,
+				},
+				{
+					Name: metric.NodeResourcesAllocatable,
+					Labels: map[string]string{
+						source.NodeLabel:       "node1",
+						source.ProviderIDLabel: "i-1",
+						source.UIDLabel:        "uuid1",
+						source.ResourceLabel:   "memory",
+						source.UnitLabel:       "byte",
+					},
+					Value:          1024.0,
+					AdditionalInfo: nil,
+				},
+				{
+					Name: metric.KubeNodeStatusAllocatableCPUCores,
+					Labels: map[string]string{
+						source.NodeLabel:       "node1",
+						source.ProviderIDLabel: "i-1",
+						source.UIDLabel:        "uuid1",
+					},
+					Value:          1.0,
+					AdditionalInfo: nil,
+				},
+				{
+					Name: metric.KubeNodeStatusAllocatableMemoryBytes,
+					Labels: map[string]string{
+						source.NodeLabel:       "node1",
+						source.ProviderIDLabel: "i-1",
+						source.UIDLabel:        "uuid1",
+					},
+					Value:          1024.0,
+					AdditionalInfo: nil,
+				},
+				{
+					Name: metric.KubeNodeLabels,
+					Labels: map[string]string{
+						source.NodeLabel:       "node1",
+						source.ProviderIDLabel: "i-1",
+						source.UIDLabel:        "uuid1",
+					},
+					Value: 0,
+					AdditionalInfo: map[string]string{
+						"label_test1": "blah",
+						"label_test2": "blah2",
+						// need label key with prefix label_ so the decoder does not exclude.
+						"label_" + testExternalLabelKey: testExternalLabelValue,
+					},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ks := &ClusterCacheScraper{
+				externalLabelProvider: mockLabelProvider,
+			}
+			var scrapeResults []metric.Update
+			for _, s := range tt.scrapes {
+				res := ks.scrapeNodes(s.Nodes)
+				scrapeResults = append(scrapeResults, res...)
+			}
+
+			if len(scrapeResults) != len(tt.expected) {
+				t.Errorf("Expected result length of %d, got %d", len(tt.expected), len(scrapeResults))
+			}
+
+			for i, expected := range tt.expected {
+				got := scrapeResults[i]
+				if !reflect.DeepEqual(expected, got) {
+					t.Errorf("Result did not match expected at index %d: got %v, want %v", i, got, expected)
+				}
+			}
+		})
+	}
+}
+
 func Test_kubernetesScraper_scrapeDeployments(t *testing.T) {
 
 	start1, _ := time.Parse(time.RFC3339, Start1Str)

modules/collector-source/pkg/scrape/scrapecontroller.go

@@ -7,6 +7,7 @@ import (
 	"github.com/opencost/opencost/core/pkg/clustercache"
 	"github.com/opencost/opencost/core/pkg/clusters"
 	coreenv "github.com/opencost/opencost/core/pkg/env"
+	"github.com/opencost/opencost/core/pkg/external"
 	"github.com/opencost/opencost/core/pkg/log"
 	"github.com/opencost/opencost/core/pkg/nodestats"
 	"github.com/opencost/opencost/core/pkg/util/atomic"
@@ -81,6 +82,7 @@ func NewScrapeController(
 	clusterInfoProvider clusters.ClusterInfoProvider,
 	clusterCache clustercache.ClusterCache,
 	statSummaryClient nodestats.StatSummaryClient,
+	externalLabelProvider external.LabelProvider,
 ) *ScrapeController {
 	// Start with env-driven defaults, then layer in any caller-supplied entries.
 	filter := getDefaultMetricFilter()
@@ -89,7 +91,7 @@ func NewScrapeController(
 	clusterInfoScrapper := withFilter(newClusterInfoScrapper(clusterUID, clusterInfoProvider), filter)
 	scrapers = append(scrapers, clusterInfoScrapper)
 
-	clusterCacheScraper := withFilter(newClusterCacheScraper(clusterCache), filter)
+	clusterCacheScraper := withFilter(newClusterCacheScraper(clusterCache, externalLabelProvider), filter)
 	scrapers = append(scrapers, clusterCacheScraper)
 
 	opencostScraper := withFilter(newOpenCostScraper(), filter)

pkg/costmodel/router.go

@@ -13,6 +13,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/opencost/opencost/core/pkg/external"
 	"github.com/opencost/opencost/core/pkg/kubeconfig"
 	"github.com/opencost/opencost/core/pkg/nodestats"
 	"github.com/opencost/opencost/core/pkg/protocol"
@@ -484,6 +485,49 @@ func Initialize(router *httprouter.Router, additionalConfigWatchers ...*watcher.
 
 		return ds, e
 	}
+
+	// Append the pricing config watcher
+	installNamespace := env.GetOpencostNamespace()
+
+	configWatchers := watcher.NewConfigMapWatchers(kubeClientset, installNamespace, additionalConfigWatchers...)
+	configWatchers.AddWatcher(provider.ConfigWatcherFor(cloudProvider))
+	configWatchers.AddWatcher(metrics.GetMetricsConfigWatcher())
+
+	// Assign external label provider spec to opencost
+	var elProvider external.LabelProvider
+	var cfg *external.Config
+	externalNodeLabelsCM := env.GetExternalNodeLabelsConfigMapName()
+	if externalNodeLabelsCM != "" {
+		nodeLabelsCfg := external.NewNodeLabelConfig(
+			externalNodeLabelsCM,
+			env.GetExternalNodeLabelsNamespace(),
+			env.GetExternalNodeLabelsKey(),
+			env.GetExternalNodeLabelsRoute(),
+		)
+		cfg = external.NewConfig(nodeLabelsCfg)
+	}
+
+	if cfg != nil {
+		elProvider = external.NewNodeLabelProvider()
+		elSource, err := external.NewLabelSource(cfg)
+		if err != nil {
+			log.Errorf("Failed to create an external Source: %s", err)
+		}
+
+		nlCfg := cfg.NodeLabelConfig()
+		elNamespace := nlCfg.Namespace()
+		// If configmap is in the same namespace as the finops agent we can just use the same configmap watcher.
+		if elNamespace == "" {
+			configWatchers.Add(nlCfg.ConfigMapName(), external.WatchFunc(elSource, elProvider))
+		} else {
+			elWatchers := watcher.NewConfigMapWatchers(kubeClientset, elNamespace)
+			elWatchers.Add(nlCfg.ConfigMapName(), external.WatchFunc(elSource, elProvider))
+			elWatchers.Watch()
+		}
+	}
+
+	configWatchers.Watch()
+
 	if env.IsCollectorDataSourceEnabled() {
 		fn = func() (source.OpenCostDataSource, error) {
 			nodeStatConf, err := NewNodeClientConfigFromEnv()
@@ -501,6 +545,7 @@ func Initialize(router *httprouter.Router, additionalConfigWatchers ...*watcher.
 				clusterInfoProvider,
 				k8sCache,
 				nodeStatClient,
+				elProvider,
 			)
 			return ds, nil
 		}
@@ -518,14 +563,6 @@ func Initialize(router *httprouter.Router, additionalConfigWatchers ...*watcher.
 		panic(fatalErr)
 	}
 
-	// Append the pricing config watcher
-	installNamespace := env.GetOpencostNamespace()
-
-	configWatchers := watcher.NewConfigMapWatchers(kubeClientset, installNamespace, additionalConfigWatchers...)
-	configWatchers.AddWatcher(provider.ConfigWatcherFor(cloudProvider))
-	configWatchers.AddWatcher(metrics.GetMetricsConfigWatcher())
-	configWatchers.Watch()
-
 	clusterMap := dataSource.ClusterMap()
 	settingsCache := cache.New(cache.NoExpiration, cache.NoExpiration)
 

pkg/env/external.go

@@ -0,0 +1,40 @@
+package env
+
+import "github.com/opencost/opencost/core/pkg/env"
+
+// External.NodeLabels environment variables configure the ConfigMap used to read custom node labels.
+// Set EXTERNAL_NODELABELS_CONFIG_MAP_NAME to enable this feature. The namespace defaults to the
+// agent's namespace if not specified.
+// For block-scalar ConfigMaps, set EXTERNAL_NODELABELS_KEY to the data key containing the YAML
+// document and EXTERNAL_NODELABELS_ROUTE to the path within the YAML document that contains the
+// node labels.
+
+const (
+	ExternalNodeLabelsConfigMapNameEnvVar = "EXTERNAL_NODELABELS_CONFIG_MAP_NAME"
+	ExternalNodeLabelsNamespaceEnvVar     = "EXTERNAL_NODELABELS_NAMESPACE"
+	ExternalNodeLabelsKeyEnvVar           = "EXTERNAL_NODELABELS_KEY"
+	ExternalNodeLabelsRouteEnvVar         = "EXTERNAL_NODELABELS_ROUTE"
+)
+
+// GetExternalNodeLabelsConfigMapName returns the name of the ConfigMap that contains the external node labels.
+func GetExternalNodeLabelsConfigMapName() string {
+	return env.Get(ExternalNodeLabelsConfigMapNameEnvVar, "")
+}
+
+// GetExternalNodeLabelsNamespace returns the namespace of the external node labels ConfigMap.
+// An empty string means the agent's own namespace should be used.
+func GetExternalNodeLabelsNamespace() string {
+	return env.Get(ExternalNodeLabelsNamespaceEnvVar, "")
+}
+
+// GetExternalNodeLabelsKey returns the ConfigMap data key that holds the YAML document
+// for block-scalar ConfigMaps. Empty for traditional ConfigMaps.
+func GetExternalNodeLabelsKey() string {
+	return env.Get(ExternalNodeLabelsKeyEnvVar, "")
+}
+
+// GetExternalNodeLabelsRoute returns the dot-separated path to the labels map within
+// the parsed YAML document. Empty for traditional ConfigMaps.
+func GetExternalNodeLabelsRoute() string {
+	return env.Get(ExternalNodeLabelsRouteEnvVar, "")
+}