Merge branch 'develop' of https://github.com/kubecost/cost-model into AjayTripathy-dedupe-warnings

README.md

@@ -27,7 +27,12 @@ Here is a summary of features enabled by this cost model:
 You can deploy Kubecost on any Kubernetes 1.8+ cluster in a matter of minutes, if not seconds. 
 Visit the Kubecost docs for [recommended install options](https://docs.kubecost.com/install). Compared to building from source, installing from Helm is faster and includes all necessary dependencies. 
 
-> If you want to deploy cost model in Prometheus exporter only mode, check out [kubecost-exporter.md](kubecost-exporter.md).
+## Usage
+
+* User interface
+* [Cost APIs](https://github.com/kubecost/docs/blob/master/apis.md)
+* [CLI / kubectl cost](https://github.com/kubecost/kubectl-cost)
+* [Prometheus metric exporter](kubecost-exporter.md)
 
 ## Contributing
 

pkg/env/costmodelenv.go

@@ -72,7 +72,7 @@ const (
 // GetAWSAccessKeyID returns the environment variable value for AWSAccessKeyIDEnvVar which represents
 // the AWS access key for authentication
 func GetAppVersion() string {
-	return Get(AppVersionEnvVar, "1.76.0")
+	return Get(AppVersionEnvVar, "1.77.0")
 }
 
 // IsEmitNamespaceAnnotationsMetric returns true if cost-model is configured to emit the kube_namespace_annotations metric

pkg/kubecost/allocation.go

@@ -772,10 +772,7 @@ func (as *AllocationSet) AggregateBy(properties Properties, options *AllocationA
 		}
 
 		// (5) generate key to use for aggregation-by-key and allocation name
-		key, err := alloc.generateKey(properties)
-		if err != nil {
-			return err
-		}
+		key := alloc.generateKey(properties)
 
 		alloc.Name = key
 		if options.MergeUnallocated && alloc.IsUnallocated() {
@@ -908,10 +905,7 @@ func (as *AllocationSet) AggregateBy(properties Properties, options *AllocationA
 			}
 		}
 		if !skip {
-			key, err := alloc.generateKey(properties)
-			if err != nil {
-				continue
-			}
+			key := alloc.generateKey(properties)
 
 			alloc.Name = key
 			aggSet.Insert(alloc)
@@ -944,7 +938,7 @@ func computeShareCoeffs(properties Properties, options *AllocationAggregationOpt
 	shareType := options.ShareSplit
 
 	// Record allocation values first, then normalize by totals to get percentages
-	for n, alloc := range as.allocations {
+	for _, alloc := range as.allocations {
 		if alloc.IsIdle() {
 			// Skip idle allocations in coefficient calculation
 			continue
@@ -952,10 +946,7 @@ func computeShareCoeffs(properties Properties, options *AllocationAggregationOpt
 
 		// Determine the post-aggregation key under which the allocation will
 		// be shared.
-		name, err := alloc.generateKey(properties)
-		if err != nil {
-			return nil, fmt.Errorf(`failed to generate key for shared allocation "%s": %s`, n, err)
-		}
+		name := alloc.generateKey(properties)
 
 		// If the current allocation will be filtered out in step 3, contribute
 		// its share of the shared coefficient to a "__filtered__" bin, which
@@ -1121,32 +1112,27 @@ func computeIdleCoeffs(properties Properties, options *AllocationAggregationOpti
 	return coeffs, nil
 }
 
-func (a *Allocation) generateKey(properties Properties) (string, error) {
+func (a *Allocation) generateKey(properties Properties) string {
+	if a == nil {
+		return ""
+	}
+
 	// Names will ultimately be joined into a single name, which uniquely
 	// identifies allocations.
 	names := []string{}
 
 	if properties.HasCluster() {
-		cluster, err := a.Properties.GetCluster()
-		if err != nil {
-			return "", err
-		}
+		cluster, _ := a.Properties.GetCluster()
 		names = append(names, cluster)
 	}
 
 	if properties.HasNode() {
-		node, err := a.Properties.GetNode()
-		if err != nil {
-			return "", err
-		}
+		node, _ := a.Properties.GetNode()
 		names = append(names, node)
 	}
 
 	if properties.HasNamespace() {
-		namespace, err := a.Properties.GetNamespace()
-		if err != nil {
-			return "", err
-		}
+		namespace, _ := a.Properties.GetNamespace()
 		names = append(names, namespace)
 	}
 
@@ -1182,20 +1168,12 @@ func (a *Allocation) generateKey(properties Properties) (string, error) {
 	}
 
 	if properties.HasPod() {
-		pod, err := a.Properties.GetPod()
-		if err != nil {
-			return "", err
-		}
-
+		pod, _ := a.Properties.GetPod()
 		names = append(names, pod)
 	}
 
 	if properties.HasContainer() {
-		container, err := a.Properties.GetContainer()
-		if err != nil {
-			return "", err
-		}
-
+		container, _ := a.Properties.GetContainer()
 		names = append(names, container)
 	}
 
@@ -1218,19 +1196,14 @@ func (a *Allocation) generateKey(properties Properties) (string, error) {
 	}
 
 	if properties.HasAnnotations() {
-		annotations, err := a.Properties.GetAnnotations() // annotations that the individual allocation possesses
+		annotations, err := a.Properties.GetAnnotations()
 		if err != nil {
 			// Indicate that allocation has no annotations
 			names = append(names, UnallocatedSuffix)
 		} else {
 			annotationNames := []string{}
 
-			aggAnnotations, err := properties.GetAnnotations() // potential annotations to aggregate on supplied by the API caller
-			if err != nil {
-				// We've already checked HasAnnotation, so this should never occur
-				return "", err
-			}
-			// calvin - support multi-annotation aggregation
+			aggAnnotations, _ := properties.GetAnnotations()
 			for annotationName := range aggAnnotations {
 				if val, ok := annotations[annotationName]; ok {
 					annotationNames = append(annotationNames, fmt.Sprintf("%s=%s", annotationName, val))
@@ -1254,19 +1227,14 @@ func (a *Allocation) generateKey(properties Properties) (string, error) {
 	}
 
 	if properties.HasLabel() {
-		labels, err := a.Properties.GetLabels() // labels that the individual allocation possesses
+		labels, err := a.Properties.GetLabels()
 		if err != nil {
 			// Indicate that allocation has no labels
 			names = append(names, UnallocatedSuffix)
 		} else {
 			labelNames := []string{}
 
-			aggLabels, err := properties.GetLabels() // potential labels to aggregate on supplied by the API caller
-			if err != nil {
-				// We've already checked HasLabel, so this should never occur
-				return "", err
-			}
-			// calvin - support multi-label aggregation
+			aggLabels, _ := properties.GetLabels()
 			for labelName := range aggLabels {
 				if val, ok := labels[labelName]; ok {
 					labelNames = append(labelNames, fmt.Sprintf("%s=%s", labelName, val))
@@ -1289,7 +1257,7 @@ func (a *Allocation) generateKey(properties Properties) (string, error) {
 		}
 	}
 
-	return strings.Join(names, "/"), nil
+	return strings.Join(names, "/")
 }
 
 // TODO:CLEANUP get rid of this

pkg/kubecost/allocation_test.go

@@ -435,6 +435,54 @@ func TestAllocation_MarshalJSON(t *testing.T) {
 	}
 }
 
+func TestAllocationSet_generateKey(t *testing.T) {
+	var alloc *Allocation
+	var key string
+
+	props := Properties{}
+	props.SetCluster("")
+
+	key = alloc.generateKey(props)
+	if key != "" {
+		t.Fatalf("generateKey: expected \"\"; actual \"%s\"", key)
+	}
+
+	alloc = &Allocation{}
+	alloc.Properties = Properties{
+		ClusterProp: "cluster1",
+		LabelProp: map[string]string{
+			"app": "app1",
+			"env": "env1",
+		},
+	}
+
+	key = alloc.generateKey(props)
+	if key != "cluster1" {
+		t.Fatalf("generateKey: expected \"cluster1\"; actual \"%s\"", key)
+	}
+
+	props.SetNamespace("")
+	props.SetLabels(map[string]string{"app": ""})
+
+	key = alloc.generateKey(props)
+	if key != "cluster1//app=app1" {
+		t.Fatalf("generateKey: expected \"cluster1//app=app1\"; actual \"%s\"", key)
+	}
+
+	alloc.Properties = Properties{
+		ClusterProp:   "cluster1",
+		NamespaceProp: "namespace1",
+		LabelProp: map[string]string{
+			"app": "app1",
+			"env": "env1",
+		},
+	}
+	key = alloc.generateKey(props)
+	if key != "cluster1/namespace1/app=app1" {
+		t.Fatalf("generateKey: expected \"cluster1/namespace1/app=app1\"; actual \"%s\"", key)
+	}
+}
+
 func TestNewAllocationSet(t *testing.T) {
 	// TODO niko/etl
 }

pkg/kubecost/asset.go

@@ -123,7 +123,7 @@ type Asset interface {
 //   => nil, err
 //
 // (See asset_test.go for assertions of these examples and more.)
-func AssetToExternalAllocation(asset Asset, aggregateBy []string) (*Allocation, error) {
+func AssetToExternalAllocation(asset Asset, aggregateBy []string, externalLabelsCfg map[string]string) (*Allocation, error) {
 	if asset == nil {
 		return nil, fmt.Errorf("asset is nil")
 	}
@@ -143,9 +143,15 @@ func AssetToExternalAllocation(asset Asset, aggregateBy []string) (*Allocation,
 		// labelName should be derived from the mapping of properties to
 		// label names, unless the aggBy is explicitly a label, in which
 		// case we should pull the label name from the aggBy string.
+		// Unless this matches a special aggregation, as we have that mapping already transformed...
 		labelName := aggBy
+		agglName := aggBy
 		if strings.HasPrefix(aggBy, "label:") {
 			labelName = strings.TrimPrefix(aggBy, "label:")
+			agglName = labelName
+			if v, ok := externalLabelsCfg[labelName]; ok {
+				agglName = v
+			}
 		}
 
 		if labelName == "" {
@@ -155,7 +161,7 @@ func AssetToExternalAllocation(asset Asset, aggregateBy []string) (*Allocation,
 			continue
 		}
 
-		if value := asset.Labels()[labelName]; value != "" {
+		if value := asset.Labels()[agglName]; value != "" {
 			// Valid label value was found for one of the aggregation properties,
 			// so add it to the name.
 			if strings.HasPrefix(aggBy, "label:") {

pkg/kubecost/asset_test.go

@@ -1028,7 +1028,7 @@ func TestAssetToExternalAllocation(t *testing.T) {
 	var alloc *Allocation
 	var err error
 
-	alloc, err = AssetToExternalAllocation(asset, []string{"namespace"})
+	alloc, err = AssetToExternalAllocation(asset, []string{"namespace"}, map[string]string{})
 	if err == nil {
 		t.Fatalf("expected error due to nil asset")
 	}
@@ -1045,15 +1045,16 @@ func TestAssetToExternalAllocation(t *testing.T) {
 	cloud.SetLabels(map[string]string{
 		"namespace": "monitoring",
 		"env":       "prod",
+		"product":   "cost-analyzer",
 	})
 	cloud.Cost = 10.00
 	asset = cloud
 
-	alloc, err = AssetToExternalAllocation(asset, []string{"namespace"})
+	alloc, err = AssetToExternalAllocation(asset, []string{"namespace"}, map[string]string{})
 	if err != nil {
 		t.Fatalf("expected to not error")
 	}
-	alloc, err = AssetToExternalAllocation(asset, nil)
+	alloc, err = AssetToExternalAllocation(asset, nil, map[string]string{})
 	if err == nil {
 		t.Fatalf("expected error due to nil aggregateBy")
 	}
@@ -1081,7 +1082,7 @@ func TestAssetToExternalAllocation(t *testing.T) {
 	//   => nil, err
 
 	// 1) single-prop full match
-	alloc, err = AssetToExternalAllocation(asset, []string{"namespace"})
+	alloc, err = AssetToExternalAllocation(asset, []string{"namespace"}, map[string]string{})
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
@@ -1099,7 +1100,7 @@ func TestAssetToExternalAllocation(t *testing.T) {
 	}
 
 	// 2) multi-prop full match
-	alloc, err = AssetToExternalAllocation(asset, []string{"namespace", "label:env"})
+	alloc, err = AssetToExternalAllocation(asset, []string{"namespace", "label:env"}, map[string]string{})
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
@@ -1120,7 +1121,7 @@ func TestAssetToExternalAllocation(t *testing.T) {
 	}
 
 	// 3) multi-prop partial match
-	alloc, err = AssetToExternalAllocation(asset, []string{"namespace", "label:foo"})
+	alloc, err = AssetToExternalAllocation(asset, []string{"namespace", "label:foo"}, map[string]string{})
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
@@ -1138,10 +1139,19 @@ func TestAssetToExternalAllocation(t *testing.T) {
 	}
 
 	// 3) no match
-	alloc, err = AssetToExternalAllocation(asset, []string{"cluster"})
+	alloc, err = AssetToExternalAllocation(asset, []string{"cluster"}, map[string]string{})
 	if err == nil {
 		t.Fatalf("expected 'no match' error")
 	}
+
+	alloc, err = AssetToExternalAllocation(asset, []string{"namespace", "label:app"}, map[string]string{"app": "product"})
+	if alloc.ExternalCost != 10.00 {
+		t.Fatalf("expected external allocation with ExternalCost %f; got %f", 10.00, alloc.ExternalCost)
+	}
+	if alloc.TotalCost() != 10.00 {
+		t.Fatalf("expected external allocation with TotalCost %f; got %f", 10.00, alloc.TotalCost())
+	}
+
 }
 
 // TODO merge conflict had this: