Merge pull request #1933 from opencost/AjayTripathy-combined-fixes

Ajay tripathy combined fixes

config/invalid.json

@@ -0,0 +1 @@
+{"provider":"base","description":"Default prices based on GCP us-central1","CPU":"0.031611","spotCPU":"0.006655","RAM":"0.004237","spotRAM":"0.000892","GPU":"0.95","spotGPU":"0.308","storage":"0.00005479452","zoneNetworkEgress":"0.01","regionNetworkEgress":"0.01","internetNetworkEgress":"0.12","firstFiveForwardingRulesCost":"","additionalForwardingRuleCost":"","LBIngressDataCost":"","athenaBucketName":"","athenaRegion":"","athenaDatabase":"","athenaTable":"","athenaWorkgroup":"","masterPayerARN":"","customPricesEnabled":"false","defaultIdle":"","azureSubscriptionID":"","azureClientID":"","azureClientSecret":"","azureTenantID":"","azureBillingRegion":"","azureOfferDurableID":"","azureStorageSubscriptionID":"","azureStorageAccount":"","azureStorageAccessKey":"","azureStorageContainer":"","azureContainerPath":"","azureCloud":"","currencyCode":"","discount":"","negotiatedDiscount":"","sharedOverhead":"","clusterName":"","sharedNamespaces":"","sharedLabelNames":"","sharedLabelValues":"","shareTenancyCosts":"true","readOnly":"","editorAccess":"","kubecostToken":"","googleAnalyticsTag":"","excludeProviderID":""}

configs/pricing_schema_pv_storageclass.csv

@@ -0,0 +1,2 @@
+EndTimestamp,InstanceID,Region,AssetClass,InstanceIDField,InstanceType,MarketPriceHourly,Version
+2019-04-17 23:34:22 UTC,storageClass0,,pv,spec.storageClassName,,0.1338,

configs/pricing_schema_special_char.csv

@@ -0,0 +1,2 @@
+EndTimestamp,InstanceID,Region,AssetClass,InstanceIDField,InstanceType,MarketPriceHourly,Version
+2019-04-17 23:34:22 UTC,gke-standard-cluster-1-pool-1-91dc432d-cg69,,node,metadata.labels.<http://metadata.label.servers.com/label|metadata.label.servers.com/label>,,0.1337,

pkg/cloud/provider/csvprovider.go

@@ -308,10 +308,10 @@ func NodeValueFromMapField(m string, n *v1.Node, useRegion bool) string {
 		if mf[1] == "name" {
 			return toReturn + n.Name
 		} else if mf[1] == "labels" {
-			lkey := strings.Join(mf[2:], "")
+			lkey := strings.Join(mf[2:len(mf)], ".")
 			return toReturn + n.Labels[lkey]
 		} else if mf[1] == "annotations" {
-			akey := strings.Join(mf[2:], "")
+			akey := strings.Join(mf[2:len(mf)], ".")
 			return toReturn + n.Annotations[akey]
 		} else {
 			log.Errorf("Unsupported InstanceIDField %s in CSV For Node", m)
@@ -329,10 +329,10 @@ func PVValueFromMapField(m string, n *v1.PersistentVolume) string {
 		if mf[1] == "name" {
 			return n.Name
 		} else if mf[1] == "labels" {
-			lkey := strings.Join(mf[2:], "")
+			lkey := strings.Join(mf[2:len(mf)], "")
 			return n.Labels[lkey]
 		} else if mf[1] == "annotations" {
-			akey := strings.Join(mf[2:], "")
+			akey := strings.Join(mf[2:len(mf)], "")
 			return n.Annotations[akey]
 		} else {
 			log.Errorf("Unsupported InstanceIDField %s in CSV For PV", m)
@@ -346,6 +346,13 @@ func PVValueFromMapField(m string, n *v1.PersistentVolume) string {
 			log.Infof("[ERROR] Unsupported InstanceIDField %s in CSV For PV", m)
 			return ""
 		}
+	} else if len(mf) > 1 && mf[0] == "spec" {
+		if mf[1] == "storageClassName" {
+			return n.Spec.StorageClassName
+		} else {
+			log.Infof("[ERROR] Unsupported InstanceIDField %s in CSV For PV", m)
+			return ""
+		}
 	} else {
 		log.Errorf("Unsupported InstanceIDField %s in CSV For PV", m)
 		return ""

pkg/metrics/kubemetrics.go

@@ -108,6 +108,9 @@ func InitKubeMetrics(clusterCache clustercache.ClusterCache, metricsConfig *Metr
 				metricsConfig:    *metricsConfig,
 			})
 		} else if opts.EmitKubeStateMetricsV1Only {
+			// We still need the kubecost_pv_info metric to look up storageclass on legacy clusters.
+			forceDisabled := []string{"kube_persistentvolume_capacity_bytes", "kube_persistentvolume_status_phase"}
+			metricsConfig.DisabledMetrics = append(metricsConfig.DisabledMetrics, forceDisabled...)
 			prometheus.MustRegister(KubeNodeCollector{
 				KubeClusterCache: clusterCache,
 				metricsConfig:    *metricsConfig,
@@ -120,6 +123,18 @@ func InitKubeMetrics(clusterCache clustercache.ClusterCache, metricsConfig *Metr
 				KubeClusterCache: clusterCache,
 				metricsConfig:    *metricsConfig,
 			})
+			prometheus.MustRegister(KubePVCollector{
+				KubeClusterCache: clusterCache,
+				metricsConfig:    *metricsConfig,
+			})
+		} else {
+			// We still need the kubecost_pv_info metric to look up storageclass on legacy clusters.
+			forceDisabled := []string{"kube_persistentvolume_capacity_bytes", "kube_persistentvolume_status_phase"}
+			metricsConfig.DisabledMetrics = append(metricsConfig.DisabledMetrics, forceDisabled...)
+			prometheus.MustRegister(KubePVCollector{
+				KubeClusterCache: clusterCache,
+				metricsConfig:    *metricsConfig,
+			})
 		}
 	})
 }

test/cloud_test.go

@@ -20,9 +20,10 @@ import (
 )
 
 const (
-	providerIDMap = "spec.providerID"
-	nameMap       = "metadata.name"
-	labelMapFoo   = "metadata.labels.foo"
+	providerIDMap  = "spec.providerID"
+	nameMap        = "metadata.name"
+	labelMapFoo    = "metadata.labels.foo"
+	labelMapFooBar = "metadata.labels.foo.bar"
 )
 
 func TestRegionValueFromMapField(t *testing.T) {
@@ -124,6 +125,38 @@ func TestPVPriceFromCSV(t *testing.T) {
 
 }
 
+func TestPVPriceFromCSVStorageClass(t *testing.T) {
+	nameWant := "pvc-08e1f205-d7a9-4430-90fc-7b3965a18c4d"
+	storageClassWant := "storageclass0"
+	pv := &v1.PersistentVolume{}
+	pv.Name = nameWant
+	pv.Spec.StorageClassName = storageClassWant
+
+	confMan := config.NewConfigFileManager(&config.ConfigFileManagerOpts{
+		LocalConfigPath: "./",
+	})
+
+	wantPrice := "0.1338"
+	c := &provider.CSVProvider{
+		CSVLocation: "../configs/pricing_schema_pv_storageclass.csv",
+		CustomProvider: &provider.CustomProvider{
+			Config: provider.NewProviderConfig(confMan, "../configs/default.json"),
+		},
+	}
+	c.DownloadPricingData()
+	k := c.GetPVKey(pv, make(map[string]string), "")
+	resPV, err := c.PVPricing(k)
+	if err != nil {
+		t.Errorf("Error in NodePricing: %s", err.Error())
+	} else {
+		gotPrice := resPV.Cost
+		if gotPrice != wantPrice {
+			t.Errorf("Wanted price '%s' got price '%s'", wantPrice, gotPrice)
+		}
+	}
+
+}
+
 func TestNodePriceFromCSVWithGPU(t *testing.T) {
 	providerIDWant := "providerid"
 	nameWant := "gke-standard-cluster-1-pool-1-91dc432d-cg69"
@@ -194,6 +227,39 @@ func TestNodePriceFromCSVWithGPU(t *testing.T) {
 
 }
 
+func TestNodePriceFromCSVSpecialChar(t *testing.T) {
+	nameWant := "gke-standard-cluster-1-pool-1-91dc432d-cg69"
+
+	confMan := config.NewConfigFileManager(&config.ConfigFileManagerOpts{
+		LocalConfigPath: "./",
+	})
+
+	n := &v1.Node{}
+	n.Name = nameWant
+	n.Labels = make(map[string]string)
+	n.Labels["<http://metadata.label.servers.com/label|metadata.label.servers.com/label>"] = nameWant
+
+	wantPrice := "0.133700"
+
+	c := &provider.CSVProvider{
+		CSVLocation: "../configs/pricing_schema_special_char.csv",
+		CustomProvider: &provider.CustomProvider{
+			Config: provider.NewProviderConfig(confMan, "../configs/default.json"),
+		},
+	}
+	c.DownloadPricingData()
+	k := c.GetKey(n.Labels, n)
+	resN, err := c.NodePricing(k)
+	if err != nil {
+		t.Errorf("Error in NodePricing: %s", err.Error())
+	} else {
+		gotPrice := resN.Cost
+		if gotPrice != wantPrice {
+			t.Errorf("Wanted price '%s' got price '%s'", wantPrice, gotPrice)
+		}
+	}
+}
+
 func TestNodePriceFromCSV(t *testing.T) {
 	providerIDWant := "providerid"
 	nameWant := "gke-standard-cluster-1-pool-1-91dc432d-cg69"