Merge pull request #129 from kubecost/AjayTripathy-azure-fixes

azure fixes

cloud/awsprovider.go

@@ -204,8 +204,8 @@ var regionToBillingRegionCode = map[string]string{
 	"us-gov-west-1":  "UGW1",
 }
 
-func (aws *AWS) GetLocalStorageCost() (float64, error) {
-	return 0, nil
+func (aws *AWS) GetLocalStorageQuery() (string, error) {
+	return "", nil
 }
 
 // KubeAttrConversion maps the k8s labels for region to an aws region
@@ -744,15 +744,15 @@ func (aws *AWS) NodePricing(k Key) (*Node, error) {
 	}
 }
 
-// ClusterName returns an object that represents the cluster. TODO: actually return the name of the cluster. Blocked on cluster federation.
-func (awsProvider *AWS) ClusterName() ([]byte, error) {
+// ClusterInfo returns an object that represents the cluster. TODO: actually return the name of the cluster. Blocked on cluster federation.
+func (awsProvider *AWS) ClusterInfo() (map[string]string, error) {
 	defaultClusterName := "AWS Cluster #1"
-	makeStructure := func(clusterName string) ([]byte, error) {
+	makeStructure := func(clusterName string) (map[string]string, error) {
 		klog.V(2).Infof("Returning \"%s\" as ClusterName", clusterName)
 		m := make(map[string]string)
 		m["name"] = clusterName
 		m["provider"] = "AWS"
-		return json.Marshal(m)
+		return m, nil
 	}
 
 	maybeClusterId := os.Getenv(ClusterIdEnvVar)

cloud/azureprovider.go

@@ -43,8 +43,8 @@ var (
 		"za": "southafrica",
 	}
 
-	// mtBasic, _     = regexp.Compile("^BASIC.A\\d+[_Promo]*$")
-	// mtStandardA, _ = regexp.Compile("^A\\d+[_Promo]*$")
+	mtBasic, _     = regexp.Compile("^BASIC.A\\d+[_Promo]*$")
+	mtStandardA, _ = regexp.Compile("^A\\d+[_Promo]*$")
 	mtStandardB, _ = regexp.Compile(`^Standard_B\d+m?[_v\d]*[_Promo]*$`)
 	mtStandardD, _ = regexp.Compile(`^Standard_D\d[_v\d]*[_Promo]*$`)
 	mtStandardE, _ = regexp.Compile(`^Standard_E\d+i?[_v\d]*[_Promo]*$`)
@@ -342,68 +342,70 @@ func (az *Azure) DownloadPricingData() error {
 	baseCPUPrice := c.CPU
 
 	for _, v := range *result.Meters {
-		region, err := toRegionID(*v.MeterRegion, regions)
-		if err != nil {
-			continue
-		}
+		if !strings.Contains(*v.MeterSubCategory, "Windows") {
 
-		meterName := *v.MeterName
-		sc := *v.MeterSubCategory
-
-		// not available now
-		if strings.Contains(sc, "Promo") {
-			continue
-		}
+			region, err := toRegionID(*v.MeterRegion, regions)
+			if err != nil {
+				continue
+			}
 
-		usageType := ""
-		if !strings.Contains(meterName, "Low Priority") {
-			usageType = "ondemand"
-		} else {
-			usageType = "preemptible"
-		}
+			meterName := *v.MeterName
+			sc := *v.MeterSubCategory
 
-		var instanceTypes []string
-		name := strings.TrimSuffix(meterName, " Low Priority")
-		instanceType := strings.Split(name, "/")
-		for _, it := range instanceType {
-			instanceTypes = append(instanceTypes, strings.Replace(it, " ", "_", 1))
-		}
+			// not available now
+			if strings.Contains(sc, "Promo") {
+				continue
+			}
 
-		instanceTypes = transformMachineType(sc, instanceTypes)
-		if strings.Contains(name, "Expired") {
-			instanceTypes = []string{}
-		}
+			usageType := ""
+			if !strings.Contains(meterName, "Low Priority") {
+				usageType = "ondemand"
+			} else {
+				usageType = "preemptible"
+			}
 
-		var priceInUsd float64
+			var instanceTypes []string
+			name := strings.TrimSuffix(meterName, " Low Priority")
+			instanceType := strings.Split(name, "/")
+			for _, it := range instanceType {
+				instanceTypes = append(instanceTypes, strings.Replace(it, " ", "_", 1))
+			}
 
-		if len(v.MeterRates) < 1 {
-			klog.V(1).Infof("missing rate info %+v", map[string]interface{}{"MeterSubCategory": *v.MeterSubCategory, "region": region})
-			continue
-		}
-		for _, rate := range v.MeterRates {
-			priceInUsd += *rate
-		}
-		priceStr := fmt.Sprintf("%f", priceInUsd)
-		for _, instanceType := range instanceTypes {
-			klog.V(1).Infof("region: %s \n", region)
-			key := fmt.Sprintf("%s,%s,%s", region, instanceType, usageType)
-			allPrices[key] = &Node{
-				Cost:         priceStr,
-				BaseCPUPrice: baseCPUPrice,
+			instanceTypes = transformMachineType(sc, instanceTypes)
+			if strings.Contains(name, "Expired") {
+				instanceTypes = []string{}
 			}
 
-			mts := getMachineTypeVariants(instanceType)
-			for _, mt := range mts {
-				key := fmt.Sprintf("%s,%s,%s", region, mt, usageType)
+			var priceInUsd float64
+
+			if len(v.MeterRates) < 1 {
+				klog.V(1).Infof("missing rate info %+v", map[string]interface{}{"MeterSubCategory": *v.MeterSubCategory, "region": region})
+				continue
+			}
+			for _, rate := range v.MeterRates {
+				priceInUsd += *rate
+			}
+			priceStr := fmt.Sprintf("%f", priceInUsd)
+			for _, instanceType := range instanceTypes {
+				klog.V(1).Infof("region: %s \n", region)
+				key := fmt.Sprintf("%s,%s,%s", region, instanceType, usageType)
 				allPrices[key] = &Node{
 					Cost:         priceStr,
 					BaseCPUPrice: baseCPUPrice,
 				}
-			}
 
+				mts := getMachineTypeVariants(instanceType)
+				for _, mt := range mts {
+					key := fmt.Sprintf("%s,%s,%s", region, mt, usageType)
+					allPrices[key] = &Node{
+						Cost:         priceStr,
+						BaseCPUPrice: baseCPUPrice,
+					}
+				}
+
+			}
 		}
 	}
-
 	az.allPrices = allPrices
 	return nil
 }
@@ -455,8 +457,12 @@ func (*Azure) GetDisks() ([]byte, error) {
 	return nil, nil
 }
 
-func (az *Azure) ClusterName() ([]byte, error) {
-	return nil, nil
+func (az *Azure) ClusterInfo() (map[string]string, error) {
+	m := make(map[string]string)
+	m["name"] = "Azure Cluster #1"
+	m["provider"] = "azure"
+	return m, nil
+
 }
 
 func (az *Azure) AddServiceKey(url url.Values) error {
@@ -513,6 +519,6 @@ func (az *Azure) PVPricing(PVKey) (*PV, error) {
 	return nil, nil
 }
 
-func (az *Azure) GetLocalStorageCost() (float64, error) {
-	return 0.04, nil
+func (az *Azure) GetLocalStorageQuery() (string, error) {
+	return "", nil
 }

cloud/gcpprovider.go

@@ -79,8 +79,9 @@ func gcpAllocationToOutOfClusterAllocation(gcpAlloc gcpAllocation) *OutOfCluster
 	}
 }
 
-func (gcp *GCP) GetLocalStorageCost() (float64, error) {
-	return 0.04, nil
+func (gcp *GCP) GetLocalStorageQuery() (string, error) {
+	localStorageCost := 0.04 // TODO: Set to the price for the appropriate storage class. It's not trivial to determine the local storage disk type
+	return fmt.Sprintf(`sum(sum(container_fs_limit_bytes{device!="tmpfs", id="/"}) by (instance) / 1024 / 1024 / 1024) * %f`, localStorageCost), nil
 }
 
 func (gcp *GCP) GetConfig() (*CustomPricing, error) {
@@ -233,7 +234,7 @@ func (gcp *GCP) QuerySQL(query string) ([]*OutOfClusterAllocation, error) {
 }
 
 // ClusterName returns the name of a GKE cluster, as provided by metadata.
-func (*GCP) ClusterName() ([]byte, error) {
+func (*GCP) ClusterInfo() (map[string]string, error) {
 	metadataClient := metadata.NewClient(&http.Client{Transport: userAgentTransport{
 		userAgent: "kubecost",
 		base:      http.DefaultTransport,
@@ -247,7 +248,7 @@ func (*GCP) ClusterName() ([]byte, error) {
 	m := make(map[string]string)
 	m["name"] = attribute
 	m["provider"] = "GCP"
-	return json.Marshal(m)
+	return m, nil
 }
 
 // AddServiceKey adds the service key as required for GetDisks
@@ -412,13 +413,14 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 					instanceType = "custom"
 				}
 
-				var partialCPU float64
-				if strings.ToLower(instanceType) == "f1micro" {
-					partialCPU = 0.2
-				} else if strings.ToLower(instanceType) == "g1small" {
-					partialCPU = 0.5
-				}
-
+				/*
+					var partialCPU float64
+					if strings.ToLower(instanceType) == "f1micro" {
+						partialCPU = 0.2
+					} else if strings.ToLower(instanceType) == "g1small" {
+						partialCPU = 0.5
+					}
+				*/
 				var gpuType string
 				provIdRx := regexp.MustCompile("(Nvidia Tesla [^ ]+) ")
 				for matchnum, group := range provIdRx.FindStringSubmatch(product.Description) {
@@ -490,9 +492,11 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 									product.Node = &Node{
 										RAMCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 									}
-									if partialCPU != 0 {
-										product.Node.VCPU = fmt.Sprintf("%f", partialCPU)
-									}
+									/*
+										if partialCPU != 0 {
+											product.Node.VCPU = fmt.Sprintf("%f", partialCPU)
+										}
+									*/
 									product.Node.UsageType = usageType
 									gcpPricingList[candidateKey] = product
 								}
@@ -505,9 +509,11 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 									product.Node = &Node{
 										RAMCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 									}
-									if partialCPU != 0 {
-										product.Node.VCPU = fmt.Sprintf("%f", partialCPU)
-									}
+									/*
+										if partialCPU != 0 {
+											product.Node.VCPU = fmt.Sprintf("%f", partialCPU)
+										}
+									*/
 									product.Node.UsageType = usageType
 									gcpPricingList[candidateKeyGPU] = product
 								}
@@ -520,9 +526,11 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 									product.Node = &Node{
 										VCPUCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 									}
-									if partialCPU != 0 {
-										product.Node.VCPU = fmt.Sprintf("%f", partialCPU)
-									}
+									/*
+										if partialCPU != 0 {
+											product.Node.VCPU = fmt.Sprintf("%f", partialCPU)
+										}
+									*/
 									product.Node.UsageType = usageType
 									gcpPricingList[candidateKey] = product
 								}
@@ -533,9 +541,11 @@ func (gcp *GCP) parsePage(r io.Reader, inputKeys map[string]Key, pvKeys map[stri
 									product.Node = &Node{
 										VCPUCost: strconv.FormatFloat(hourlyPrice, 'f', -1, 64),
 									}
-									if partialCPU != 0 {
-										product.Node.VCPU = fmt.Sprintf("%f", partialCPU)
-									}
+									/*
+										if partialCPU != 0 {
+											product.Node.VCPU = fmt.Sprintf("%f", partialCPU)
+										}
+									*/
 									product.Node.UsageType = usageType
 									gcpPricingList[candidateKeyGPU] = product
 								}

cloud/provider.go

@@ -75,7 +75,7 @@ type OutOfClusterAllocation struct {
 
 // Provider represents a k8s provider.
 type Provider interface {
-	ClusterName() ([]byte, error)
+	ClusterInfo() (map[string]string, error)
 	AddServiceKey(url.Values) error
 	GetDisks() ([]byte, error)
 	NodePricing(Key) (*Node, error)
@@ -87,7 +87,7 @@ type Provider interface {
 	UpdateConfig(r io.Reader, updateType string) (*CustomPricing, error)
 	GetConfig() (*CustomPricing, error)
 	GetManagementPlatform() (string, error)
-	GetLocalStorageCost() (float64, error)
+	GetLocalStorageQuery() (string, error)
 
 	ExternalAllocations(string, string, string) ([]*OutOfClusterAllocation, error)
 }
@@ -210,8 +210,8 @@ type CustomProvider struct {
 	DownloadPricingDataLock sync.RWMutex
 }
 
-func (*CustomProvider) GetLocalStorageCost() (float64, error) {
-	return 0.04, nil
+func (*CustomProvider) GetLocalStorageQuery() (string, error) {
+	return "", nil
 }
 
 func (*CustomProvider) GetConfig() (*CustomPricing, error) {
@@ -259,8 +259,10 @@ func (*CustomProvider) UpdateConfig(r io.Reader, updateType string) (*CustomPric
 
 }
 
-func (*CustomProvider) ClusterName() ([]byte, error) {
-	return nil, nil
+func (*CustomProvider) ClusterInfo() (map[string]string, error) {
+	m := make(map[string]string)
+	m["provider"] = "custom"
+	return m, nil
 }
 
 func (*CustomProvider) AddServiceKey(url.Values) error {

costmodel/cluster.go

@@ -24,15 +24,13 @@ const (
 	queryStorage = `sum(
 		avg(avg_over_time(pv_hourly_cost[%s] %s)) by (persistentvolume) * 730 
 		* avg(avg_over_time(kube_persistentvolume_capacity_bytes[%s] %s)) by (persistentvolume) / 1024 / 1024 / 1024
-	  ) +
-	  sum(avg(container_fs_limit_bytes{device!="tmpfs", id="/"} %s) by (instance) / 1024 / 1024 / 1024) * %f`
+	  ) %s`
 
 	queryTotal = `sum(avg(node_total_hourly_cost) by (node)) * 730 +
 	  sum(
 		avg(avg_over_time(pv_hourly_cost[1h])) by (persistentvolume) * 730 
 		* avg(avg_over_time(kube_persistentvolume_capacity_bytes[1h])) by (persistentvolume) / 1024 / 1024 / 1024
-	  ) +
-	  sum(avg(container_fs_limit_bytes{device!="tmpfs", id="/"}) by (instance) / 1024 / 1024 / 1024) * %f`
+	  ) %s`
 )
 
 type Totals struct {
@@ -114,15 +112,18 @@ func resultToTotal(qr interface{}) ([][]string, error) {
 // ClusterCostsOverTime gives the current full cluster costs averaged over a window of time.
 func ClusterCosts(cli prometheusClient.Client, cloud costAnalyzerCloud.Provider, windowString, offset string) (*Totals, error) {
 
-	localStorageCost, err := cloud.GetLocalStorageCost()
+	localStorageQuery, err := cloud.GetLocalStorageQuery()
 	if err != nil {
 		return nil, err
 	}
+	if localStorageQuery != "" {
+		localStorageQuery = fmt.Sprintf("+ %s", localStorageQuery)
+	}
 
 	qCores := fmt.Sprintf(queryClusterCores, offset, offset, offset)
 	qRAM := fmt.Sprintf(queryClusterRAM, offset, offset)
-	qStorage := fmt.Sprintf(queryStorage, windowString, offset, windowString, offset, offset, localStorageCost)
-	qTotal := fmt.Sprintf(queryTotal, localStorageCost)
+	qStorage := fmt.Sprintf(queryStorage, windowString, offset, windowString, offset, localStorageQuery)
+	qTotal := fmt.Sprintf(queryTotal, localStorageQuery)
 	log.Printf("%s", qTotal)
 
 	resultClusterCores, err := query(cli, qCores)
@@ -176,10 +177,13 @@ func ClusterCosts(cli prometheusClient.Client, cloud costAnalyzerCloud.Provider,
 // ClusterCostsOverTime gives the full cluster costs over time
 func ClusterCostsOverTime(cli prometheusClient.Client, cloud costAnalyzerCloud.Provider, startString, endString, windowString, offset string) (*Totals, error) {
 
-	localStorageCost, err := cloud.GetLocalStorageCost()
+	localStorageQuery, err := cloud.GetLocalStorageQuery()
 	if err != nil {
 		return nil, err
 	}
+	if localStorageQuery != "" {
+		localStorageQuery = fmt.Sprintf("+ %s", localStorageQuery)
+	}
 
 	layout := "2006-01-02T15:04:05.000Z"
 
@@ -201,8 +205,8 @@ func ClusterCostsOverTime(cli prometheusClient.Client, cloud costAnalyzerCloud.P
 
 	qCores := fmt.Sprintf(queryClusterCores, offset, offset, offset)
 	qRAM := fmt.Sprintf(queryClusterRAM, offset, offset)
-	qStorage := fmt.Sprintf(queryStorage, windowString, offset, windowString, offset, offset, localStorageCost)
-	qTotal := fmt.Sprintf(queryTotal, localStorageCost)
+	qStorage := fmt.Sprintf(queryStorage, windowString, offset, windowString, offset, localStorageQuery)
+	qTotal := fmt.Sprintf(queryTotal, localStorageQuery)
 	log.Printf("%s", qTotal)
 
 	resultClusterCores, err := queryRange(cli, qCores, start, end, window)

go.sum

@@ -71,6 +71,7 @@ github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g=
 github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8 h1:L9JPKrtsHMQ4VCRQfHvbbHBfB2Urn8xf6QZeXZ+OrN4=
 github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
@@ -81,6 +82,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=

main.go

@@ -285,6 +285,15 @@ func (p *Accesses) ManagementPlatform(w http.ResponseWriter, r *http.Request, ps
 	return
 }
 
+func (p *Accesses) ClusterInfo(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+
+	data, err := p.Cloud.ClusterInfo()
+	w.Write(wrapData(data, err))
+
+}
+
 func Healthz(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) {
 	w.WriteHeader(200)
 	w.Header().Set("Content-Length", "0")
@@ -520,6 +529,7 @@ func main() {
 	router.GET("/clusterCosts", a.ClusterCosts)
 	router.GET("/validatePrometheus", a.GetPrometheusMetadata)
 	router.GET("/managementPlatform", a.ManagementPlatform)
+	router.GET("/clusterInfo", a.ClusterInfo)
 
 	rootMux := http.NewServeMux()
 	rootMux.Handle("/", router)