Merge pull request #358 from kubecost/bolt/azure-disk-fix

Fix Azure PV Price Units

pkg/cloud/awsprovider.go

@@ -928,33 +928,18 @@ func (*AWS) AddServiceKey(formValues url.Values) error {
 	return ioutil.WriteFile("/var/configs/key.json", result, 0644)
 }
 
-func configureAWSAuth(keyFile string) error {
-	jsonFile, err := os.Open(keyFile)
-	if err != nil {
-		if os.IsNotExist(err) {
-			klog.V(2).Infof("Using Default Credentials")
-			return nil
+func (aws *AWS) configureAWSAuth() error {
+	accessKeyID := aws.ServiceKeyName
+	accessKeySecret := aws.ServiceKeySecret
+	if accessKeyID != "" && accessKeySecret != "" { // credentials may exist on the actual AWS node-- if so, use those. If not, override with the service key
+		err := os.Setenv(awsAccessKeyIDEnvVar, accessKeyID)
+		if err != nil {
+			return err
+		}
+		err = os.Setenv(awsAccessKeySecretEnvVar, accessKeySecret)
+		if err != nil {
+			return err
 		}
-
-		return err
-	}
-	defer jsonFile.Close()
-
-	byteValue, _ := ioutil.ReadAll(jsonFile)
-	var result map[string]string
-	err = json.Unmarshal([]byte(byteValue), &result)
-	if err != nil {
-		return err
-	}
-
-	err = os.Setenv(awsAccessKeyIDEnvVar, result["awsServiceKeyName"])
-	if err != nil {
-		return err
-	}
-
-	err = os.Setenv(awsAccessKeySecretEnvVar, result["awsServiceKeySecret"])
-	if err != nil {
-		return err
 	}
 	return nil
 }
@@ -979,8 +964,8 @@ func getClusterConfig(ccFile string) (map[string]string, error) {
 }
 
 // GetDisks returns the AWS disks backing PVs. Useful because sometimes k8s will not clean up PVs correctly. Requires a json config in /var/configs with key region.
-func (*AWS) GetDisks() ([]byte, error) {
-	err := configureAWSAuth("/var/configs/key.json")
+func (a *AWS) GetDisks() ([]byte, error) {
+	err := a.configureAWSAuth()
 	if err != nil {
 		return nil, err
 	}
@@ -1701,9 +1686,9 @@ func getRegionReservedInstances(region string) ([]*AWSReservedInstance, error) {
 }
 
 func (a *AWS) getReservedInstances() ([]*AWSReservedInstance, error) {
-	err := configureAWSAuth("/var/configs/aws.json")
+	err := a.configureAWSAuth()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("Error Configuring aws auth: %s", err.Error())
 	}
 
 	var reservedInstances []*AWSReservedInstance

pkg/cloud/azureprovider.go

@@ -408,7 +408,9 @@ func (az *Azure) DownloadPricingData() error {
 						for _, rate := range v.MeterRates {
 							priceInUsd += *rate
 						}
-						priceStr := fmt.Sprintf("%f", priceInUsd)
+						// rate is in GB per month, resolve to GB per hour
+						pricePerHour := priceInUsd / 730.0
+						priceStr := fmt.Sprintf("%f", pricePerHour)
 
 						key := region + "," + storageClass
 						klog.V(4).Infof("Adding PV.Key: %s, Cost: %s", key, priceStr)

pkg/cloud/provider.go

@@ -59,6 +59,7 @@ type Node struct {
 	GPUName          string                `json:"gpuName"`
 	GPUCost          string                `json:"gpuCost"`
 	InstanceType     string                `json:"instanceType,omitempty"`
+	Region           string                `json:"region,omitempty"`
 	Reserved         *ReservedInstanceData `json:"reserved,omitempty"`
 }
 

pkg/costmodel/cluster.go

@@ -155,7 +155,7 @@ func ComputeClusterCosts(client prometheus.Client, provider cloud.Provider, wind
 	}
 	mins := end.Sub(*start).Minutes()
 
-	const fmtQueryDataCount = `max(count_over_time(kube_node_status_capacity_cpu_cores[%s:1m]%s))`
+	const fmtQueryDataCount = `max(sum(count_over_time(kube_node_status_capacity_cpu_cores[%s:1m]%s)) by (node, cluster_id))`
 
 	const fmtQueryTotalGPU = `sum(
 		sum_over_time(node_gpu_hourly_cost[%s:1m]%s) / 60

pkg/costmodel/costmodel.go

@@ -1144,6 +1144,9 @@ func (cm *CostModel) GetNodeCost(cp costAnalyzerCloud.Provider) (map[string]*cos
 		if newCnode.InstanceType == "" {
 			newCnode.InstanceType = n.Labels[v1.LabelInstanceType]
 		}
+		if newCnode.Region == "" {
+			newCnode.Region = n.Labels[v1.LabelZoneRegion]
+		}
 
 		var cpu float64
 		if newCnode.VCPU == "" {

pkg/costmodel/router.go

@@ -709,13 +709,15 @@ func (a *Accesses) recordPrices() {
 					}
 				}
 				nodeType := node.InstanceType
+				nodeRegion := node.Region
+				nodeUsageType := node.UsageType
 
 				totalCost := cpu*cpuCost + ramCost*(ram/1024/1024/1024) + gpu*gpuCost
 
-				a.CPUPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType).Set(cpuCost)
-				a.RAMPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType).Set(ramCost)
-				a.GPUPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType).Set(gpuCost)
-				a.NodeTotalPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType).Set(totalCost)
+				a.CPUPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, nodeUsageType).Set(cpuCost)
+				a.RAMPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, nodeUsageType).Set(ramCost)
+				a.GPUPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, nodeUsageType).Set(gpuCost)
+				a.NodeTotalPriceRecorder.WithLabelValues(nodeName, nodeName, nodeType, nodeRegion, nodeUsageType).Set(totalCost)
 				labelKey := getKeyFromLabelStrings(nodeName, nodeName)
 				nodeSeen[labelKey] = true
 			}
@@ -968,22 +970,22 @@ func Initialize(additionalConfigWatchers ...ConfigWatchers) {
 	cpuGv := prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "node_cpu_hourly_cost",
 		Help: "node_cpu_hourly_cost hourly cost for each cpu on this node",
-	}, []string{"instance", "node", "instance_type"})
+	}, []string{"instance", "node", "instance_type", "region", "usage_type"})
 
 	ramGv := prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "node_ram_hourly_cost",
 		Help: "node_ram_hourly_cost hourly cost for each gb of ram on this node",
-	}, []string{"instance", "node", "instance_type"})
+	}, []string{"instance", "node", "instance_type", "region", "usage_type"})
 
 	gpuGv := prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "node_gpu_hourly_cost",
 		Help: "node_gpu_hourly_cost hourly cost for each gpu on this node",
-	}, []string{"instance", "node", "instance_type"})
+	}, []string{"instance", "node", "instance_type", "region", "usage_type"})
 
 	totalGv := prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "node_total_hourly_cost",
 		Help: "node_total_hourly_cost Total node cost per hour",
-	}, []string{"instance", "node", "instance_type"})
+	}, []string{"instance", "node", "instance_type", "region", "usage_type"})
 
 	pvGv := prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "pv_hourly_cost",