load properties which are static per cluster into clusterMap, other small fixes

pkg/cloud/awsprovider.go

@@ -156,6 +156,8 @@ type AWS struct {
 	Config                      *ProviderConfig
 	ServiceAccountChecks        map[string]*ServiceAccountCheck
 	clusterManagementPrice      float64
+	clusterAccountId            string
+	clusterRegion               string
 	clusterProvisioner          string
 	*CustomProvider
 }
@@ -1180,6 +1182,8 @@ func (awsProvider *AWS) ClusterInfo() (map[string]string, error) {
 		m := make(map[string]string)
 		m["name"] = c.ClusterName
 		m["provider"] = "AWS"
+		m["account"] = c.AthenaProjectID // this value requires configuration but is unavailable else where
+		m["region"] = awsProvider.clusterRegion
 		m["id"] = env.GetClusterID()
 		m["remoteReadEnabled"] = strconv.FormatBool(remoteEnabled)
 		m["provisioner"] = awsProvider.clusterProvisioner
@@ -1190,6 +1194,8 @@ func (awsProvider *AWS) ClusterInfo() (map[string]string, error) {
 		m := make(map[string]string)
 		m["name"] = clusterName
 		m["provider"] = "AWS"
+		m["account"] = c.AthenaProjectID // this value requires configuration but is unavailable else where
+		m["region"] = awsProvider.clusterRegion
 		m["id"] = env.GetClusterID()
 		m["remoteReadEnabled"] = strconv.FormatBool(remoteEnabled)
 		return m, nil

pkg/cloud/azureprovider.go

@@ -383,6 +383,8 @@ type Azure struct {
 	Config                  *ProviderConfig
 	ServiceAccountChecks    map[string]*ServiceAccountCheck
 	RateCardPricingError    error
+	clusterAccountId        string
+	clusterRegion           string
 }
 
 type azureKey struct {
@@ -1130,6 +1132,8 @@ func (az *Azure) ClusterInfo() (map[string]string, error) {
 		m["name"] = c.ClusterName
 	}
 	m["provider"] = "azure"
+	m["account"] = az.clusterAccountId
+	m["region"] = az.clusterRegion
 	m["remoteReadEnabled"] = strconv.FormatBool(remoteEnabled)
 	m["id"] = env.GetClusterID()
 	return m, nil
@@ -1418,3 +1422,15 @@ func (az *Azure) CombinedDiscountForNode(instanceType string, isPreemptible bool
 func (az *Azure) Regions() []string {
 	return azureRegions
 }
+
+func parseAzureSubscriptionID(id string) string {
+	// azure:///subscriptions/0bd50fdf-c923-4e1e-850c-196dd3dcc5d3/...
+	//  => 0bd50fdf-c923-4e1e-850c-196dd3dcc5d3
+	rx := regexp.MustCompile("azure:///subscriptions/([^/]*)/*")
+	match := rx.FindStringSubmatch(id)
+	if len(match) >= 2 {
+		return match[1]
+	}
+	// Return empty string if an account could not be parsed from provided string
+	return ""
+}

pkg/cloud/gcpprovider.go

@@ -92,6 +92,8 @@ type GCP struct {
 	ServiceKeyProvided      bool
 	ValidPricingKeys        map[string]bool
 	clusterManagementPrice  float64
+	clusterProjectId        string
+	clusterRegion           string
 	clusterProvisioner      string
 	*CustomProvider
 }
@@ -529,6 +531,8 @@ func (gcp *GCP) ClusterInfo() (map[string]string, error) {
 	m := make(map[string]string)
 	m["name"] = attribute
 	m["provider"] = "GCP"
+	m["project"] = gcp.clusterProjectId
+	m["region"] = gcp.clusterRegion
 	m["provisioner"] = gcp.clusterProvisioner
 	m["id"] = env.GetClusterID()
 	m["remoteReadEnabled"] = strconv.FormatBool(remoteEnabled)
@@ -1524,3 +1528,15 @@ func sustainedUseDiscount(class string, defaultDiscount float64, isPreemptible b
 	}
 	return discount
 }
+
+func parseGCPProjectID (id string) string {
+	// gce://guestbook-227502/...
+	//  => guestbook-227502
+	rx := regexp.MustCompile("gce://([^/]*)/*")
+	match := rx.FindStringSubmatch(id)
+	if len(match) >= 2 {
+		return match[1]
+	}
+	// Return empty string if an account could not be parsed from provided string
+	return ""
+}

pkg/cloud/provider.go

@@ -412,21 +412,11 @@ func NewProvider(cache clustercache.ClusterCache, apiKey string) (Provider, erro
 		return nil, fmt.Errorf("Could not locate any nodes for cluster.")
 	}
 
-	provider := strings.ToLower(nodes[0].Spec.ProviderID)
+	provider, configFileName, region, accountID, projectID := getClusterProperties(nodes[0])
 
-	if env.IsUseCSVProvider() {
+	switch provider {
+	case "CSV":
 		klog.Infof("Using CSV Provider with CSV at %s", env.GetCSVPath())
-		configFileName := ""
-		if metadata.OnGCE() {
-			configFileName = "gcp.json"
-		} else if strings.HasPrefix(provider, "aws") {
-			configFileName = "aws.json"
-		} else if strings.HasPrefix(provider, "azure") {
-			configFileName = "azure.json"
-
-		} else {
-			configFileName = "default.json"
-		}
 		return &CSVProvider{
 			CSVLocation: env.GetCSVPath(),
 			CustomProvider: &CustomProvider{
@@ -434,8 +424,7 @@ func NewProvider(cache clustercache.ClusterCache, apiKey string) (Provider, erro
 				Config:    NewProviderConfig(configFileName),
 			},
 		}, nil
-	}
-	if metadata.OnGCE() {
+	case "GCP":
 		klog.V(3).Info("metadata reports we are in GCE")
 		if apiKey == "" {
 			return nil, errors.New("Supply a GCP Key to start getting data")
@@ -443,31 +432,61 @@ func NewProvider(cache clustercache.ClusterCache, apiKey string) (Provider, erro
 		return &GCP{
 			Clientset: cache,
 			APIKey:    apiKey,
-			Config:    NewProviderConfig("gcp.json"),
+			Config:    NewProviderConfig(configFileName),
+			clusterRegion: region,
+			clusterProjectId: projectID,
 		}, nil
-	}
-
-	if strings.HasPrefix(provider, "aws") {
+	case "AWS":
 		klog.V(2).Info("Found ProviderID starting with \"aws\", using AWS Provider")
 		return &AWS{
 			Clientset: cache,
-			Config:    NewProviderConfig("aws.json"),
+			Config:    NewProviderConfig(configFileName),
+			clusterRegion: region,
+			clusterAccountId: accountID,
 		}, nil
-	} else if strings.HasPrefix(provider, "azure") {
+	case "AZURE":
 		klog.V(2).Info("Found ProviderID starting with \"azure\", using Azure Provider")
 		return &Azure{
 			Clientset: cache,
-			Config:    NewProviderConfig("azure.json"),
+			Config:    NewProviderConfig(configFileName),
+			clusterRegion: region,
+			clusterAccountId: accountID,
 		}, nil
-	} else {
+	default:
 		klog.V(2).Info("Unsupported provider, falling back to default")
 		return &CustomProvider{
 			Clientset: cache,
-			Config:    NewProviderConfig("default.json"),
+			Config:    NewProviderConfig(configFileName),
 		}, nil
 	}
 }
 
+func getClusterProperties(node *v1.Node) (string, string, string, string, string) {
+	providerID := strings.ToLower(node.Spec.ProviderID)
+	provider := "DEFAULT"
+	configFileName := "default.json"
+	region := node.Labels["topology.kubernetes.io/region"]
+	accountID := ""
+	projectID := ""
+	if metadata.OnGCE() {
+		provider = "GCP"
+		configFileName = "gcp.json"
+		projectID = parseGCPProjectID(providerID)
+	} else if strings.HasPrefix(providerID, "aws") {
+		provider = "AWS"
+		configFileName = "aws.json"
+	} else if strings.HasPrefix(providerID, "azure") {
+		provider = "AZURE"
+		configFileName = "azure.json"
+		accountID = parseAzureSubscriptionID(providerID)
+	}
+	if env.IsUseCSVProvider() {
+		provider = "CSV"
+	}
+
+	return provider, configFileName, region, accountID, projectID
+}
+
 func UpdateClusterMeta(cluster_id, cluster_name string) error {
 	pw := env.GetRemotePW()
 	address := env.GetSQLAddress()

pkg/costmodel/clusters/clustermap.go

@@ -21,11 +21,15 @@ const (
 	LoadRetryDelay time.Duration = 10 * time.Second
 )
 
+// ClusterInfo holds attributes of Cluster from metrics pulled from Prometheus
 type ClusterInfo struct {
 	ID          string `json:"id"`
 	Name        string `json:"name"`
 	Profile     string `json:"profile"`
 	Provider    string `json:"provider"`
+	Account     string `json:"account"`
+	Project     string `json:"project"`
+	Region      string `json:"region"`
 	Provisioner string `json:"provisioner"`
 }
 
@@ -40,6 +44,9 @@ func (ci *ClusterInfo) Clone() *ClusterInfo {
 		Name:        ci.Name,
 		Profile:     ci.Profile,
 		Provider:    ci.Provider,
+		Account:     ci.Account,
+		Project:     ci.Project,
+		Region:      ci.Region,
 		Provisioner: ci.Provisioner,
 	}
 }
@@ -170,6 +177,21 @@ func (pcm *PrometheusClusterMap) loadClusters() (map[string]*ClusterInfo, error)
 			provider = ""
 		}
 
+		account, err := result.GetString("account")
+		if err != nil {
+			account = ""
+		}
+
+		project, err := result.GetString("project")
+		if err != nil {
+			project = ""
+		}
+
+		region, err := result.GetString("region")
+		if err != nil {
+			region = ""
+		}
+
 		provisioner, err := result.GetString("provisioner")
 		if err != nil {
 			provisioner = ""
@@ -180,6 +202,9 @@ func (pcm *PrometheusClusterMap) loadClusters() (map[string]*ClusterInfo, error)
 			Name:        name,
 			Profile:     profile,
 			Provider:    provider,
+			Account:     account,
+			Project:     project,
+			Region:      region,
 			Provisioner: provisioner,
 		}
 	}
@@ -218,14 +243,31 @@ func (pcm *PrometheusClusterMap) getLocalClusterInfo() (*ClusterInfo, error) {
 
 	var clusterProfile string
 	var provider string
+	var account string
+	var project string
+	var region string
 	var provisioner string
 
 	if cp, ok := info["clusterProfile"]; ok {
 		clusterProfile = cp
 	}
+
 	if pvdr, ok := info["provider"]; ok {
 		provider = pvdr
 	}
+
+	if acct, ok := info["account"]; ok {
+		account = acct
+	}
+
+	if proj, ok := info["project"]; ok {
+		project = proj
+	}
+
+	if reg, ok := info["region"]; ok {
+		region = reg
+	}
+
 	if pvsr, ok := info["provisioner"]; ok {
 		provisioner = pvsr
 	}
@@ -235,6 +277,9 @@ func (pcm *PrometheusClusterMap) getLocalClusterInfo() (*ClusterInfo, error) {
 		Name:        name,
 		Profile:     clusterProfile,
 		Provider:    provider,
+		Account:     account,
+		Project:     project,
+		Region:      region,
 		Provisioner: provisioner,
 	}, nil
 }

pkg/kubecost/allocation.go

@@ -1754,12 +1754,12 @@ func (as *AllocationSet) ComputeIdleAllocations(assetSet *AssetSet) (map[string]
 			}
 
 			cpuCost := node.CPUCost * (1.0 - node.Discount) * adjustmentRate
-			gpuCost := node.GPUCost * (1.0 - node.Discount) * adjustmentRate
 			ramCost := node.RAMCost * (1.0 - node.Discount) * adjustmentRate
+			gpuCost := node.GPUCost * (1.0) * adjustmentRate
 
 			assetClusterResourceCosts[node.Properties().Cluster]["cpu"] += cpuCost
-			assetClusterResourceCosts[node.Properties().Cluster]["gpu"] += gpuCost
 			assetClusterResourceCosts[node.Properties().Cluster]["ram"] += ramCost
+			assetClusterResourceCosts[node.Properties().Cluster]["gpu"] += gpuCost
 		}
 	})
 
@@ -1891,12 +1891,12 @@ func (as *AllocationSet) ComputeIdleAllocationsByNode(assetSet *AssetSet) (map[s
 			}
 
 			cpuCost := node.CPUCost * (1.0 - node.Discount) * adjustmentRate
-			gpuCost := node.GPUCost * (1.0 - node.Discount) * adjustmentRate
 			ramCost := node.RAMCost * (1.0 - node.Discount) * adjustmentRate
+			gpuCost := node.GPUCost * adjustmentRate
 
 			assetNodeResourceCosts[node.Properties().ProviderID]["cpu"] += cpuCost
-			assetNodeResourceCosts[node.Properties().ProviderID]["gpu"] += gpuCost
 			assetNodeResourceCosts[node.Properties().ProviderID]["ram"] += ramCost
+			assetNodeResourceCosts[node.Properties().ProviderID]["gpu"] += gpuCost
 		}
 	})