Add Azure Storage Config values to provider config

Signed-off-by: Sean Holcomb <seanholcomb@gmail.com>

pkg/cloud/azureprovider.go

@@ -37,6 +37,7 @@ const (
 	AzureDiskStandardStorageClass    = "standard_hdd"
 	defaultSpotLabel                 = "kubernetes.azure.com/scalesetpriority"
 	defaultSpotLabelValue            = "spot"
+	AzureStorageUpdateType           = "AzureStorage"
 )
 
 var (
@@ -509,7 +510,7 @@ func (ask *AzureServiceKey) IsValid() bool {
 }
 
 // Loads the azure authentication via configuration or a secret set at install time.
-func (az *Azure) getAzureAuth(forceReload bool, cp *CustomPricing) (subscriptionID, clientID, clientSecret, tenantID string) {
+func (az *Azure) getAzureRateCardAuth(forceReload bool, cp *CustomPricing) (subscriptionID, clientID, clientSecret, tenantID string) {
 	// 1. Check config values first (set from frontend UI)
 	if cp.AzureSubscriptionID != "" && cp.AzureClientID != "" && cp.AzureClientSecret != "" && cp.AzureTenantID != "" {
 		subscriptionID = cp.AzureSubscriptionID
@@ -535,32 +536,48 @@ func (az *Azure) getAzureAuth(forceReload bool, cp *CustomPricing) (subscription
 }
 
 // GetAzureStorageConfig retrieves storage config from secret and sets default values
-func (az *Azure) GetAzureStorageConfig(forceReload bool) (*AzureStorageConfig, error) {
-	// retrieve config for default subscription id
-	defaultSubscriptionID := ""
-	config, err := az.GetConfig()
-	if err == nil {
-		defaultSubscriptionID = config.AzureSubscriptionID
+func (az *Azure) GetAzureStorageConfig(forceReload bool, cp *CustomPricing) (*AzureStorageConfig, error) {
+	// default subscription id
+	defaultSubscriptionID := cp.AzureSubscriptionID
+
+	// 1. Check Config for storage set up
+	asc := &AzureStorageConfig{
+		SubscriptionId: cp.AzureStorageSubscriptionID,
+		AccountName:    cp.AzureStorageAccount,
+		AccessKey:      cp.AzureStorageAccessKey,
+		ContainerName:  cp.AzureStorageContainer,
+		ContainerPath:  cp.AzureContainerPath,
+		AzureCloud:     cp.AzureCloud,
+	}
+
+	// check for required fields
+	if asc != nil && asc.AccessKey != "" && asc.AccountName != "" && asc.ContainerName != "" && asc.SubscriptionId != "" {
+		az.serviceAccountChecks.set("hasStorage", &ServiceAccountCheck{
+			Message: "Azure Storage Config exists",
+			Status:  true,
+		})
+		return asc, nil
 	}
 
-	// 1. Check for secret
-	s, err := az.loadAzureStorageConfig(forceReload)
+	// 2. Check for secret
+	asc, err := az.loadAzureStorageConfig(forceReload)
 	if err != nil {
 		log.Errorf("Error, %s", err.Error())
 	}
-	if s != nil && s.AccessKey != "" && s.AccountName != "" && s.ContainerName != "" {
+	// To support already configured users, subscriptionID may not be set in secret in which case, the subscriptionID
+	// for the rate card API is used
+	if asc.SubscriptionId == "" {
+		asc.SubscriptionId = defaultSubscriptionID
+	}
+	// check for required fields
+	if asc != nil && asc.AccessKey != "" && asc.AccountName != "" && asc.ContainerName != "" && asc.SubscriptionId == "" {
 		az.serviceAccountChecks.set("hasStorage", &ServiceAccountCheck{
 			Message: "Azure Storage Config exists",
 			Status:  true,
 		})
-
-		// To support already configured users, subscriptionID may not be set in secret in which case, the subscriptionID
-		// for the rate card API is used
-		if s.SubscriptionId == "" {
-			s.SubscriptionId = defaultSubscriptionID
-		}
-		return s, nil
+		return asc, nil
 	}
+
 	az.serviceAccountChecks.set("hasStorage", &ServiceAccountCheck{
 		Message: "Azure Storage Config exists",
 		Status:  false,
@@ -744,7 +761,7 @@ func (az *Azure) DownloadPricingData() error {
 	}
 
 	// Load the service provider keys
-	subscriptionID, clientID, clientSecret, tenantID := az.getAzureAuth(true, config)
+	subscriptionID, clientID, clientSecret, tenantID := az.getAzureRateCardAuth(true, config)
 	config.AzureSubscriptionID = subscriptionID
 	config.AzureClientID = clientID
 	config.AzureClientSecret = clientSecret
@@ -1165,27 +1182,42 @@ func (az *Azure) UpdateConfigFromConfigMap(a map[string]string) (*CustomPricing,
 }
 
 func (az *Azure) UpdateConfig(r io.Reader, updateType string) (*CustomPricing, error) {
-	defer az.DownloadPricingData()
-
 	return az.Config.Update(func(c *CustomPricing) error {
-		a := make(map[string]interface{})
-		err := json.NewDecoder(r).Decode(&a)
-		if err != nil {
-			return err
-		}
-		for k, v := range a {
-			kUpper := strings.Title(k) // Just so we consistently supply / receive the same values, uppercase the first letter.
-			vstr, ok := v.(string)
-			if ok {
-				err := SetCustomPricingField(c, kUpper, vstr)
-				if err != nil {
-					return err
+		if updateType == AzureStorageUpdateType {
+			asc := &AzureStorageConfig{}
+			err := json.NewDecoder(r).Decode(&asc)
+			if err != nil {
+				return err
+			}
+
+			c.AzureStorageSubscriptionID = asc.SubscriptionId
+			c.AzureStorageAccount = asc.AccountName
+			if asc.AccessKey != "" {
+				c.AzureStorageAccessKey = asc.AccessKey
+			}
+			c.AzureStorageContainer = asc.ContainerName
+			c.AzureContainerPath = asc.ContainerPath
+			c.AzureCloud = asc.AzureCloud
+		} else {
+			defer az.DownloadPricingData()
+			a := make(map[string]interface{})
+			err := json.NewDecoder(r).Decode(&a)
+			if err != nil {
+				return err
+			}
+			for k, v := range a {
+				kUpper := strings.Title(k) // Just so we consistently supply / receive the same values, uppercase the first letter.
+				vstr, ok := v.(string)
+				if ok {
+					err := SetCustomPricingField(c, kUpper, vstr)
+					if err != nil {
+						return err
+					}
+				} else {
+					return fmt.Errorf("type error while updating config for %s", kUpper)
 				}
-			} else {
-				return fmt.Errorf("type error while updating config for %s", kUpper)
 			}
 		}
-
 		if env.IsRemoteEnabled() {
 			err := UpdateClusterMeta(env.GetClusterID(), c.ClusterName)
 			if err != nil {

pkg/cloud/provider.go

@@ -178,6 +178,12 @@ type CustomPricing struct {
 	AzureTenantID                string `json:"azureTenantID"`
 	AzureBillingRegion           string `json:"azureBillingRegion"`
 	AzureOfferDurableID          string `json:"azureOfferDurableID"`
+	AzureStorageSubscriptionID   string `json:"azureStorageSubscriptionID"`
+	AzureStorageAccount          string `json:"azureStorageAccount"`
+	AzureStorageAccessKey        string `json:"azureStorageAccessKey"`
+	AzureStorageContainer        string `json:"azureStorageContainer"`
+	AzureContainerPath           string `json:"azureContainerPath"`
+	AzureCloud                   string `json:"azureCloud"`
 	CurrencyCode                 string `json:"currencyCode"`
 	Discount                     string `json:"discount"`
 	NegotiatedDiscount           string `json:"negotiatedDiscount"`

pkg/costmodel/allocation.go

@@ -293,8 +293,10 @@ func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Dur
 	}
 
 	// TODO:CLEANUP remove "max batch" idea and clusterStart/End
-	cm.buildPodMap(window, resolution, env.GetETLMaxPrometheusQueryDuration(), podMap, clusterStart, clusterEnd, ingestPodUID, podUIDKeyMap)
-
+	err := cm.buildPodMap(window, resolution, env.GetETLMaxPrometheusQueryDuration(), podMap, clusterStart, clusterEnd, ingestPodUID, podUIDKeyMap)
+	if err != nil {
+		log.Errorf("CostModel.ComputeAllocation: failed to build pod map: %s", err.Error())
+	}
 	// (2) Run and apply remaining queries
 
 	// Query for the duration between start and end
@@ -476,7 +478,7 @@ func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Dur
 
 	if ctx.HasErrors() {
 		for _, err := range ctx.Errors() {
-			log.Errorf("CostModel.ComputeAllocation: %s", err)
+			log.Errorf("CostModel.ComputeAllocation: query context error %s", err)
 		}
 
 		return allocSet, ctx.ErrorCollection()

pkg/costmodel/router.go

@@ -615,6 +615,18 @@ func (a *Accesses) UpdateBigQueryInfoConfigs(w http.ResponseWriter, r *http.Requ
 	return
 }
 
+func (a *Accesses) UpdateAzureStorageConfigs(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	data, err := a.CloudProvider.UpdateConfig(r.Body, cloud.AzureStorageUpdateType)
+	if err != nil {
+		w.Write(WrapData(data, err))
+		return
+	}
+	w.Write(WrapData(data, err))
+	return
+}
+
 func (a *Accesses) UpdateConfigByKey(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	w.Header().Set("Content-Type", "application/json")
 	w.Header().Set("Access-Control-Allow-Origin", "*")