Cleanup. Separate function for cleaning local disk.

Signed-off-by: thomasvn <thomasnguyen96@gmail.com>

pkg/cloud/azure/storagebillingparser.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-	"runtime"
 	"strings"
 	"time"
 
@@ -48,6 +47,7 @@ func (asbp *AzureStorageBillingParser) ParseBillingData(start, end time.Time, re
 		return err
 	}
 	ctx := context.Background()
+	// Example blobNames: [ export/myExport/20240101-20240131/myExport_758a42af-0731-4edb-b498-1e523bb40f12.csv ]
 	blobNames, err := asbp.getMostRecentBlobs(start, end, client, ctx)
 	if err != nil {
 		asbp.ConnectionStatus = cloud.FailedConnection
@@ -60,15 +60,12 @@ func (asbp *AzureStorageBillingParser) ParseBillingData(start, end time.Time, re
 	}
 
 	for _, blobName := range blobNames {
-
-		// ----- INSTRUMENTATION -----
-		var m runtime.MemStats
-		runtime.ReadMemStats(&m)
-		log.Infof("HeapAlloc:  %v MB", m.HeapAlloc/1024/1024)
-		// ----- INSTRUMENTATION -----
-
-		if env.IsAzureParseBillingPaginated() {
-			localFilePath := filepath.Join(env.GetConfigPathWithDefault(env.DefaultConfigMountPath), "db", "cloudCost", "azurebilling.csv")
+		if env.IsAzureDownloadBillingDataToDisk() {
+			localPath := filepath.Join(env.GetConfigPathWithDefault(env.DefaultConfigMountPath), "db", "cloudCost")
+			localFilePath := filepath.Join(localPath, filepath.Base(blobName))
+			if _, err := asbp.deleteFilesOlderThan7d(localPath); err != nil {
+				log.Warnf("CloudCost: Azure: ParseBillingData: failed to remove the following stale files: %v", err)
+			}
 			err := asbp.DownloadBlobToFile(localFilePath, blobName, client, ctx)
 			if err != nil {
 				asbp.ConnectionStatus = cloud.FailedConnection
@@ -214,3 +211,34 @@ func (asbp *AzureStorageBillingParser) timeToMonthString(input time.Time) string
 	endOfMonth := input.AddDate(0, 1, -input.Day())
 	return startOfMonth.Format(format) + "-" + endOfMonth.Format(format)
 }
+
+// deleteFilesOlderThan7d recursively walks the directory specified and deletes
+// files which have not been modified in the last 7 days. Returns a list of
+// files deleted.
+func (asbp *AzureStorageBillingParser) deleteFilesOlderThan7d(localPath string) ([]string, error) {
+	duration := 7 * 24 * time.Hour
+	cleaned := []string{}
+	errs := []string{}
+
+	filepath.Walk(localPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			errs = append(errs, err.Error())
+			return err
+		}
+
+		if time.Since(info.ModTime()) > duration {
+			err := os.Remove(path)
+			if err != nil {
+				errs = append(errs, err.Error())
+			}
+			cleaned = append(cleaned, path)
+		}
+		return nil
+	})
+
+	if len(errs) == 0 {
+		return cleaned, nil
+	} else {
+		return cleaned, fmt.Errorf("deleteFilesOlderThan7d: %v", errs)
+	}
+}

pkg/cloud/azure/storageconnection.go

@@ -71,32 +71,30 @@ func (sc *StorageConnection) DownloadBlob(blobName string, client *azblob.Client
 
 // DownloadBlobToFile downloads the Azure Billing CSV to a local file
 func (sc *StorageConnection) DownloadBlobToFile(localFilePath string, blobName string, client *azblob.Client, ctx context.Context) error {
-	// Remove existing Azure Billing CSV on disk
+	// If file exists, don't download it again
 	if _, err := os.Stat(localFilePath); err == nil {
-		err := os.Remove(localFilePath)
-		if err != nil {
-			return fmt.Errorf("Azure: DownloadBlobToFile: failed to delete existing file %w", err)
-		}
+		log.DedupedInfof(3, "CloudCost: Azure: DownloadBlobToFile: file %v already exists, not downloading %v", localFilePath, blobName)
+		return nil
 	}
 
 	// Create filepath
 	dir := filepath.Dir(localFilePath)
 	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
-		return fmt.Errorf("Azure: DownloadBlobToFile: failed to create directory %w", err)
+		return fmt.Errorf("CloudCost: Azure: DownloadBlobToFile: failed to create directory %w", err)
 	}
 	fp, err := os.Create(localFilePath)
 	if err != nil {
-		return fmt.Errorf("Azure: DownloadBlobToFile: failed to create file %w", err)
+		return fmt.Errorf("CloudCost: Azure: DownloadBlobToFile: failed to create file %w", err)
 	}
 	defer fp.Close()
 
 	// Download newest Azure Billing CSV to disk
-	log.Infof("Azure: DownloadBlobToFile: retrieving blob: %v", blobName)
+	log.Infof("CloudCost: Azure: DownloadBlobToFile: retrieving blob: %v", blobName)
 	filesize, err := client.DownloadFile(ctx, sc.Container, blobName, fp, nil)
 	if err != nil {
-		return fmt.Errorf("Azure: DownloadBlobToFile: failed to download %w", err)
+		return fmt.Errorf("CloudCost: Azure: DownloadBlobToFile: failed to download %w", err)
 	}
-	log.Infof("Azure: DownloadBlobToFile: retrieved %v of size %d", blobName, filesize)
+	log.Infof("CloudCost: Azure: DownloadBlobToFile: retrieved %v of size %dMB", blobName, filesize/1024/1024)
 
 	return nil
 }

pkg/env/costmodelenv.go

@@ -21,9 +21,9 @@ const (
 	AlibabaAccessKeyIDEnvVar     = "ALIBABA_ACCESS_KEY_ID"
 	AlibabaAccessKeySecretEnvVar = "ALIBABA_SECRET_ACCESS_KEY"
 
-	AzureOfferIDEnvVar               = "AZURE_OFFER_ID"
-	AzureBillingAccountEnvVar        = "AZURE_BILLING_ACCOUNT"
-	AzureParseBillingPaginatedEnvVar = "AZURE_PARSE_BILLING_PAGINATED"
+	AzureOfferIDEnvVar                   = "AZURE_OFFER_ID"
+	AzureBillingAccountEnvVar            = "AZURE_BILLING_ACCOUNT"
+	AzureDownloadBillingDataToDiskEnvVar = "AZURE_DOWNLOAD_BILLING_DATA_TO_DISK"
 
 	KubecostNamespaceEnvVar        = "KUBECOST_NAMESPACE"
 	PodNameEnvVar                  = "POD_NAME"
@@ -307,8 +307,11 @@ func GetAzureBillingAccount() string {
 	return Get(AzureBillingAccountEnvVar, "")
 }
 
-func IsAzureParseBillingPaginated() bool {
-	return GetBool(AzureParseBillingPaginatedEnvVar, false)
+// IsAzureDownloadBillingDataToDisk returns the environment variable value for
+// AzureDownloadBillingDataToDiskEnvVar which indicates whether the Azure
+// Billing Data should be held in memory or written to disk.
+func IsAzureDownloadBillingDataToDisk() bool {
+	return GetBool(AzureDownloadBillingDataToDiskEnvVar, false)
 }
 
 // GetKubecostNamespace returns the environment variable value for KubecostNamespaceEnvVar which