Fix errors when creating/deleting local files

Signed-off-by: thomasvn <thomasnguyen96@gmail.com>

pkg/cloud/azure/storagebillingparser.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
+	"runtime"
 	"strings"
 	"time"
 
@@ -58,23 +60,30 @@ func (asbp *AzureStorageBillingParser) ParseBillingData(start, end time.Time, re
 	}
 
 	for _, blobName := range blobNames {
+
+		// ----- INSTRUMENTATION -----
+		var m runtime.MemStats
+		runtime.ReadMemStats(&m)
+		log.Infof("HeapAlloc:  %v MB", m.HeapAlloc/1024/1024)
+		// ----- INSTRUMENTATION -----
+
 		if env.IsAzureParseBillingPaginated() {
-			localFilepath := "/var/configs/db/cloudCost/azurebilling.csv"
-			err := asbp.DownloadBlobToFile(localFilepath, blobName, client, ctx)
+			localFilePath := filepath.Join(env.GetConfigPathWithDefault(env.DefaultConfigMountPath), "db", "cloudCost", "azurebilling.csv")
+			err := asbp.DownloadBlobToFile(localFilePath, blobName, client, ctx)
 			if err != nil {
 				asbp.ConnectionStatus = cloud.FailedConnection
 				return err
 			}
-			file, err := os.Open(localFilepath)
+			fp, err := os.Open(localFilePath)
 			if err != nil {
 				asbp.ConnectionStatus = cloud.FailedConnection
 				return err
 			}
-			defer file.Close()
-			err2 = asbp.parseCSV(start, end, csv.NewReader(bytes.NewReader(blobBytes)), resultFn)
-			if err2 != nil {
+			defer fp.Close()
+			err = asbp.parseCSV(start, end, csv.NewReader(fp), resultFn)
+			if err != nil {
 				asbp.ConnectionStatus = cloud.ParseError
-				return err2
+				return err
 			}
 		} else {
 			blobBytes, err2 := asbp.DownloadBlob(blobName, client, ctx)

pkg/cloud/azure/storageconnection.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
@@ -68,32 +69,33 @@ func (sc *StorageConnection) DownloadBlob(blobName string, client *azblob.Client
 	return downloadedData.Bytes(), nil
 }
 
-// DownloadBlobToFile downloads the Azure Cloud Billing CSV file to a local file
+// DownloadBlobToFile downloads the Azure Billing CSV to a local file
 func (sc *StorageConnection) DownloadBlobToFile(localFilePath string, blobName string, client *azblob.Client, ctx context.Context) error {
-	log.Infof("Azure: DownloadBlobToFile: retrieving blob: %v", blobName)
-
-	// Remove existing
+	// Remove existing Azure Billing CSV on disk
 	if _, err := os.Stat(localFilePath); err == nil {
 		err := os.Remove(localFilePath)
 		if err != nil {
 			return fmt.Errorf("Azure: DownloadBlobToFile: failed to delete existing file %w", err)
 		}
-	} else {
-		return fmt.Errorf("Azure: DownloadBlobToFile: error checking for file %w", err)
 	}
 
-	// Create new
-	file, err := os.Create(localFilePath)
+	// Create filepath
+	dir := filepath.Dir(localFilePath)
+	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
+		return fmt.Errorf("Azure: DownloadBlobToFile: failed to create directory %w", err)
+	}
+	fp, err := os.Create(localFilePath)
 	if err != nil {
 		return fmt.Errorf("Azure: DownloadBlobToFile: failed to create file %w", err)
 	}
-	defer file.Close()
+	defer fp.Close()
 
-	filesize, err := client.DownloadFile(ctx, sc.Container, blobName, file, nil)
+	// Download newest Azure Billing CSV to disk
+	log.Infof("Azure: DownloadBlobToFile: retrieving blob: %v", blobName)
+	filesize, err := client.DownloadFile(ctx, sc.Container, blobName, fp, nil)
 	if err != nil {
 		return fmt.Errorf("Azure: DownloadBlobToFile: failed to download %w", err)
 	}
-
 	log.Infof("Azure: DownloadBlobToFile: retrieved %v of size %d", blobName, filesize)
 
 	return nil