|
|
@@ -1,13 +1,18 @@
|
|
|
package azure
|
|
|
|
|
|
import (
|
|
|
+ "bytes"
|
|
|
+ "compress/gzip"
|
|
|
+ "io"
|
|
|
+ "os"
|
|
|
+ "strings"
|
|
|
"testing"
|
|
|
"time"
|
|
|
)
|
|
|
|
|
|
func TestAzureStorageBillingParser_getMonthStrings(t *testing.T) {
|
|
|
asbp := AzureStorageBillingParser{}
|
|
|
- loc, _ := time.LoadLocation("UTC")
|
|
|
+ loc := time.UTC // Use time.UTC constant instead of LoadLocation
|
|
|
testCases := map[string]struct {
|
|
|
start time.Time
|
|
|
end time.Time
|
|
|
@@ -51,7 +56,7 @@ func TestAzureStorageBillingParser_getMonthStrings(t *testing.T) {
|
|
|
}
|
|
|
|
|
|
func TestAzureStorageBillingParser_parseCSV(t *testing.T) {
|
|
|
- loc, _ := time.LoadLocation("UTC")
|
|
|
+ loc := time.UTC // Use time.UTC constant instead of LoadLocation
|
|
|
start := time.Date(2021, 2, 1, 00, 00, 00, 00, loc)
|
|
|
end := time.Date(2021, 2, 3, 00, 00, 00, 00, loc)
|
|
|
tests := map[string]struct {
|
|
|
@@ -202,3 +207,344 @@ func TestAzureStorageBillingParser_parseCSV(t *testing.T) {
|
|
|
|
|
|
}
|
|
|
}
|
|
|
+
|
|
|
+func TestAzureStorageBillingParser_processLocalBillingFile(t *testing.T) {
|
|
|
+ loc := time.UTC
|
|
|
+ start := time.Date(2024, 10, 1, 0, 0, 0, 0, loc)
|
|
|
+ end := time.Date(2024, 11, 30, 0, 0, 0, 0, loc)
|
|
|
+
|
|
|
+ testCases := map[string]struct {
|
|
|
+ fileName string
|
|
|
+ expectedRows int
|
|
|
+ expectError bool
|
|
|
+ }{
|
|
|
+ "Gzipped file": {
|
|
|
+ fileName: "test_azure_billing.csv.gz",
|
|
|
+ expectedRows: 5,
|
|
|
+ expectError: false,
|
|
|
+ },
|
|
|
+ "Non-gzipped file": {
|
|
|
+ fileName: "test_azure_billing.csv",
|
|
|
+ expectedRows: 5,
|
|
|
+ expectError: false,
|
|
|
+ },
|
|
|
+ }
|
|
|
+
|
|
|
+ for name, tc := range testCases {
|
|
|
+ t.Run(name, func(t *testing.T) {
|
|
|
+ asbp := &AzureStorageBillingParser{}
|
|
|
+ filePath := valueCasesPath + tc.fileName
|
|
|
+
|
|
|
+ var rowCount int
|
|
|
+ resultFn := func(abv *BillingRowValues) error {
|
|
|
+ rowCount++
|
|
|
+ if abv == nil {
|
|
|
+ t.Error("Received nil BillingRowValues")
|
|
|
+ }
|
|
|
+ return nil
|
|
|
+ }
|
|
|
+
|
|
|
+ err := asbp.processLocalBillingFile(filePath, tc.fileName, start, end, resultFn)
|
|
|
+ if tc.expectError && err == nil {
|
|
|
+ t.Error("Expected error but got none")
|
|
|
+ }
|
|
|
+ if !tc.expectError && err != nil {
|
|
|
+ t.Fatalf("Unexpected error: %v", err)
|
|
|
+ }
|
|
|
+
|
|
|
+ if rowCount != tc.expectedRows {
|
|
|
+ t.Errorf("Expected %d rows, got %d rows", tc.expectedRows, rowCount)
|
|
|
+ }
|
|
|
+ })
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+func TestAzureStorageBillingParser_processStreamBillingData(t *testing.T) {
|
|
|
+ loc := time.UTC
|
|
|
+ start := time.Date(2024, 10, 1, 0, 0, 0, 0, loc)
|
|
|
+ end := time.Date(2024, 11, 30, 0, 0, 0, 0, loc)
|
|
|
+
|
|
|
+ testCases := map[string]struct {
|
|
|
+ fileName string
|
|
|
+ expectedRows int
|
|
|
+ }{
|
|
|
+ "Gzipped stream": {
|
|
|
+ fileName: "test_azure_billing.csv.gz",
|
|
|
+ expectedRows: 5,
|
|
|
+ },
|
|
|
+ "Non-gzipped stream": {
|
|
|
+ fileName: "test_azure_billing.csv",
|
|
|
+ expectedRows: 5,
|
|
|
+ },
|
|
|
+ }
|
|
|
+
|
|
|
+ for name, tc := range testCases {
|
|
|
+ t.Run(name, func(t *testing.T) {
|
|
|
+ asbp := &AzureStorageBillingParser{}
|
|
|
+
|
|
|
+ // Read file into memory to simulate stream
|
|
|
+ data, err := os.ReadFile(valueCasesPath + tc.fileName)
|
|
|
+ if err != nil {
|
|
|
+ t.Fatalf("Failed to read test file: %v", err)
|
|
|
+ }
|
|
|
+ streamReader := bytes.NewReader(data)
|
|
|
+
|
|
|
+ var rowCount int
|
|
|
+ resultFn := func(abv *BillingRowValues) error {
|
|
|
+ rowCount++
|
|
|
+ if abv == nil {
|
|
|
+ t.Error("Received nil BillingRowValues")
|
|
|
+ }
|
|
|
+ return nil
|
|
|
+ }
|
|
|
+
|
|
|
+ err = asbp.processStreamBillingData(streamReader, tc.fileName, start, end, resultFn)
|
|
|
+ if err != nil {
|
|
|
+ t.Fatalf("Unexpected error: %v", err)
|
|
|
+ }
|
|
|
+
|
|
|
+ if rowCount != tc.expectedRows {
|
|
|
+ t.Errorf("Expected %d rows, got %d rows", tc.expectedRows, rowCount)
|
|
|
+ }
|
|
|
+ })
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+func TestDecompressIfGzipped(t *testing.T) {
|
|
|
+ testCases := map[string]struct {
|
|
|
+ blobName string
|
|
|
+ content string
|
|
|
+ shouldGzip bool
|
|
|
+ expectError bool
|
|
|
+ }{
|
|
|
+ "Gzipped file with .gz extension": {
|
|
|
+ blobName: "billing_export.csv.gz",
|
|
|
+ content: "test,data\n1,2\n",
|
|
|
+ shouldGzip: true,
|
|
|
+ expectError: false,
|
|
|
+ },
|
|
|
+ "Gzipped file with .GZ extension (case insensitive)": {
|
|
|
+ blobName: "billing_export.CSV.GZ",
|
|
|
+ content: "test,data\n1,2\n",
|
|
|
+ shouldGzip: true,
|
|
|
+ expectError: false,
|
|
|
+ },
|
|
|
+ "Non-gzipped CSV file": {
|
|
|
+ blobName: "billing_export.csv",
|
|
|
+ content: "test,data\n1,2\n",
|
|
|
+ shouldGzip: false,
|
|
|
+ expectError: false,
|
|
|
+ },
|
|
|
+ "Non-gzipped file without extension": {
|
|
|
+ blobName: "billing_export",
|
|
|
+ content: "test,data\n1,2\n",
|
|
|
+ shouldGzip: false,
|
|
|
+ expectError: false,
|
|
|
+ },
|
|
|
+ }
|
|
|
+
|
|
|
+ for name, tc := range testCases {
|
|
|
+ t.Run(name, func(t *testing.T) {
|
|
|
+ var inputReader io.Reader
|
|
|
+
|
|
|
+ if tc.shouldGzip {
|
|
|
+ // Create gzipped content
|
|
|
+ var buf bytes.Buffer
|
|
|
+ gw := gzip.NewWriter(&buf)
|
|
|
+ _, err := gw.Write([]byte(tc.content))
|
|
|
+ if err != nil {
|
|
|
+ t.Fatalf("Failed to write gzip content: %v", err)
|
|
|
+ }
|
|
|
+ gw.Close()
|
|
|
+ inputReader = &buf
|
|
|
+ } else {
|
|
|
+ // Use plain content
|
|
|
+ inputReader = strings.NewReader(tc.content)
|
|
|
+ }
|
|
|
+
|
|
|
+ // Call decompressIfGzipped
|
|
|
+ reader, err := decompressIfGzipped(inputReader, tc.blobName)
|
|
|
+ if tc.expectError {
|
|
|
+ if err == nil {
|
|
|
+ t.Errorf("Expected error but got none")
|
|
|
+ }
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ if err != nil {
|
|
|
+ t.Fatalf("Unexpected error: %v", err)
|
|
|
+ }
|
|
|
+ defer reader.Close()
|
|
|
+
|
|
|
+ // Read and verify content
|
|
|
+ output, err := io.ReadAll(reader)
|
|
|
+ if err != nil {
|
|
|
+ t.Fatalf("Failed to read from reader: %v", err)
|
|
|
+ }
|
|
|
+
|
|
|
+ if string(output) != tc.content {
|
|
|
+ t.Errorf("Content mismatch. Expected: %q, Got: %q", tc.content, string(output))
|
|
|
+ }
|
|
|
+ })
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+func TestDecompressIfGzipped_InvalidGzip(t *testing.T) {
|
|
|
+ // Test with invalid gzip data
|
|
|
+ blobName := "invalid.csv.gz"
|
|
|
+ invalidData := strings.NewReader("this is not gzipped data")
|
|
|
+
|
|
|
+ reader, err := decompressIfGzipped(invalidData, blobName)
|
|
|
+ if err == nil {
|
|
|
+ if reader != nil {
|
|
|
+ reader.Close()
|
|
|
+ }
|
|
|
+ t.Error("Expected error for invalid gzip data, but got none")
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+func TestDecompressIfGzipped_EmptyGzipFile(t *testing.T) {
|
|
|
+ // Test with empty gzipped file
|
|
|
+ blobName := "empty.csv.gz"
|
|
|
+ var buf bytes.Buffer
|
|
|
+ gw := gzip.NewWriter(&buf)
|
|
|
+ gw.Close()
|
|
|
+
|
|
|
+ reader, err := decompressIfGzipped(&buf, blobName)
|
|
|
+ if err != nil {
|
|
|
+ t.Fatalf("Unexpected error for empty gzip file: %v", err)
|
|
|
+ }
|
|
|
+ defer reader.Close()
|
|
|
+
|
|
|
+ output, err := io.ReadAll(reader)
|
|
|
+ if err != nil {
|
|
|
+ t.Fatalf("Failed to read empty gzip file: %v", err)
|
|
|
+ }
|
|
|
+
|
|
|
+ if len(output) != 0 {
|
|
|
+ t.Errorf("Expected empty output, got %d bytes", len(output))
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+// TestDecompressIfGzipped_MultipleFiles tests processing multiple files in sequence
|
|
|
+// to ensure proper resource cleanup between iterations
|
|
|
+func TestDecompressIfGzipped_MultipleFiles(t *testing.T) {
|
|
|
+ testFiles := []struct {
|
|
|
+ name string
|
|
|
+ content string
|
|
|
+ shouldGzip bool
|
|
|
+ }{
|
|
|
+ {"file1.csv.gz", "data1,data2\nvalue1,value2\n", true},
|
|
|
+ {"file2.csv", "data3,data4\nvalue3,value4\n", false},
|
|
|
+ {"file3.csv.GZ", "data5,data6\nvalue5,value6\n", true},
|
|
|
+ }
|
|
|
+
|
|
|
+ for _, tf := range testFiles {
|
|
|
+ t.Run(tf.name, func(t *testing.T) {
|
|
|
+ var input io.Reader
|
|
|
+ if tf.shouldGzip {
|
|
|
+ var buf bytes.Buffer
|
|
|
+ gw := gzip.NewWriter(&buf)
|
|
|
+ _, err := gw.Write([]byte(tf.content))
|
|
|
+ if err != nil {
|
|
|
+ t.Fatalf("Failed to write gzip data: %v", err)
|
|
|
+ }
|
|
|
+ gw.Close()
|
|
|
+ input = &buf
|
|
|
+ } else {
|
|
|
+ input = strings.NewReader(tf.content)
|
|
|
+ }
|
|
|
+
|
|
|
+ reader, err := decompressIfGzipped(input, tf.name)
|
|
|
+ if err != nil {
|
|
|
+ t.Fatalf("Failed to decompress %s: %v", tf.name, err)
|
|
|
+ }
|
|
|
+ defer reader.Close()
|
|
|
+
|
|
|
+ output, err := io.ReadAll(reader)
|
|
|
+ if err != nil {
|
|
|
+ t.Fatalf("Failed to read from reader for %s: %v", tf.name, err)
|
|
|
+ }
|
|
|
+
|
|
|
+ if string(output) != tf.content {
|
|
|
+ t.Errorf("Content mismatch for %s. Expected: %q, Got: %q", tf.name, tf.content, string(output))
|
|
|
+ }
|
|
|
+ })
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+// TestDecompressIfGzipped_CaseInsensitiveExtension tests various case combinations
|
|
|
+func TestDecompressIfGzipped_CaseInsensitiveExtension(t *testing.T) {
|
|
|
+ testCases := []string{
|
|
|
+ "file.gz",
|
|
|
+ "file.GZ",
|
|
|
+ "file.Gz",
|
|
|
+ "file.gZ",
|
|
|
+ }
|
|
|
+
|
|
|
+ content := "test,data\n1,2\n"
|
|
|
+ for _, blobName := range testCases {
|
|
|
+ t.Run(blobName, func(t *testing.T) {
|
|
|
+ var buf bytes.Buffer
|
|
|
+ gw := gzip.NewWriter(&buf)
|
|
|
+ _, err := gw.Write([]byte(content))
|
|
|
+ if err != nil {
|
|
|
+ t.Fatalf("Failed to write gzip data: %v", err)
|
|
|
+ }
|
|
|
+ gw.Close()
|
|
|
+
|
|
|
+ reader, err := decompressIfGzipped(&buf, blobName)
|
|
|
+ if err != nil {
|
|
|
+ t.Fatalf("Failed to decompress %s: %v", blobName, err)
|
|
|
+ }
|
|
|
+ defer reader.Close()
|
|
|
+
|
|
|
+ output, err := io.ReadAll(reader)
|
|
|
+ if err != nil {
|
|
|
+ t.Fatalf("Failed to read from reader: %v", err)
|
|
|
+ }
|
|
|
+
|
|
|
+ if string(output) != content {
|
|
|
+ t.Errorf("Content mismatch. Expected: %q, Got: %q", content, string(output))
|
|
|
+ }
|
|
|
+ })
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+// TestDecompressIfGzipped_LargeFile tests handling of larger gzipped files
|
|
|
+func TestDecompressIfGzipped_LargeFile(t *testing.T) {
|
|
|
+ // Create a larger CSV content (1000 rows)
|
|
|
+ var contentBuilder strings.Builder
|
|
|
+ contentBuilder.WriteString("col1,col2,col3,col4\n")
|
|
|
+ for i := 0; i < 1000; i++ {
|
|
|
+ contentBuilder.WriteString("value1,value2,value3,value4\n")
|
|
|
+ }
|
|
|
+ content := contentBuilder.String()
|
|
|
+
|
|
|
+ // Gzip the content
|
|
|
+ var buf bytes.Buffer
|
|
|
+ gw := gzip.NewWriter(&buf)
|
|
|
+ _, err := gw.Write([]byte(content))
|
|
|
+ if err != nil {
|
|
|
+ t.Fatalf("Failed to write gzip data: %v", err)
|
|
|
+ }
|
|
|
+ gw.Close()
|
|
|
+
|
|
|
+ blobName := "large_file.csv.gz"
|
|
|
+ reader, err := decompressIfGzipped(&buf, blobName)
|
|
|
+ if err != nil {
|
|
|
+ t.Fatalf("Failed to decompress large file: %v", err)
|
|
|
+ }
|
|
|
+ defer reader.Close()
|
|
|
+
|
|
|
+ output, err := io.ReadAll(reader)
|
|
|
+ if err != nil {
|
|
|
+ t.Fatalf("Failed to read large file: %v", err)
|
|
|
+ }
|
|
|
+
|
|
|
+ if string(output) != content {
|
|
|
+ t.Errorf("Content mismatch for large file. Expected %d bytes, got %d bytes", len(content), len(output))
|
|
|
+ }
|
|
|
+
|
|
|
+ t.Logf("Successfully processed large gzipped file with %d bytes", len(output))
|
|
|
+}
|
Hunter Metcalfe