cost-model/core/pkg/storage/bucketstorage_test.go

package storage

import (
	"fmt"
	"os"
	"path"
	"testing"

	"github.com/opencost/opencost/core/pkg/util/json"
)

// This suite of integration tests is meant to validate if an implementation of Storage that relies on a could
// bucket service properly implements the interface. To run these tests the env variable "TEST_BUCKET_CONFIG"
// must be set with the path to a valid bucket config as defined in the NewBucketStorage() function.

type testFileContent struct {
	Field1 int    `json:"field_1"`
	Field2 string `json:"field_2"`
}

var tfc = testFileContent{
	Field1: 101,
	Field2: "TEST_FILE_CONTENT",
}

const testpath = "opencost/storage/"

func createStorage(configPath string) (Storage, error) {

	bytes, err := os.ReadFile(configPath)
	if err != nil {
		return nil, fmt.Errorf("failed to read config file: %w", err)
	}
	store, err := NewBucketStorage(bytes)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize storage: %w", err)
	}

	return store, nil
}

func createFiles(files []string, testName string, store Storage) error {
	b, err := json.Marshal(tfc)
	if err != nil {
		return fmt.Errorf("failed to marshal file content: %w", err)
	}

	for _, fileName := range files {
		filePath := path.Join(testpath, testName, fileName)
		err = store.Write(filePath, b)
		if err != nil {
			return fmt.Errorf("failed to write file '%s': %w ", filePath, err)
		}
	}

	return nil
}

func cleanupFiles(files []string, testName string, store Storage) error {
	for _, fileName := range files {
		filePath := path.Join(testpath, testName, fileName)
		err := store.Remove(filePath)
		if err != nil {
			return fmt.Errorf("failed to remove file '%s': %w ", filePath, err)
		}
	}

	return nil
}

func TestBucketStorage_List(t *testing.T) {
	configPath := os.Getenv("TEST_BUCKET_CONFIG")
	if configPath == "" {
		t.Skip("skipping integration test, set environment variable TEST_BUCKET_CONFIG")
	}
	store, err := createStorage(configPath)
	if err != nil {
		t.Errorf("failed to create storage: %s", err.Error())
		return
	}

	testName := "list"

	fileNames := []string{
		"/file0.json",
		"/file1.json",
		"/dir0/file2.json",
		"/dir0/file3.json",
	}

	err = createFiles(fileNames, testName, store)
	if err != nil {
		t.Errorf("failed to create files: %s", err)
	}

	defer func() {
		err = cleanupFiles(fileNames, testName, store)
		if err != nil {
			t.Errorf("failed to clean up files: %s", err)
		}
	}()

	testCases := map[string]struct {
		path      string
		expected  []string
		expectErr bool
	}{
		"base dir files": {
			path: path.Join(testpath, testName),
			expected: []string{
				"file0.json",
				"file1.json",
			},
			expectErr: false,
		},
		"single nested dir files": {
			path: path.Join(testpath, testName, "dir0"),
			expected: []string{
				"file2.json",
				"file3.json",
			},
			expectErr: false,
		},
		"nonexistent dir files": {
			path:      path.Join(testpath, testName, "dir1"),
			expected:  []string{},
			expectErr: false,
		},
	}

	for name, tc := range testCases {
		t.Run(name, func(t *testing.T) {
			fileList, err := store.List(tc.path)
			if tc.expectErr == (err == nil) {
				if tc.expectErr {
					t.Errorf("expected error was not thrown")
					return
				}
				t.Errorf("unexpected error: %s", err.Error())
				return
			}

			if len(fileList) != len(tc.expected) {
				t.Errorf("file list length does not match expected length, actual: %d, expected: %d", len(fileList), len(tc.expected))
			}

			expectedSet := map[string]struct{}{}
			for _, expName := range tc.expected {
				expectedSet[expName] = struct{}{}
			}

			for _, file := range fileList {
				_, ok := expectedSet[file.Name]
				if !ok {
					t.Errorf("unexpect file in list %s", file.Name)
				}

				if file.Size == 0 {
					t.Errorf("file size is not set")
				}

				if file.ModTime.IsZero() {
					t.Errorf("file mod time is not set")
				}
			}
		})
	}
}

func TestBucketStorage_ListDirectories(t *testing.T) {
	configPath := os.Getenv("TEST_BUCKET_CONFIG")
	if configPath == "" {
		t.Skip("skipping integration test, set environment variable TEST_BUCKET_CONFIG")
	}
	store, err := createStorage(configPath)
	if err != nil {
		t.Errorf("failed to create storage: %s", err.Error())
		return
	}

	testName := "list_directories"

	fileNames := []string{
		"/file0.json",
		"/dir0/file2.json",
		"/dir0/file3.json",
		"/dir0/dir1/file4.json",
		"/dir0/dir2/file5.json",
	}

	err = createFiles(fileNames, testName, store)
	if err != nil {
		t.Errorf("failed to create files: %s", err)
	}

	defer func() {
		err = cleanupFiles(fileNames, testName, store)
		if err != nil {
			t.Errorf("failed to clean up files: %s", err)
		}
	}()

	testCases := map[string]struct {
		path      string
		expected  []string
		expectErr bool
	}{
		"base dir dir": {
			path: path.Join(testpath, testName),
			expected: []string{
				path.Join(testpath, testName, "dir0") + "/",
			},
			expectErr: false,
		},
		"single nested dir files": {
			path: path.Join(testpath, testName, "dir0"),
			expected: []string{
				path.Join(testpath, testName, "dir0", "dir1") + "/",
				path.Join(testpath, testName, "dir0", "dir2") + "/",
			},
			expectErr: false,
		},
		"dir with no sub dirs": {
			path:      path.Join(testpath, testName, "dir0/dir1"),
			expected:  []string{},
			expectErr: false,
		},
		"non-existent dir": {
			path:      path.Join(testpath, testName, "dir1"),
			expected:  []string{},
			expectErr: false,
		},
	}

	for name, tc := range testCases {
		t.Run(name, func(t *testing.T) {
			dirList, err := store.ListDirectories(tc.path)
			if tc.expectErr == (err == nil) {
				if tc.expectErr {
					t.Errorf("expected error was not thrown")
					return
				}
				t.Errorf("unexpected error: %s", err.Error())
				return
			}

			if len(dirList) != len(tc.expected) {
				t.Errorf("dir list length does not match expected length, actual: %d, expected: %d", len(dirList), len(tc.expected))
			}

			expectedSet := map[string]struct{}{}
			for _, expName := range tc.expected {
				expectedSet[expName] = struct{}{}
			}

			for _, dir := range dirList {
				_, ok := expectedSet[dir.Name]
				if !ok {
					t.Errorf("unexpect dir in list %s", dir.Name)
				}
			}
		})
	}
}

func TestBucketStorage_Exists(t *testing.T) {
	configPath := os.Getenv("TEST_BUCKET_CONFIG")
	if configPath == "" {
		t.Skip("skipping integration test, set environment variable TEST_BUCKET_CONFIG")
	}
	store, err := createStorage(configPath)
	if err != nil {
		t.Errorf("failed to create storage: %s", err.Error())
		return
	}

	testName := "exists"

	fileNames := []string{
		"/file0.json",
	}

	err = createFiles(fileNames, testName, store)
	if err != nil {
		t.Errorf("failed to create files: %s", err)
	}

	defer func() {
		err = cleanupFiles(fileNames, testName, store)
		if err != nil {
			t.Errorf("failed to clean up files: %s", err)
		}
	}()

	testCases := map[string]struct {
		path      string
		expected  bool
		expectErr bool
	}{
		"file exists": {
			path:      path.Join(testpath, testName, "file0.json"),
			expected:  true,
			expectErr: false,
		},
		"file does not exist": {
			path:      path.Join(testpath, testName, "file1.json"),
			expected:  false,
			expectErr: false,
		},
		"dir does not exist": {
			path:      path.Join(testpath, testName, "dir0/file.json"),
			expected:  false,
			expectErr: false,
		},
	}

	for name, tc := range testCases {
		t.Run(name, func(t *testing.T) {
			exists, err := store.Exists(tc.path)
			if tc.expectErr == (err == nil) {
				if tc.expectErr {
					t.Errorf("expected error was not thrown")
					return
				}
				t.Errorf("unexpected error: %s", err.Error())
				return
			}

			if exists != tc.expected {
				t.Errorf("file exists output did not match expected")
			}
		})
	}
}

func TestBucketStorage_Read(t *testing.T) {
	configPath := os.Getenv("TEST_BUCKET_CONFIG")
	if configPath == "" {
		t.Skip("skipping integration test, set environment variable TEST_BUCKET_CONFIG")
	}
	store, err := createStorage(configPath)
	if err != nil {
		t.Errorf("failed to create storage: %s", err.Error())
		return
	}

	testName := "read"

	fileNames := []string{
		"/file0.json",
	}

	err = createFiles(fileNames, testName, store)
	if err != nil {
		t.Errorf("failed to create files: %s", err)
	}

	defer func() {
		err = cleanupFiles(fileNames, testName, store)
		if err != nil {
			t.Errorf("failed to clean up files: %s", err)
		}
	}()

	testCases := map[string]struct {
		path      string
		expectErr bool
	}{
		"file exists": {
			path:      path.Join(testpath, testName, "file0.json"),
			expectErr: false,
		},
		"file does not exist": {
			path:      path.Join(testpath, testName, "file1.json"),
			expectErr: true,
		},
		"dir does not exist": {
			path:      path.Join(testpath, testName, "dir0/file.json"),
			expectErr: true,
		},
	}

	for name, tc := range testCases {
		t.Run(name, func(t *testing.T) {
			b, err := store.Read(tc.path)
			if tc.expectErr && err != nil {
				return
			}
			if tc.expectErr == (err == nil) {
				if tc.expectErr {
					t.Errorf("expected error was not thrown")
					return
				}
				t.Errorf("unexpected error: %s", err.Error())
				return
			}
			var content testFileContent
			err = json.Unmarshal(b, &content)
			if err != nil {
				t.Errorf("could not unmarshal file content")
				return
			}

			if content != tfc {
				t.Errorf("file content did not match writen value")
			}
		})
	}
}

func TestBucketStorage_Stat(t *testing.T) {
	configPath := os.Getenv("TEST_BUCKET_CONFIG")
	if configPath == "" {
		t.Skip("skipping integration test, set environment variable TEST_BUCKET_CONFIG")
	}
	store, err := createStorage(configPath)
	if err != nil {
		t.Errorf("failed to create storage: %s", err.Error())
		return
	}

	testName := "stat"

	fileNames := []string{
		"/file0.json",
	}

	err = createFiles(fileNames, testName, store)
	if err != nil {
		t.Errorf("failed to create files: %s", err)
	}

	defer func() {
		err = cleanupFiles(fileNames, testName, store)
		if err != nil {
			t.Errorf("failed to clean up files: %s", err)
		}
	}()

	testCases := map[string]struct {
		path      string
		expected  *StorageInfo
		expectErr bool
	}{
		"base dir": {
			path: path.Join(testpath, testName, "file0.json"),
			expected: &StorageInfo{
				Name: "file0.json",
				Size: 45,
			},
			expectErr: false,
		},
		"file does not exist": {
			path:      path.Join(testpath, testName, "file1.json"),
			expected:  nil,
			expectErr: true,
		},
	}

	for name, tc := range testCases {
		t.Run(name, func(t *testing.T) {
			status, err := store.Stat(tc.path)
			if tc.expectErr && err != nil {
				return
			}
			if tc.expectErr == (err == nil) {
				if tc.expectErr {
					t.Errorf("expected error was not thrown")
					return
				}
				t.Errorf("unexpected error: %s", err.Error())
				return
			}

			if status.Name != tc.expected.Name {
				t.Errorf("status name did name match expected, actual: %s, expected: %s", status.Name, tc.expected.Name)
			}

			if status.Size != tc.expected.Size {
				t.Errorf("status name did size match expected, actual: %d, expected: %d", status.Size, tc.expected.Size)
			}

			if status.ModTime.IsZero() {
				t.Errorf("status mod time is not set")
			}

		})
	}
}