Adding a GZipEncoder adapter for compressing encoded data. Added a MemoryStorage implementation for use in tests that leverages a file system interface in memory.

core/pkg/exporter/encoder.go

@@ -1,6 +1,8 @@
 package exporter
 
 import (
+	"bytes"
+	"compress/gzip"
 	"encoding"
 
 	"github.com/opencost/opencost/core/pkg/util/json"
@@ -64,3 +66,41 @@ func (j *JSONEncoder[T]) Encode(data *T) ([]byte, error) {
 func (j *JSONEncoder[T]) FileExt() string {
 	return "json"
 }
+
+type GZipEncoder[T any] struct {
+	encoder Encoder[T]
+}
+
+// NewGZipEncoder creates a new GZip encoder which wraps the provided encoder.
+// The encoder is used to encode the data before compressing it with GZip.
+func NewGZipEncoder[T any](encoder Encoder[T]) Encoder[T] {
+	return &GZipEncoder[T]{
+		encoder: encoder,
+	}
+}
+
+// Encode encodes the provided data of type T into a byte slice using JSON encoding.
+func (gz *GZipEncoder[T]) Encode(data *T) ([]byte, error) {
+	encoded, err := gz.encoder.Encode(data)
+	if err != nil {
+		return nil, err
+	}
+
+	var buf bytes.Buffer
+
+	gzWriter, err := gzip.NewWriterLevel(&buf, gzip.BestCompression)
+	if err != nil {
+		return nil, err
+	}
+
+	gzWriter.Write(encoded)
+	gzWriter.Close()
+
+	return buf.Bytes(), nil
+}
+
+// FileExt returns the file extension for the encoded data. In this case, it returns the wrapped encoder's
+// file extension with ".gz" appended to indicate that the data is compressed with GZip.
+func (gz *GZipEncoder[T]) FileExt() string {
+	return gz.encoder.FileExt() + ".gz"
+}

core/pkg/heartbeat/heartbeat_test.go

@@ -28,7 +28,7 @@ func (m *MockHeartbeatMetadataProvider) GetMetadata() map[string]any {
 
 func TestHeartbeatExporter(t *testing.T) {
 	mdp := NewMockHeartbeatMetadataProvider()
-	store := NewMockStorage()
+	store := storage.NewMemoryStorage()
 
 	controller := NewHeartbeatExportController(MockClusterId, store, mdp)
 
@@ -64,7 +64,7 @@ func TestHeartbeatExporter(t *testing.T) {
 			t.Fatalf("Failed to unmarshal heartbeat data: %v", err)
 		}
 
-		fmt.Printf("%s\n%s\n\n", f, string(data))
+		fmt.Printf("%s: %d bytes\n%s\n\n", f, len(data), string(data))
 
 		if hb.Metadata["cluster_id"] != MockClusterId {
 			t.Fatalf("Expected cluster ID %s, got %s", MockClusterId, hb.Metadata["cluster_id"])

core/pkg/heartbeat/storage_test.go

@@ -1,235 +0,0 @@
-package heartbeat
-
-import (
-	"fmt"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/opencost/opencost/core/pkg/storage"
-)
-
-type file struct {
-	name     string
-	contents []byte
-}
-
-func newFile(name string, contents []byte) *file {
-	return &file{
-		name:     name,
-		contents: contents,
-	}
-}
-
-type dir struct {
-	name  string
-	dirs  map[string]*dir
-	files map[string]*file
-}
-
-func newDir(name string) *dir {
-	return &dir{
-		name:  name,
-		dirs:  make(map[string]*dir),
-		files: make(map[string]*file),
-	}
-}
-
-func (d *dir) size() int64 {
-	var size int64
-	for _, f := range d.files {
-		size += int64(len(f.contents))
-	}
-	for _, subdir := range d.dirs {
-		size += subdir.size()
-	}
-	return size
-}
-
-func (d *dir) addFile(f *file) {
-	d.files[f.name] = f
-}
-
-func (d *dir) addDir(subdir *dir) {
-	d.dirs[subdir.name] = subdir
-}
-
-func (d *dir) deleteFile(name string) {
-	delete(d.files, name)
-}
-
-func (d *dir) deleteDir(name string) {
-	delete(d.dirs, name)
-}
-
-type MockStorage struct {
-	s map[string][]byte
-	d *dir
-}
-
-func NewMockStorage() *MockStorage {
-	return &MockStorage{
-		s: make(map[string][]byte),
-		d: newDir(""),
-	}
-}
-
-// StorageType returns a string identifier for the type of storage used by the implementation.
-func (ms *MockStorage) StorageType() storage.StorageType {
-	return storage.StorageType("mock")
-}
-
-// FullPath returns the storage working path combined with the path provided
-func (ms *MockStorage) FullPath(path string) string {
-	return path
-}
-
-// Stat returns the StorageStats for the specific path.
-func (ms *MockStorage) Stat(path string) (*storage.StorageInfo, error) {
-	path = filepath.Clean(path)
-	if data, ok := ms.s[path]; ok {
-		return &storage.StorageInfo{
-			Name:    path,
-			Size:    int64(len(data)),
-			ModTime: time.Now(),
-		}, nil
-	}
-
-	return nil, fmt.Errorf("no valid data found")
-}
-
-// Read uses the relative path of the storage combined with the provided path to
-// read the contents.
-func (ms *MockStorage) Read(path string) ([]byte, error) {
-	path = filepath.Clean(path)
-
-	if data, ok := ms.s[path]; ok {
-		return data, nil
-	}
-
-	return nil, fmt.Errorf("no valid data found")
-}
-
-// Write uses the relative path of the storage combined with the provided path
-// to write a new file or overwrite an existing file.
-func (ms *MockStorage) Write(path string, data []byte) error {
-	paths, pFile := pathsAndFile(path)
-
-	f := newFile(pFile, data)
-	currentDir := writeDir(ms.d, paths)
-
-	currentDir.addFile(f)
-	ms.s[path] = data
-	return nil
-}
-
-// Remove uses the relative path of the storage combined with the provided path to
-// remove a file from storage permanently.
-func (ms *MockStorage) Remove(path string) error {
-	paths, pFile := pathsAndFile(path)
-
-	currentDir, err := searchDir(ms.d, paths)
-	if err != nil {
-		return err
-	}
-
-	currentDir.deleteFile(pFile)
-
-	delete(ms.s, path)
-	return nil
-}
-
-// Exists uses the relative path of the storage combined with the provided path to
-// determine if the file exists.
-func (ms *MockStorage) Exists(path string) (bool, error) {
-	path = filepath.Clean(path)
-
-	_, ok := ms.s[path]
-	return ok, nil
-}
-
-// List uses the relative path of the storage combined with the provided path to return
-// storage information for the files.
-func (ms *MockStorage) List(path string) ([]*storage.StorageInfo, error) {
-	paths := toPaths(path)
-	currentDir, err := searchDir(ms.d, paths)
-	if err != nil {
-		return nil, err
-	}
-
-	storageInfos := make([]*storage.StorageInfo, 0, len(currentDir.files))
-	for _, f := range currentDir.files {
-		storageInfos = append(storageInfos, &storage.StorageInfo{
-			Name:    f.name,
-			Size:    int64(len(f.contents)),
-			ModTime: time.Now(),
-		})
-	}
-
-	return storageInfos, nil
-}
-
-// ListDirectories uses the relative path of the storage combined with the provided path
-// to return storage information for only directories contained along the path. This
-// functions as List, but returns storage information for only directories.
-func (ms *MockStorage) ListDirectories(path string) ([]*storage.StorageInfo, error) {
-	paths := toPaths(path)
-	currentDir, err := searchDir(ms.d, paths)
-	if err != nil {
-		return nil, err
-	}
-
-	storageInfos := make([]*storage.StorageInfo, 0, len(currentDir.files))
-	for _, d := range currentDir.dirs {
-		storageInfos = append(storageInfos, &storage.StorageInfo{
-			Name:    d.name,
-			Size:    d.size(),
-			ModTime: time.Now(),
-		})
-	}
-
-	return storageInfos, nil
-}
-
-func toPaths(path string) []string {
-	path = filepath.Clean(path)
-	if path[len(path)-1] == filepath.Separator {
-		path = path[:len(path)-1]
-	}
-	return strings.Split(path, string(filepath.Separator))
-}
-
-func pathsAndFile(path string) ([]string, string) {
-	path = filepath.Clean(path)
-	pDir, pFile := filepath.Split(path)
-	pDir = filepath.Dir(pDir)
-	return strings.Split(pDir, string(filepath.Separator)), pFile
-}
-
-func writeDir(d *dir, paths []string) *dir {
-	currentDir := d
-
-	for i := 0; i < len(paths); i++ {
-		dirName := paths[i]
-		if _, ok := currentDir.dirs[dirName]; !ok {
-			currentDir.addDir(newDir(dirName))
-		}
-		currentDir = currentDir.dirs[dirName]
-	}
-
-	return currentDir
-}
-
-func searchDir(d *dir, paths []string) (*dir, error) {
-	currentDir := d
-
-	for i := 0; i < len(paths); i++ {
-		dirName := paths[i]
-		if _, ok := currentDir.dirs[dirName]; !ok {
-			return nil, fmt.Errorf("directory %s not found", filepath.Join(paths[:i+1]...))
-		}
-		currentDir = currentDir.dirs[dirName]
-	}
-
-	return currentDir, nil
-}

core/pkg/storage/memfile/memfile.go

@@ -0,0 +1,124 @@
+package memfile
+
+import (
+	"iter"
+	"maps"
+	"time"
+)
+
+// MemoryFile represents a file in memory storage. It's part of the directory tree
+// structure used to look up files by path.
+type MemoryFile struct {
+	Name     string
+	Contents []byte
+	ModTime  time.Time
+
+	directory *MemoryDirectory
+}
+
+// Size returns the size of the file in bytes.
+func (mf *MemoryFile) Size() int64 {
+	return int64(len(mf.Contents))
+}
+
+// NewMemoryFile creates a new MemoryFile instance with the provided name and and byte contents.
+func NewMemoryFile(name string, contents []byte) *MemoryFile {
+	return &MemoryFile{
+		Name:      name,
+		Contents:  contents,
+		ModTime:   time.Now().UTC(),
+		directory: nil,
+	}
+}
+
+// MemoryDirectory represents a directory in memory storage. It is the root of the file system
+// tree structure used to look up files by path.
+type MemoryDirectory struct {
+	Name    string
+	ModTime time.Time
+
+	dirs      map[string]*MemoryDirectory
+	files     map[string]*MemoryFile
+	directory *MemoryDirectory
+}
+
+// NewMemoryDirectory creates a new Directory instance with the provided path name.
+func NewMemoryDirectory(name string) *MemoryDirectory {
+	return &MemoryDirectory{
+		Name:  name,
+		dirs:  make(map[string]*MemoryDirectory),
+		files: make(map[string]*MemoryFile),
+	}
+}
+
+// Size returns the size of all subdirectories and files within this directory.
+func (d *MemoryDirectory) Size() int64 {
+	var size int64
+	for _, f := range d.files {
+		size += f.Size()
+	}
+	for _, subdir := range d.dirs {
+		size += subdir.Size()
+	}
+	return size
+}
+
+// AddFile adds a file to the directory. Note that files can only exist within a single directory
+// at a time.
+func (d *MemoryDirectory) AddFile(f *MemoryFile) {
+	if f.directory != nil {
+		f.directory.RemoveFile(f.Name)
+		f.directory = nil
+	}
+
+	d.files[f.Name] = f
+	d.ModTime = time.Now().UTC()
+	f.directory = d
+}
+
+// AddDirectory adds a subdirectory to the parent directory. Note that directories can only exist within a single directory.
+func (d *MemoryDirectory) AddDirectory(subdir *MemoryDirectory) {
+	if subdir.directory != nil {
+		subdir.directory.RemoveDirectory(subdir.Name)
+		subdir.directory = nil
+	}
+
+	d.dirs[subdir.Name] = subdir
+	d.ModTime = time.Now().UTC()
+	subdir.directory = d
+}
+
+// RemoveFile removes a file from the directoory tree.
+func (d *MemoryDirectory) RemoveFile(name string) {
+	if _, ok := d.files[name]; ok {
+		delete(d.files, name)
+		d.ModTime = time.Now().UTC()
+	}
+}
+
+// RemoveDirectory remove a subdirectory from the directory tree.
+func (d *MemoryDirectory) RemoveDirectory(name string) {
+	if _, ok := d.dirs[name]; ok {
+		delete(d.dirs, name)
+		d.ModTime = time.Now().UTC()
+	}
+}
+
+// FileCount returns the total number of files in this directory.
+func (d *MemoryDirectory) FileCount() int {
+	return len(d.files)
+}
+
+// DirCount returns the total number of subdirectories in this directory.
+func (d *MemoryDirectory) DirCount() int {
+	return len(d.dirs)
+}
+
+// Files returns a slice of files located within this directory.
+func (d *MemoryDirectory) Files() iter.Seq[*MemoryFile] {
+	return maps.Values(d.files)
+}
+
+func (d *MemoryDirectory) Directories() iter.Seq[*MemoryDirectory] {
+	return maps.Values(d.dirs)
+}

core/pkg/storage/memfile/util.go

@@ -0,0 +1,57 @@
+package memfile
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+// SplitPaths splits the directory path into a slice of directory names.
+func SplitPaths(path string) []string {
+	path = filepath.Clean(path)
+	if path[len(path)-1] == filepath.Separator {
+		path = path[:len(path)-1]
+	}
+	return strings.Split(path, string(filepath.Separator))
+}
+
+// Split splits the path into a slice of directory names and the file name.
+func Split(path string) ([]string, string) {
+	path = filepath.Clean(path)
+	pDir, pFile := filepath.Split(path)
+	pDir = filepath.Dir(pDir)
+
+	return strings.Split(pDir, string(filepath.Separator)), pFile
+}
+
+// CreateSubdirectory creates the necessary subdirectories within the provided MemoryDirectory.
+func CreateSubdirectory(d *MemoryDirectory, paths []string) *MemoryDirectory {
+	currentDir := d
+
+	for i := 0; i < len(paths); i++ {
+		dirName := paths[i]
+		if _, ok := currentDir.dirs[dirName]; !ok {
+			currentDir.AddDirectory(NewMemoryDirectory(dirName))
+		}
+		currentDir = currentDir.dirs[dirName]
+	}
+
+	return currentDir
+}
+
+// FindSubdirectory searches through the provided path slice starting with the provided directory,
+// and returns the correct MemoryDirectory if it exists. If the directory does not exist, an error is
+// returned containing the path where the find failed.
+func FindSubdirectory(d *MemoryDirectory, paths []string) (*MemoryDirectory, error) {
+	currentDir := d
+
+	for i := 0; i < len(paths); i++ {
+		dirName := paths[i]
+		if _, ok := currentDir.dirs[dirName]; !ok {
+			return nil, fmt.Errorf("directory %s not found", filepath.Join(paths[:i+1]...))
+		}
+		currentDir = currentDir.dirs[dirName]
+	}
+
+	return currentDir, nil
+}

core/pkg/storage/memorystorage.go

@@ -0,0 +1,162 @@
+package storage
+
+import (
+	"fmt"
+	"path/filepath"
+	"sync"
+
+	"github.com/opencost/opencost/core/pkg/log"
+	"github.com/opencost/opencost/core/pkg/storage/memfile"
+)
+
+// MemoryStorage is a thread-safe in-memory file system storage implementation. It can be used for testing storage.Storage dependents
+// or to serve as a lightweight storage implementation within a production system.
+type MemoryStorage struct {
+	lock        sync.Mutex
+	directPaths map[string]*memfile.MemoryFile
+	fileTree    *memfile.MemoryDirectory
+}
+
+// NewMemoryStorage creates a new in-memory file system storage implementation.
+func NewMemoryStorage() *MemoryStorage {
+	return &MemoryStorage{
+		directPaths: make(map[string]*memfile.MemoryFile),
+		fileTree:    memfile.NewMemoryDirectory(""),
+	}
+}
+
+// StorageType returns a string identifier for the type of storage used by the implementation.
+func (ms *MemoryStorage) StorageType() StorageType {
+	return StorageTypeMemory
+}
+
+// FullPath returns the storage working path combined with the path provided
+func (ms *MemoryStorage) FullPath(path string) string {
+	return path
+}
+
+// Stat returns the StorageStats for the specific path.
+func (ms *MemoryStorage) Stat(path string) (*StorageInfo, error) {
+	path = filepath.Clean(path)
+	if file, ok := ms.directPaths[path]; ok {
+		return &StorageInfo{
+			Name:    file.Name,
+			Size:    file.Size(),
+			ModTime: file.ModTime,
+		}, nil
+	}
+
+	return nil, fmt.Errorf("file not found: %s - %w", path, DoesNotExistError)
+}
+
+// Read uses the relative path of the storage combined with the provided path to
+// read the contents.
+func (ms *MemoryStorage) Read(path string) ([]byte, error) {
+	path = filepath.Clean(path)
+
+	if file, ok := ms.directPaths[path]; ok {
+		return file.Contents, nil
+	}
+
+	return nil, fmt.Errorf("file not found: %s - %w", path, DoesNotExistError)
+}
+
+// Write uses the relative path of the storage combined with the provided path
+// to write a new file or overwrite an existing file.
+func (ms *MemoryStorage) Write(path string, data []byte) error {
+	paths, pFile := memfile.Split(path)
+
+	f := memfile.NewMemoryFile(pFile, data)
+	currentDir := memfile.CreateSubdirectory(ms.fileTree, paths)
+
+	currentDir.AddFile(f)
+	ms.directPaths[path] = f
+	return nil
+}
+
+// Remove uses the relative path of the storage combined with the provided path to
+// remove a file from storage permanently.
+func (ms *MemoryStorage) Remove(path string) error {
+	ms.lock.Lock()
+	defer ms.lock.Unlock()
+
+	path = filepath.Clean(path)
+	paths, pFile := memfile.Split(path)
+
+	currentDir, err := memfile.FindSubdirectory(ms.fileTree, paths)
+	if err != nil {
+		return fmt.Errorf("file not found: %s - %w", path, DoesNotExistError)
+	}
+
+	currentDir.RemoveFile(pFile)
+
+	delete(ms.directPaths, path)
+	return nil
+}
+
+// Exists uses the relative path of the storage combined with the provided path to
+// determine if the file exists.
+func (ms *MemoryStorage) Exists(path string) (bool, error) {
+	ms.lock.Lock()
+	defer ms.lock.Unlock()
+
+	path = filepath.Clean(path)
+
+	_, ok := ms.directPaths[path]
+	return ok, nil
+}
+
+// List uses the relative path of the storage combined with the provided path to return
+// storage information for the files.
+func (ms *MemoryStorage) List(path string) ([]*StorageInfo, error) {
+	ms.lock.Lock()
+	defer ms.lock.Unlock()
+
+	paths := memfile.SplitPaths(path)
+	currentDir, err := memfile.FindSubdirectory(ms.fileTree, paths)
+	if err != nil {
+		// contract for bucket storages returns an empty list in this case
+		// so just log a warning, and return an empty list
+		log.Warnf("failed to resolve path: %s - %s", path, err)
+		return []*StorageInfo{}, nil
+	}
+
+	storageInfos := make([]*StorageInfo, 0, currentDir.FileCount())
+	for f := range currentDir.Files() {
+		storageInfos = append(storageInfos, &StorageInfo{
+			Name:    f.Name,
+			Size:    f.Size(),
+			ModTime: f.ModTime,
+		})
+	}
+
+	return storageInfos, nil
+}
+
+// ListDirectories uses the relative path of the storage combined with the provided path
+// to return storage information for only directories contained along the path. This
+// functions as List, but returns storage information for only directories.
+func (ms *MemoryStorage) ListDirectories(path string) ([]*StorageInfo, error) {
+	ms.lock.Lock()
+	defer ms.lock.Unlock()
+
+	paths := memfile.SplitPaths(path)
+	currentDir, err := memfile.FindSubdirectory(ms.fileTree, paths)
+	if err != nil {
+		// contract for bucket storages returns an empty list in this case
+		// so just log a warning, and return an empty list
+		log.Warnf("failed to resolve path: %s - %s", path, err)
+		return []*StorageInfo{}, nil
+	}
+
+	storageInfos := make([]*StorageInfo, 0, currentDir.DirCount())
+	for d := range currentDir.Directories() {
+		storageInfos = append(storageInfos, &StorageInfo{
+			Name:    filepath.Join(append(paths, d.Name)...) + "/",
+			Size:    d.Size(),
+			ModTime: d.ModTime,
+		})
+	}
+
+	return storageInfos, nil
+}

core/pkg/storage/memorystorage_test.go

@@ -0,0 +1,382 @@
+package storage
+
+import (
+	"path"
+	"testing"
+
+	"github.com/opencost/opencost/core/pkg/util/json"
+)
+
+func TestMemoryStorage_List(t *testing.T) {
+	store := NewMemoryStorage()
+	testName := "list"
+
+	fileNames := []string{
+		"/file0.json",
+		"/file1.json",
+		"/dir0/file2.json",
+		"/dir0/file3.json",
+	}
+
+	err := createFiles(fileNames, testName, store)
+	if err != nil {
+		t.Errorf("failed to create files: %s", err)
+	}
+
+	defer func() {
+		err = cleanupFiles(fileNames, testName, store)
+		if err != nil {
+			t.Errorf("failed to clean up files: %s", err)
+		}
+	}()
+
+	testCases := map[string]struct {
+		path      string
+		expected  []string
+		expectErr bool
+	}{
+		"base dir files": {
+			path: path.Join(testpath, testName),
+			expected: []string{
+				"file0.json",
+				"file1.json",
+			},
+			expectErr: false,
+		},
+		"single nested dir files": {
+			path: path.Join(testpath, testName, "dir0"),
+			expected: []string{
+				"file2.json",
+				"file3.json",
+			},
+			expectErr: false,
+		},
+		"nonexistent dir files": {
+			path:      path.Join(testpath, testName, "dir1"),
+			expected:  []string{},
+			expectErr: false,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			fileList, err := store.List(tc.path)
+			if tc.expectErr == (err == nil) {
+				if tc.expectErr {
+					t.Errorf("expected error was not thrown")
+					return
+				}
+				t.Errorf("unexpected error: %s", err.Error())
+				return
+			}
+
+			if len(fileList) != len(tc.expected) {
+				t.Errorf("file list length does not match expected length, actual: %d, expected: %d", len(fileList), len(tc.expected))
+			}
+
+			expectedSet := map[string]struct{}{}
+			for _, expName := range tc.expected {
+				expectedSet[expName] = struct{}{}
+			}
+
+			for _, file := range fileList {
+				_, ok := expectedSet[file.Name]
+				if !ok {
+					t.Errorf("unexpect file in list %s", file.Name)
+				}
+
+				if file.Size == 0 {
+					t.Errorf("file size is not set")
+				}
+
+				if file.ModTime.IsZero() {
+					t.Errorf("file mod time is not set")
+				}
+			}
+		})
+	}
+}
+
+func TestMemoryStorage_ListDirectories(t *testing.T) {
+	store := NewMemoryStorage()
+	testName := "list_directories"
+
+	fileNames := []string{
+		"/file0.json",
+		"/dir0/file2.json",
+		"/dir0/file3.json",
+		"/dir0/dir1/file4.json",
+		"/dir0/dir2/file5.json",
+	}
+
+	err := createFiles(fileNames, testName, store)
+	if err != nil {
+		t.Errorf("failed to create files: %s", err)
+	}
+
+	defer func() {
+		err = cleanupFiles(fileNames, testName, store)
+		if err != nil {
+			t.Errorf("failed to clean up files: %s", err)
+		}
+	}()
+
+	testCases := map[string]struct {
+		path      string
+		expected  []string
+		expectErr bool
+	}{
+		"base dir dir": {
+			path: path.Join(testpath, testName),
+			expected: []string{
+				path.Join(testpath, testName, "dir0") + "/",
+			},
+			expectErr: false,
+		},
+		"single nested dir files": {
+			path: path.Join(testpath, testName, "dir0"),
+			expected: []string{
+				path.Join(testpath, testName, "dir0", "dir1") + "/",
+				path.Join(testpath, testName, "dir0", "dir2") + "/",
+			},
+			expectErr: false,
+		},
+		"dir with no sub dirs": {
+			path:      path.Join(testpath, testName, "dir0/dir1"),
+			expected:  []string{},
+			expectErr: false,
+		},
+		"non-existent dir": {
+			path:      path.Join(testpath, testName, "dir1"),
+			expected:  []string{},
+			expectErr: false,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			dirList, err := store.ListDirectories(tc.path)
+			if tc.expectErr == (err == nil) {
+				if tc.expectErr {
+					t.Errorf("expected error was not thrown")
+					return
+				}
+				t.Errorf("unexpected error: %s", err.Error())
+				return
+			}
+
+			if len(dirList) != len(tc.expected) {
+				t.Errorf("dir list length does not match expected length, actual: %d, expected: %d", len(dirList), len(tc.expected))
+			}
+
+			expectedSet := map[string]struct{}{}
+			for _, expName := range tc.expected {
+				expectedSet[expName] = struct{}{}
+			}
+
+			for _, dir := range dirList {
+				_, ok := expectedSet[dir.Name]
+				if !ok {
+					t.Errorf("unexpect dir: %s in list %s", dir.Name, tc.path)
+				}
+			}
+		})
+	}
+}
+
+func TestMemoryStorage_Exists(t *testing.T) {
+	store := NewMemoryStorage()
+	testName := "exists"
+	fileNames := []string{
+		"/file0.json",
+	}
+
+	err := createFiles(fileNames, testName, store)
+	if err != nil {
+		t.Errorf("failed to create files: %s", err)
+	}
+
+	defer func() {
+		err = cleanupFiles(fileNames, testName, store)
+		if err != nil {
+			t.Errorf("failed to clean up files: %s", err)
+		}
+	}()
+
+	testCases := map[string]struct {
+		path      string
+		expected  bool
+		expectErr bool
+	}{
+		"file exists": {
+			path:      path.Join(testpath, testName, "file0.json"),
+			expected:  true,
+			expectErr: false,
+		},
+		"file does not exist": {
+			path:      path.Join(testpath, testName, "file1.json"),
+			expected:  false,
+			expectErr: false,
+		},
+		"dir does not exist": {
+			path:      path.Join(testpath, testName, "dir0/file.json"),
+			expected:  false,
+			expectErr: false,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			exists, err := store.Exists(tc.path)
+			if tc.expectErr == (err == nil) {
+				if tc.expectErr {
+					t.Errorf("expected error was not thrown")
+					return
+				}
+				t.Errorf("unexpected error: %s", err.Error())
+				return
+			}
+
+			if exists != tc.expected {
+				t.Errorf("file exists output did not match expected")
+			}
+		})
+	}
+}
+
+func TestMemoryStorage_Read(t *testing.T) {
+	store := NewMemoryStorage()
+	testName := "read"
+
+	fileNames := []string{
+		"/file0.json",
+	}
+
+	err := createFiles(fileNames, testName, store)
+	if err != nil {
+		t.Errorf("failed to create files: %s", err)
+	}
+
+	defer func() {
+		err = cleanupFiles(fileNames, testName, store)
+		if err != nil {
+			t.Errorf("failed to clean up files: %s", err)
+		}
+	}()
+
+	testCases := map[string]struct {
+		path      string
+		expectErr bool
+	}{
+		"file exists": {
+			path:      path.Join(testpath, testName, "file0.json"),
+			expectErr: false,
+		},
+		"file does not exist": {
+			path:      path.Join(testpath, testName, "file1.json"),
+			expectErr: true,
+		},
+		"dir does not exist": {
+			path:      path.Join(testpath, testName, "dir0/file.json"),
+			expectErr: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			b, err := store.Read(tc.path)
+			if tc.expectErr && err != nil {
+				return
+			}
+			if tc.expectErr == (err == nil) {
+				if tc.expectErr {
+					t.Errorf("expected error was not thrown")
+					return
+				}
+				t.Errorf("unexpected error: %s", err.Error())
+				return
+			}
+			var content testFileContent
+			err = json.Unmarshal(b, &content)
+			if err != nil {
+				t.Errorf("could not unmarshal file content")
+				return
+			}
+
+			if content != tfc {
+				t.Errorf("file content did not match writen value")
+			}
+		})
+	}
+}
+
+func TestMemoryStorage_Stat(t *testing.T) {
+	store := NewMemoryStorage()
+	testName := "stat"
+
+	fileNames := []string{
+		"/file0.json",
+	}
+
+	err := createFiles(fileNames, testName, store)
+	if err != nil {
+		t.Errorf("failed to create files: %s", err)
+	}
+
+	defer func() {
+		err = cleanupFiles(fileNames, testName, store)
+		if err != nil {
+			t.Errorf("failed to clean up files: %s", err)
+		}
+	}()
+
+	testCases := map[string]struct {
+		path      string
+		expected  *StorageInfo
+		expectErr bool
+	}{
+		"base dir": {
+			path: path.Join(testpath, testName, "file0.json"),
+			expected: &StorageInfo{
+				Name: "file0.json",
+				Size: 45,
+			},
+			expectErr: false,
+		},
+		"file does not exist": {
+			path:      path.Join(testpath, testName, "file1.json"),
+			expected:  nil,
+			expectErr: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			status, err := store.Stat(tc.path)
+			if tc.expectErr && err != nil {
+				return
+			}
+			if tc.expectErr == (err == nil) {
+				if tc.expectErr {
+					t.Errorf("expected error was not thrown")
+					return
+				}
+				t.Errorf("unexpected error: %s", err.Error())
+				return
+			}
+
+			if status.Name != tc.expected.Name {
+				t.Errorf("status name did name match expected, actual: %s, expected: %s", status.Name, tc.expected.Name)
+			}
+
+			if status.Size != tc.expected.Size {
+				t.Errorf("status name did size match expected, actual: %d, expected: %d", status.Size, tc.expected.Size)
+			}
+
+			if status.ModTime.IsZero() {
+				t.Errorf("status mod time is not set")
+			}
+
+		})
+	}
+}

core/pkg/storage/storagetypes.go

@@ -18,6 +18,7 @@ import (
 type StorageType string
 
 const (
+	StorageTypeMemory      StorageType = "memory"
 	StorageTypeFile        StorageType = "file"
 	StorageTypeBucketS3    StorageType = "bucket|s3"
 	StorageTypeBucketGCS   StorageType = "bucket|gcs"
@@ -55,6 +56,11 @@ func (st *StorageType) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
+// IsMemoryStorage returns true if the StorageType is a memory storage type.
+func (st StorageType) IsMemoryStorage() bool {
+	return st.BackendType() == "memory"
+}
+
 // IsFileStorage returns true if the StorageType is a file storage type.
 func (st StorageType) IsFileStorage() bool {
 	return st.BackendType() == "file"