rename storagev2 => filemanager

Signed-off-by: r2k1 <yokree@gmail.com>

pkg/cmd/costmodel/costmodel.go

@@ -13,9 +13,9 @@ import (
 	"github.com/opencost/opencost/pkg/costmodel"
 	"github.com/opencost/opencost/pkg/env"
 	"github.com/opencost/opencost/pkg/errors"
+	"github.com/opencost/opencost/pkg/filemanager"
 	"github.com/opencost/opencost/pkg/log"
 	"github.com/opencost/opencost/pkg/metrics"
-	"github.com/opencost/opencost/pkg/storagev2"
 	"github.com/opencost/opencost/pkg/version"
 )
 
@@ -55,7 +55,7 @@ func StartExportWorker(ctx context.Context, model costmodel.AllocationModel) {
 		return
 	}
 
-	fm, err := storagev2.NewFileManager(exportPath)
+	fm, err := filemanager.NewFileManager(exportPath)
 	if err != nil {
 		log.Errorf("could not start CSV exporter", err)
 		return

pkg/costmodel/csv_export.go

@@ -11,9 +11,9 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/opencost/opencost/pkg/filemanager"
 	"github.com/opencost/opencost/pkg/kubecost"
 	"github.com/opencost/opencost/pkg/log"
-	"github.com/opencost/opencost/pkg/storagev2"
 )
 
 type AllocationModel interface {
@@ -23,7 +23,7 @@ type AllocationModel interface {
 
 var errNoData = errors.New("no data")
 
-func UpdateCSV(ctx context.Context, fileManager storagev2.FileManager, model AllocationModel) error {
+func UpdateCSV(ctx context.Context, fileManager filemanager.FileManager, model AllocationModel) error {
 	exporter := &csvExporter{
 		FileManager: fileManager,
 		Model:       model,
@@ -32,7 +32,7 @@ func UpdateCSV(ctx context.Context, fileManager storagev2.FileManager, model All
 }
 
 type csvExporter struct {
-	FileManager storagev2.FileManager
+	FileManager filemanager.FileManager
 	Model       AllocationModel
 }
 
@@ -47,13 +47,13 @@ func (e *csvExporter) Update(ctx context.Context) error {
 		return errors.New("no data to export from prometheus")
 	}
 
-	resultTmp, err := os.CreateTemp("opencost", "export-*.csv")
+	resultTmp, err := os.CreateTemp("", "opencost-export-*.csv")
 	if err != nil {
 		return err
 	}
 	defer closeAndDelete(resultTmp)
 
-	previousExportTmp, err := os.CreateTemp("opencost", "previous-export-*.csv")
+	previousExportTmp, err := os.CreateTemp("", "opencost-previous-export-*.csv")
 	if err != nil {
 		return err
 	}
@@ -61,7 +61,7 @@ func (e *csvExporter) Update(ctx context.Context) error {
 
 	err = e.FileManager.Download(ctx, previousExportTmp)
 	switch {
-	case errors.Is(err, storagev2.ErrNotFound):
+	case errors.Is(err, filemanager.ErrNotFound):
 		// there is no previous file, so we need to create it
 		err := e.writeCSVToWriter(ctx, resultTmp, mapTimeToSlice(allocationDates))
 		if err != nil {
@@ -110,7 +110,7 @@ func (e *csvExporter) updateExportCSV(ctx context.Context, previousExportTmp *os
 		return nil
 	}
 
-	newExportTmp, err := os.CreateTemp("opencost", "new-export-*.csv")
+	newExportTmp, err := os.CreateTemp("", "opencost-new-export-*.csv")
 	if err != nil {
 		return err
 	}

pkg/costmodel/csv_export_test.go

@@ -8,8 +8,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/opencost/opencost/pkg/filemanager"
 	"github.com/opencost/opencost/pkg/kubecost"
-	"github.com/opencost/opencost/pkg/storagev2"
 )
 
 //go:generate moq -out moq_cloud_storage_test.go . CloudStorage:CloudStorageMock
@@ -17,7 +17,7 @@ import (
 
 func Test_UpdateCSV(t *testing.T) {
 	t.Run("previous data doesn't exist, upload new data", func(t *testing.T) {
-		storage := &storagev2.InMemoryFile{}
+		storage := &filemanager.InMemoryFile{}
 		model := &AllocationModelMock{
 			DateRangeFunc: func() (time.Time, time.Time, error) {
 				return time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC), nil
@@ -72,7 +72,7 @@ func Test_UpdateCSV(t *testing.T) {
 	})
 
 	t.Run("merge new data with previous data (with different CSV structure)", func(t *testing.T) {
-		storage := &storagev2.InMemoryFile{
+		storage := &filemanager.InMemoryFile{
 			Data: []byte(`Date,Namespace,CPUCoreUsageAverage,CPUCoreRequestAverage,CPUCost,RAMBytesUsageAverage,RAMBytesRequestAverage,RAMCost
 2021-01-01,test-namespace,0.1,0.2,0.3,0.4,0.5,0.6
 `),
@@ -112,7 +112,7 @@ func Test_UpdateCSV(t *testing.T) {
 		data := `Date,Name,CPUCoreUsageAverage,CPUCoreRequestAverage,CPUCost,RAMBytesUsageAverage,RAMBytesRequestAverage,RAMCost
 2021-01-01,test,0.1,0.2,0.3,0.4,0.5,0.6
 `
-		storage := &storagev2.InMemoryFile{
+		storage := &filemanager.InMemoryFile{
 			Data: []byte(data),
 		}
 		model := &AllocationModelMock{
@@ -137,7 +137,7 @@ func Test_UpdateCSV(t *testing.T) {
 				return time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2021, 1, 3, 0, 0, 0, 0, time.UTC), nil
 			},
 		}
-		storage := &storagev2.InMemoryFile{}
+		storage := &filemanager.InMemoryFile{}
 		err := UpdateCSV(context.TODO(), storage, model)
 		require.Equal(t, err, errNoData)
 	})

pkg/storagev2/file.go → pkg/filemanager/filemanager.go

@@ -1,11 +1,10 @@
-package storagev2
+package filemanager
 
 import (
 	"context"
 	"errors"
 	"fmt"
 	"io"
-	"io/fs"
 	"net/url"
 	"os"
 	"strings"
@@ -82,8 +81,6 @@ type S3File struct {
 	key      string
 }
 
-// NewS3File creates a new S3File from a path.
-// Example of path: s3://bucket-name/path/to/file.csv
 func NewS3File(path string) (*S3File, error) {
 	u, err := url.Parse(path)
 	if err != nil {
@@ -185,9 +182,9 @@ func (g *GCSStorageFile) Upload(ctx context.Context, f *os.File) error {
 
 func NewSystemFile(path string) (*SystemFile, error) {
 	// validate path
-	if !fs.ValidPath(path) {
-		return nil, fmt.Errorf("invalid path: %s", path)
-	}
+	//if !fs.ValidPath(path) {
+	//	return nil, fmt.Errorf("invalid path: %s", path)
+	//}
 	return &SystemFile{path: path}, nil
 }
 
@@ -209,7 +206,14 @@ func (s *SystemFile) Download(ctx context.Context, f *os.File) error {
 }
 
 func (s *SystemFile) Upload(ctx context.Context, f *os.File) error {
-	tmpF, err := os.CreateTemp("opencost", "upload-*")
+	_, err := f.Seek(0, io.SeekStart)
+	if err != nil {
+		return err
+	}
+	tmpF, err := os.CreateTemp("", "opencost-upload-*")
+	if err != nil {
+		return err
+	}
 	defer os.Remove(tmpF.Name())
 	defer tmpF.Close()
 	_, err = io.Copy(tmpF, f)

pkg/filemanager/filemanager_test.go

@@ -0,0 +1,42 @@
+package filemanager
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_NewFileManager(t *testing.T) {
+	t.Run("file system", func(t *testing.T) {
+		tmpPath := filepath.Join(os.TempDir(), fmt.Sprintf("opencost-test-file-manager-%d", rand.Int31()))
+		fm, err := NewFileManager(tmpPath)
+		require.NoError(t, err)
+
+		downloadFile, err := os.CreateTemp("", "opencost-test-file-manager-*")
+		require.NoError(t, err)
+		err = fm.Download(context.TODO(), downloadFile)
+		require.ErrorIs(t, err, ErrNotFound)
+
+		uploadFile, err := os.CreateTemp("", "opencost-test-file-manager-*")
+		require.NoError(t, err)
+		_, err = uploadFile.WriteString("test-content")
+		require.NoError(t, err)
+		require.NoError(t, err)
+		err = fm.Upload(context.TODO(), uploadFile)
+		require.NoError(t, err)
+
+		err = fm.Download(context.TODO(), downloadFile)
+		require.NoError(t, err)
+		_, err = downloadFile.Seek(0, io.SeekStart)
+		require.NoError(t, err)
+		data, err := io.ReadAll(downloadFile)
+		require.NoError(t, err)
+		require.Equal(t, "test-content", string(data))
+	})
+}