export all available data, use compatible with other code interfaces

Signed-off-by: r2k1 <yokree@gmail.com>

pkg/costmodel/allocation.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/opencost/opencost/pkg/util/timeutil"
 
 	"github.com/opencost/opencost/pkg/env"
@@ -57,6 +59,8 @@ const (
 	queryFmtReplicaSetsWithoutOwners = `avg(avg_over_time(kube_replicaset_owner{owner_kind="<none>", owner_name="<none>"}[%s])) by (replicaset, namespace, %s)`
 	queryFmtLBCostPerHr              = `avg(avg_over_time(kubecost_load_balancer_cost[%s])) by (namespace, service_name, %s)`
 	queryFmtLBActiveMins             = `count(kubecost_load_balancer_cost) by (namespace, service_name, %s)[%s:%s]`
+	queryFmtOldestSample             = `max_over_time(timestamp(up{job="opencost"})[%s:%s])`
+	queryFmtNewestSample             = `min_over_time(timestamp(up{job="opencost"})[%s:%s])`
 )
 
 // Constants for Network Cost Subtype
@@ -84,6 +88,7 @@ func (cm *CostModel) Name() string {
 // for the window defined by the given start and end times. The Allocations
 // returned are unaggregated (i.e. down to the container level).
 func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Duration) (*kubecost.AllocationSet, error) {
+
 	// If the duration is short enough, compute the AllocationSet directly
 	if end.Sub(start) <= cm.MaxPrometheusQueryDuration {
 		return cm.computeAllocation(start, end, resolution)
@@ -252,6 +257,26 @@ func (cm *CostModel) ComputeAllocation(start, end time.Time, resolution time.Dur
 	return result, nil
 }
 
+// DateRange checks the data (up to 90 days in the past), and returns the oldest and newest sample timestamp from opencost scraping metric
+// it supposed to be a good indicator of available allocation data
+func (cm *CostModel) DateRange() (time.Time, time.Time, error) {
+	ctx := prom.NewNamedContext(cm.PrometheusClient, prom.AllocationContextName)
+
+	resOldest, _, err := ctx.QuerySync(fmt.Sprintf(queryFmtOldestSample, "90d", "1h"))
+	if err != nil {
+		return time.Time{}, time.Time{}, errors.Wrap(err, "querying oldest sample")
+	}
+	oldest := time.Unix(int64(resOldest[0].Values[0].Timestamp), 0)
+
+	resNewest, _, err := ctx.QuerySync(fmt.Sprintf(queryFmtNewestSample, "90d", "1h"))
+	if err != nil {
+		return time.Time{}, time.Time{}, errors.Wrap(err, "querying oldest sample")
+	}
+	newest := time.Unix(int64(resNewest[0].Values[0].Timestamp), 0)
+
+	return oldest, newest, nil
+}
+
 func (cm *CostModel) computeAllocation(start, end time.Time, resolution time.Duration) (*kubecost.AllocationSet, error) {
 	// 1. Build out Pod map from resolution-tuned, batched Pod start/end query
 	// 2. Run and apply the results of the remaining queries to

pkg/costmodel/allocation_csv.go

@@ -1,10 +1,11 @@
 package costmodel
 
 import (
+	"bytes"
 	"context"
 	"encoding/csv"
 	"io"
-	"os"
+	"sort"
 	"strconv"
 	"time"
 
@@ -15,16 +16,18 @@ import (
 )
 
 type CloudStorage interface {
-	FileReplace(ctx context.Context, f *os.File, path string) error
-	FileDownload(ctx context.Context, path string) (*os.File, error)
-	FileExists(ctx context.Context, path string) (bool, error)
+	Write(name string, data []byte) error
+	Read(name string) ([]byte, error)
+	Exists(name string) (bool, error)
 }
 
 type AllocationModel interface {
 	ComputeAllocation(start, end time.Time, resolution time.Duration) (*kubecost.AllocationSet, error)
-	DateRange(ctx context.Context) (time.Time, time.Time, error)
+	DateRange() (time.Time, time.Time, error)
 }
 
+var errNoData = errors.New("no data")
+
 // UpdateCSVWorker launches a worker that updates CSV file in cloud storage with allocation data
 // It updates data immediately on launch and then runs every day at 00:10 UTC
 // It expected to run a goroutine
@@ -36,8 +39,7 @@ func UpdateCSVWorker(ctx context.Context, storage CloudStorage, model Allocation
 		case <-ctx.Done():
 			return ctx.Err()
 		case <-time.After(nextRunAt.Sub(time.Now())):
-			dayBefore := time.Date(nextRunAt.Year(), nextRunAt.Month(), nextRunAt.Day(), 0, 0, 0, 0, time.UTC).AddDate(0, 0, -1)
-			err := UpdateCSV(ctx, storage, model, path, dayBefore)
+			err := UpdateCSV(ctx, storage, model, path)
 			if err != nil {
 				// it's background worker, log error and carry on, maybe next time it will work
 				log.Errorf("Error updating CSV: %s", err)
@@ -50,13 +52,13 @@ func UpdateCSVWorker(ctx context.Context, storage CloudStorage, model Allocation
 	}
 }
 
-func UpdateCSV(ctx context.Context, storage CloudStorage, model AllocationModel, path string, date time.Time) error {
+func UpdateCSV(ctx context.Context, storage CloudStorage, model AllocationModel, path string) error {
 	exporter := &csvExporter{
 		Storage:  storage,
 		Model:    model,
 		FilePath: path,
 	}
-	return exporter.Update(ctx, date)
+	return exporter.Update(ctx)
 }
 
 type csvExporter struct {
@@ -65,48 +67,59 @@ type csvExporter struct {
 	FilePath string
 }
 
-// TODO: logging
-func (e *csvExporter) Update(ctx context.Context, date time.Time) error {
-	exist, err := e.Storage.FileExists(ctx, e.FilePath)
+// Update updates CSV file in cloud storage with new allocation data
+func (e *csvExporter) Update(ctx context.Context) error {
+	allocationDates, err := e.availableAllocationDates()
 	if err != nil {
 		return err
 	}
 
-	dateExport, err := e.writeCSVToFile(ctx, date)
+	exist, err := e.Storage.Exists(e.FilePath)
 	if err != nil {
 		return err
 	}
-	defer dateExport.Close()
 
-	var result *os.File
+	var result []byte
+
+	// cloud storage doesn't have an existing file
+	// dump all the data exist to the file
+	if !exist {
+		result, err = e.allocationsToCSV(ctx, mapTimeToSlice(allocationDates))
+		if err != nil {
+			return err
+		}
+	}
+
+	// existing export file exists
+	// scan through it and ignore all dates that are already in the file
+	// avoid modifying existing data or producing duplicates
 	if exist {
-		// merge existing file with new data
-		previousExport, err := e.Storage.FileDownload(ctx, e.FilePath)
+		previousExport, err := e.Storage.Read(e.FilePath)
 		if err != nil {
 			return err
 		}
-		defer previousExport.Close()
 
-		result, err = os.CreateTemp("", "cost-model-*.csv")
+		csvDates, err := e.loadDates(previousExport)
 		if err != nil {
-			return errors.Wrap(err, "creating temp file")
+			return err
+		}
+
+		for date := range csvDates {
+			delete(allocationDates, date)
 		}
-		err = mergeCSV([]*os.File{previousExport, dateExport}, result)
+
+		dateExport, err := e.allocationsToCSV(ctx, mapTimeToSlice(allocationDates))
 		if err != nil {
 			return err
 		}
-	} else {
-		// no existing file, create a new one
-		result = dateExport
-	}
 
-	// we just finished writing to the file, so we need to seek to the beginning, so we can read from it
-	_, err = result.Seek(0, io.SeekStart)
-	if err != nil {
-		return errors.Wrap(err, "seeking to the beginning of the csv file")
+		result, err = mergeCSV([][]byte{previousExport, dateExport})
+		if err != nil {
+			return err
+		}
 	}
 
-	err = e.Storage.FileReplace(ctx, result, e.FilePath)
+	err = e.Storage.Write(e.FilePath, result)
 	if err != nil {
 		return err
 	}
@@ -114,30 +127,43 @@ func (e *csvExporter) Update(ctx context.Context, date time.Time) error {
 	return nil
 }
 
-func (e *csvExporter) writeCSVToFile(ctx context.Context, date time.Time) (*os.File, error) {
-	f, err := os.CreateTemp("", "cost-model-*.csv")
+func (e *csvExporter) availableAllocationDates() (map[time.Time]struct{}, error) {
+	start, end, err := e.Model.DateRange()
 	if err != nil {
-		return nil, errors.Wrap(err, "creating temp file")
+		return nil, err
+	}
+	if start != time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, time.UTC) {
+		// start doesn't start from 00:00 UTC, it could be truncated by prometheus retention policy
+		// skip incomplete data and begin from the day after, otherwise it may corrupt existing data
+		start = time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, time.UTC).AddDate(0, 0, 1)
 	}
+	end = time.Date(end.Year(), end.Month(), end.Day(), 0, 0, 0, 0, time.UTC)
+	dates := make(map[time.Time]struct{})
+	for date := start; date.Before(end); date = date.AddDate(0, 0, 1) {
+		dates[date] = struct{}{}
+	}
+	if len(dates) == 0 {
+		return nil, errors.New("no allocation data available")
+	}
+	return dates, nil
+}
 
-	err = e.writeCSVToWriter(ctx, f, date)
+func (e *csvExporter) allocationsToCSV(ctx context.Context, dates []time.Time) ([]byte, error) {
+	buf := new(bytes.Buffer)
+	err := e.writeCSVToWriter(ctx, buf, dates)
 	if err != nil {
 		return nil, err
 	}
-	return f, nil
+	return buf.Bytes(), nil
 }
 
-func (e *csvExporter) writeCSVToWriter(ctx context.Context, w io.Writer, date time.Time) error {
-	start := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.UTC)
-	end := start.AddDate(0, 1, 0)
-	data, err := e.Model.ComputeAllocation(start, end, 5*time.Minute)
-	if err != nil {
-		return err
+func (e *csvExporter) writeCSVToWriter(ctx context.Context, w io.Writer, dates []time.Time) error {
+	fmtFloat := func(f float64) string {
+		return strconv.FormatFloat(f, 'f', -1, 64)
 	}
-	log.Infof("data: %d", len(data.Allocations))
 	csvWriter := csv.NewWriter(w)
 	// TODO: confirm columns we want to export
-	err = csvWriter.Write([]string{
+	err := csvWriter.Write([]string{
 		"Date",
 		"Name",
 		"CPUCoreUsageAverage",
@@ -146,73 +172,126 @@ func (e *csvExporter) writeCSVToWriter(ctx context.Context, w io.Writer, date ti
 		"RAMBytesUsageAverage",
 		"RAMBytesRequestAverage",
 		"RAMCost",
+		"GPUs",
+		"GPUCost",
+		"NetworkCost",
+		"PVBytes",
+		"PVCost",
+		"TotalCost",
 	})
 	if err != nil {
 		return err
 	}
-	for _, alloc := range data.Allocations {
-		if err := ctx.Err(); err != nil {
+	lines := 0
+	for _, date := range dates {
+		start := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, time.UTC)
+		end := start.AddDate(0, 0, 1)
+		data, err := e.Model.ComputeAllocation(start, end, 5*time.Minute)
+		if err != nil {
 			return err
 		}
+		log.Infof("fetched %d records for %s", len(data.Allocations), date.Format("2006-01-02"))
+		for _, alloc := range data.Allocations {
+			if err := ctx.Err(); err != nil {
+				return err
+			}
 
-		err := csvWriter.Write([]string{
-			date.Format("2006-01-02"),
-			alloc.Name,
-			fmtFloat(alloc.CPUCoreUsageAverage),
-			fmtFloat(alloc.CPUCoreRequestAverage),
-			fmtFloat(alloc.CPUCost),
-			fmtFloat(alloc.RAMBytesUsageAverage),
-			fmtFloat(alloc.RAMBytesRequestAverage),
-			fmtFloat(alloc.RAMCost),
-		})
-		if err != nil {
-			return err
+			err := csvWriter.Write([]string{
+				date.Format("2006-01-02"),
+				alloc.Name,
+				fmtFloat(alloc.CPUCoreUsageAverage),
+				fmtFloat(alloc.CPUCoreRequestAverage),
+				fmtFloat(alloc.CPUTotalCost()),
+				fmtFloat(alloc.RAMBytesUsageAverage),
+				fmtFloat(alloc.RAMBytesRequestAverage),
+				fmtFloat(alloc.RAMTotalCost()),
+				fmtFloat(alloc.GPUs()),
+				fmtFloat(alloc.GPUCost),
+				fmtFloat(alloc.NetworkTotalCost()),
+				fmtFloat(alloc.PVBytes()),
+				fmtFloat(alloc.PVCost()),
+				fmtFloat(alloc.TotalCost()),
+			})
+			if err != nil {
+				return err
+			}
+			lines++
 		}
 	}
+
+	if lines == 0 {
+		return errNoData
+	}
+
 	csvWriter.Flush()
 	if err := csvWriter.Error(); err != nil {
 		return err
 	}
-
+	log.Infof("exported %d lines", lines)
 	return nil
 }
 
-func fmtFloat(f float64) string {
-	return strconv.FormatFloat(f, 'f', -1, 64)
+// loadDate scans through CSV export file and extract all dates from "Date" column
+func (e *csvExporter) loadDates(csvFile []byte) (map[time.Time]struct{}, error) {
+	csvReader := csv.NewReader(bytes.NewReader(csvFile))
+	header, err := csvReader.Read()
+	if err != nil {
+		return nil, errors.Wrap(err, "reading csv header")
+	}
+	dateColIndex := 0
+	for i, col := range header {
+		if col == "Date" {
+			dateColIndex = i
+			break
+		}
+	}
+	dates := make(map[time.Time]struct{})
+	for {
+		row, err := csvReader.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, errors.Wrap(err, "reading csv row")
+		}
+		date, err := time.Parse("2006-01-02", row[dateColIndex])
+		if err != nil {
+			return nil, errors.Wrap(err, "parsing date")
+		}
+		dates[date] = struct{}{}
+	}
+	return dates, nil
 }
 
 // mergeCSV merges multiple csv files into one.
 // Files may have different headers, but the result will have a header that is a union of all headers.
 // The main goal here is to allow changing CSV format without breaking or loosing existing data.
-func mergeCSV(files []*os.File, output *os.File) error {
+func mergeCSV(files [][]byte) ([]byte, error) {
 	var err error
 	headers := make([][]string, 0, len(files))
 	csvReaders := make([]*csv.Reader, 0, len(files))
 
 	// first, get information about the result header
 	for _, file := range files {
-		if _, err := file.Seek(0, io.SeekStart); err != nil {
-			return errors.Wrapf(err, "seeking to start of %s", file.Name())
-		}
-		csvReader := csv.NewReader(file)
+		csvReader := csv.NewReader(bytes.NewReader(file))
 		header, err := csvReader.Read()
 		if errors.Is(err, io.EOF) {
 			// ignore empty files
 			continue
 		}
 		if err != nil {
-			return errors.Wrapf(err, "reading header of %s", file.Name())
+			return nil, errors.Wrap(err, "reading header of csv file")
 		}
 		headers = append(headers, header)
 		csvReaders = append(csvReaders, csvReader)
 	}
 
 	mapping, header := combineHeaders(headers)
-
+	output := new(bytes.Buffer)
 	csvWriter := csv.NewWriter(output)
 	err = csvWriter.Write(mergeHeaders(headers))
 	if err != nil {
-		return errors.Wrapf(err, "writing header to %s", output.Name())
+		return nil, errors.Wrap(err, "writing header to csv file")
 	}
 
 	for csvIndex, csvReader := range csvReaders {
@@ -222,7 +301,7 @@ func mergeCSV(files []*os.File, output *os.File) error {
 				break
 			}
 			if err != nil {
-				return errors.Wrap(err, "reading csv file line")
+				return nil, errors.Wrap(err, "reading csv file line")
 			}
 
 			outputLine := make([]string, len(header))
@@ -235,18 +314,17 @@ func mergeCSV(files []*os.File, output *os.File) error {
 			}
 			err = csvWriter.Write(outputLine)
 			if err != nil {
-				return errors.Wrapf(err, "writing line to %s", output.Name())
+				return nil, errors.Wrap(err, "writing line to csv file")
 			}
 		}
 
 	}
 	csvWriter.Flush()
-	_, err = output.Seek(0, io.SeekStart)
-	if err != nil {
-		return errors.Wrapf(err, "seeking to start of %s", output.Name())
+	if csvWriter.Error() != nil {
+		return nil, errors.Wrapf(csvWriter.Error(), "flushing csv file")
 	}
 
-	return nil
+	return output.Bytes(), nil
 }
 
 func combineHeaders(headers [][]string) ([]map[int]int, []string) {
@@ -295,3 +373,14 @@ func indexOf(slice []string, element string) int {
 	}
 	return -1
 }
+
+func mapTimeToSlice(data map[time.Time]struct{}) []time.Time {
+	result := make([]time.Time, 0, len(data))
+	for key := range data {
+		result = append(result, key)
+	}
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Before(result[j])
+	})
+	return result
+}

pkg/costmodel/allocation_csv_test.go

@@ -2,11 +2,10 @@ package costmodel
 
 import (
 	"context"
-	"io"
-	"os"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/opencost/opencost/pkg/kubecost"
@@ -17,23 +16,24 @@ import (
 
 func Test_UpdateCSV(t *testing.T) {
 	t.Run("previous data doesn't exist, upload new data", func(t *testing.T) {
-		var csv string
 		storage := &CloudStorageMock{
-			FileExistsFunc: func(ctx context.Context, path string) (bool, error) {
+			ExistsFunc: func(path string) (bool, error) {
 				return false, nil
 			},
-			FileReplaceFunc: func(ctx context.Context, f *os.File, path string) error {
-				data, err := io.ReadAll(f)
-				csv = string(data)
-				require.NoError(t, err)
+			WriteFunc: func(name string, data []byte) error {
 				return nil
 			},
 		}
 		model := &AllocationModelMock{
+			DateRangeFunc: func() (time.Time, time.Time, error) {
+				return time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC), nil
+			},
 			ComputeAllocationFunc: func(start time.Time, end time.Time, resolution time.Duration) (*kubecost.AllocationSet, error) {
 				return &kubecost.AllocationSet{
 					Allocations: map[string]*kubecost.Allocation{
 						"test": {
+							Start:                  time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), // required for GPU metrics
+							End:                    time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC),
 							Name:                   "test",
 							CPUCoreUsageAverage:    0.1,
 							CPUCoreRequestAverage:  0.2,
@@ -41,67 +41,96 @@ func Test_UpdateCSV(t *testing.T) {
 							RAMBytesUsageAverage:   0.4,
 							RAMBytesRequestAverage: 0.5,
 							RAMCost:                0.6,
+							GPUHours:               48,
+							GPUCost:                0.8,
+							NetworkCost:            0.9,
+							PVs: map[kubecost.PVKey]*kubecost.PVAllocation{
+								kubecost.PVKey{
+									Cluster: "test-cluster",
+									Name:    "test-pv",
+								}: {
+									ByteHours: 48,
+									Cost:      2.0,
+								},
+							}, // 2 PVBytes, 2 PVCost
 						},
 					},
 				}, nil
 			},
 		}
-		err := UpdateCSV(context.TODO(), storage, model, "/test.csv", time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
+		err := UpdateCSV(context.TODO(), storage, model, "/test.csv")
 		require.NoError(t, err)
 		// uploaded a single file with the data
-		require.Len(t, storage.FileReplaceCalls(), 1)
-		require.Equal(t, `Date,Name,CPUCoreUsageAverage,CPUCoreRequestAverage,CPUCost,RAMBytesUsageAverage,RAMBytesRequestAverage,RAMCost
-2021-01-01,test,0.1,0.2,0.3,0.4,0.5,0.6
-`, csv)
+		assert.Len(t, storage.WriteCalls(), 1)
+		assert.Len(t, model.ComputeAllocationCalls(), 1)
+		assert.Equal(t, time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), model.ComputeAllocationCalls()[0].Start)
+		assert.Equal(t, time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC), model.ComputeAllocationCalls()[0].End)
+		assert.Equal(t, `Date,Name,CPUCoreUsageAverage,CPUCoreRequestAverage,CPUCost,RAMBytesUsageAverage,RAMBytesRequestAverage,RAMCost,GPUs,GPUCost,NetworkCost,PVBytes,PVCost,TotalCost
+2021-01-01,test,0.1,0.2,0.3,0.4,0.5,0.6,2,0.8,0.9,2,2,4.6000000000000005
+`, string(storage.WriteCalls()[0].Data))
 	})
 
-	t.Run("merge new data with previous data", func(t *testing.T) {
-		var csv string
+	t.Run("merge new data with previous data (with different CSV structure)", func(t *testing.T) {
 		storage := &CloudStorageMock{
-			FileExistsFunc: func(ctx context.Context, path string) (bool, error) {
+			ExistsFunc: func(name string) (bool, error) {
 				return true, nil
 			},
-			FileDownloadFunc: func(ctx context.Context, path string) (*os.File, error) {
-				f, err := os.CreateTemp("", "")
-				require.NoError(t, err)
-				_, err = f.WriteString(`Date,Name,CPUCoreUsageAverage,CPUCoreRequestAverage,CPUCost,RAMBytesUsageAverage,RAMBytesRequestAverage,RAMCost
+			ReadFunc: func(name string) ([]byte, error) {
+				return []byte(`Date,Name,CPUCoreUsageAverage,CPUCoreRequestAverage,CPUCost,RAMBytesUsageAverage,RAMBytesRequestAverage,RAMCost
 2021-01-01,test,0.1,0.2,0.3,0.4,0.5,0.6
-`)
-				_, err = f.Seek(0, io.SeekStart)
-				require.NoError(t, err)
-				return f, err
+`), nil
 			},
-			FileReplaceFunc: func(ctx context.Context, f *os.File, path string) error {
-				data, err := io.ReadAll(f)
-				csv = string(data)
-				require.NoError(t, err)
+			WriteFunc: func(name string, data []byte) error {
 				return nil
 			},
 		}
 		model := &AllocationModelMock{
+			DateRangeFunc: func() (time.Time, time.Time, error) {
+				return time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2021, 1, 3, 0, 0, 0, 0, time.UTC), nil
+			},
 			ComputeAllocationFunc: func(start time.Time, end time.Time, resolution time.Duration) (*kubecost.AllocationSet, error) {
 				return &kubecost.AllocationSet{
 					Allocations: map[string]*kubecost.Allocation{
 						"test": {
-							Name:                   "test",
-							CPUCoreUsageAverage:    1,
-							CPUCoreRequestAverage:  2,
-							CPUCost:                3,
-							RAMBytesUsageAverage:   4,
-							RAMBytesRequestAverage: 5,
-							RAMCost:                6,
+							Name:    "test",
+							CPUCost: 1,
 						},
 					},
 				}, nil
 			},
 		}
-		err := UpdateCSV(context.TODO(), storage, model, "/test.csv", time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC))
+		err := UpdateCSV(context.TODO(), storage, model, "/test.csv")
 		require.NoError(t, err)
 		// uploaded a single file with the data
-		require.Len(t, storage.FileReplaceCalls(), 1)
-		require.Equal(t, `Date,Name,CPUCoreUsageAverage,CPUCoreRequestAverage,CPUCost,RAMBytesUsageAverage,RAMBytesRequestAverage,RAMCost
-2021-01-01,test,0.1,0.2,0.3,0.4,0.5,0.6
-2021-01-02,test,1,2,3,4,5,6
-`, csv)
+		assert.Len(t, storage.WriteCalls(), 1)
+		assert.Len(t, model.ComputeAllocationCalls(), 1)
+		assert.Len(t, model.ComputeAllocationCalls(), 1)
+		// 2021-01-01 is already in the export file, so we only compute for 2021-01-02
+		assert.Equal(t, time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC), model.ComputeAllocationCalls()[0].Start)
+		assert.Equal(t, time.Date(2021, 1, 3, 0, 0, 0, 0, time.UTC), model.ComputeAllocationCalls()[0].End)
+		assert.Equal(t, `Date,Name,CPUCoreUsageAverage,CPUCoreRequestAverage,CPUCost,RAMBytesUsageAverage,RAMBytesRequestAverage,RAMCost,GPUs,GPUCost,NetworkCost,PVBytes,PVCost,TotalCost
+2021-01-01,test,0.1,0.2,0.3,0.4,0.5,0.6,,,,,,
+2021-01-02,test,0,0,1,0,0,0,0,0,0,0,0,1
+`, string(storage.WriteCalls()[0].Data))
+	})
+
+	t.Run("allocation data is empty", func(t *testing.T) {
+		model := &AllocationModelMock{
+			ComputeAllocationFunc: func(start time.Time, end time.Time, resolution time.Duration) (*kubecost.AllocationSet, error) {
+				return &kubecost.AllocationSet{
+					Allocations: nil,
+				}, nil
+			},
+			DateRangeFunc: func() (time.Time, time.Time, error) {
+				return time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2021, 1, 3, 0, 0, 0, 0, time.UTC), nil
+			},
+		}
+		storage := &CloudStorageMock{
+			ExistsFunc: func(name string) (bool, error) {
+				return false, nil
+			},
+		}
+		err := UpdateCSV(context.TODO(), storage, model, "/test.csv")
+		require.Equal(t, err, errNoData)
 	})
 }

pkg/costmodel/moq_allocation_model_test.go

@@ -4,7 +4,6 @@
 package costmodel
 
 import (
-	"context"
 	"github.com/opencost/opencost/pkg/kubecost"
 	"sync"
 	"time"
@@ -23,7 +22,7 @@ var _ AllocationModel = &AllocationModelMock{}
 //			ComputeAllocationFunc: func(start time.Time, end time.Time, resolution time.Duration) (*kubecost.AllocationSet, error) {
 //				panic("mock out the ComputeAllocation method")
 //			},
-//			DateRangeFunc: func(ctx context.Context) (time.Time, time.Time, error) {
+//			DateRangeFunc: func() (time.Time, time.Time, error) {
 //				panic("mock out the DateRange method")
 //			},
 //		}
@@ -37,7 +36,7 @@ type AllocationModelMock struct {
 	ComputeAllocationFunc func(start time.Time, end time.Time, resolution time.Duration) (*kubecost.AllocationSet, error)
 
 	// DateRangeFunc mocks the DateRange method.
-	DateRangeFunc func(ctx context.Context) (time.Time, time.Time, error)
+	DateRangeFunc func() (time.Time, time.Time, error)
 
 	// calls tracks calls to the methods.
 	calls struct {
@@ -52,8 +51,6 @@ type AllocationModelMock struct {
 		}
 		// DateRange holds details about calls to the DateRange method.
 		DateRange []struct {
-			// Ctx is the ctx argument value.
-			Ctx context.Context
 		}
 	}
 	lockComputeAllocation sync.RWMutex
@@ -101,19 +98,16 @@ func (mock *AllocationModelMock) ComputeAllocationCalls() []struct {
 }
 
 // DateRange calls DateRangeFunc.
-func (mock *AllocationModelMock) DateRange(ctx context.Context) (time.Time, time.Time, error) {
+func (mock *AllocationModelMock) DateRange() (time.Time, time.Time, error) {
 	if mock.DateRangeFunc == nil {
 		panic("AllocationModelMock.DateRangeFunc: method is nil but AllocationModel.DateRange was just called")
 	}
 	callInfo := struct {
-		Ctx context.Context
-	}{
-		Ctx: ctx,
-	}
+	}{}
 	mock.lockDateRange.Lock()
 	mock.calls.DateRange = append(mock.calls.DateRange, callInfo)
 	mock.lockDateRange.Unlock()
-	return mock.DateRangeFunc(ctx)
+	return mock.DateRangeFunc()
 }
 
 // DateRangeCalls gets all the calls that were made to DateRange.
@@ -121,10 +115,8 @@ func (mock *AllocationModelMock) DateRange(ctx context.Context) (time.Time, time
 //
 //	len(mockedAllocationModel.DateRangeCalls())
 func (mock *AllocationModelMock) DateRangeCalls() []struct {
-	Ctx context.Context
 } {
 	var calls []struct {
-		Ctx context.Context
 	}
 	mock.lockDateRange.RLock()
 	calls = mock.calls.DateRange

pkg/costmodel/moq_cloud_storage_test.go

@@ -4,8 +4,6 @@
 package costmodel
 
 import (
-	"context"
-	"os"
 	"sync"
 )
 
@@ -19,14 +17,14 @@ var _ CloudStorage = &CloudStorageMock{}
 //
 //		// make and configure a mocked CloudStorage
 //		mockedCloudStorage := &CloudStorageMock{
-//			FileDownloadFunc: func(ctx context.Context, path string) (*os.File, error) {
-//				panic("mock out the FileDownload method")
+//			ExistsFunc: func(name string) (bool, error) {
+//				panic("mock out the Exists method")
 //			},
-//			FileExistsFunc: func(ctx context.Context, path string) (bool, error) {
-//				panic("mock out the FileExists method")
+//			ReadFunc: func(name string) ([]byte, error) {
+//				panic("mock out the Read method")
 //			},
-//			FileReplaceFunc: func(ctx context.Context, f *os.File, path string) error {
-//				panic("mock out the FileReplace method")
+//			WriteFunc: func(name string, data []byte) error {
+//				panic("mock out the Write method")
 //			},
 //		}
 //
@@ -35,154 +33,136 @@ var _ CloudStorage = &CloudStorageMock{}
 //
 //	}
 type CloudStorageMock struct {
-	// FileDownloadFunc mocks the FileDownload method.
-	FileDownloadFunc func(ctx context.Context, path string) (*os.File, error)
+	// ExistsFunc mocks the Exists method.
+	ExistsFunc func(name string) (bool, error)
 
-	// FileExistsFunc mocks the FileExists method.
-	FileExistsFunc func(ctx context.Context, path string) (bool, error)
+	// ReadFunc mocks the Read method.
+	ReadFunc func(name string) ([]byte, error)
 
-	// FileReplaceFunc mocks the FileReplace method.
-	FileReplaceFunc func(ctx context.Context, f *os.File, path string) error
+	// WriteFunc mocks the Write method.
+	WriteFunc func(name string, data []byte) error
 
 	// calls tracks calls to the methods.
 	calls struct {
-		// FileDownload holds details about calls to the FileDownload method.
-		FileDownload []struct {
-			// Ctx is the ctx argument value.
-			Ctx context.Context
-			// Path is the path argument value.
-			Path string
+		// Exists holds details about calls to the Exists method.
+		Exists []struct {
+			// Name is the name argument value.
+			Name string
 		}
-		// FileExists holds details about calls to the FileExists method.
-		FileExists []struct {
-			// Ctx is the ctx argument value.
-			Ctx context.Context
-			// Path is the path argument value.
-			Path string
+		// Read holds details about calls to the Read method.
+		Read []struct {
+			// Name is the name argument value.
+			Name string
 		}
-		// FileReplace holds details about calls to the FileReplace method.
-		FileReplace []struct {
-			// Ctx is the ctx argument value.
-			Ctx context.Context
-			// F is the f argument value.
-			F *os.File
-			// Path is the path argument value.
-			Path string
+		// Write holds details about calls to the Write method.
+		Write []struct {
+			// Name is the name argument value.
+			Name string
+			// Data is the data argument value.
+			Data []byte
 		}
 	}
-	lockFileDownload sync.RWMutex
-	lockFileExists   sync.RWMutex
-	lockFileReplace  sync.RWMutex
+	lockExists sync.RWMutex
+	lockRead   sync.RWMutex
+	lockWrite  sync.RWMutex
 }
 
-// FileDownload calls FileDownloadFunc.
-func (mock *CloudStorageMock) FileDownload(ctx context.Context, path string) (*os.File, error) {
-	if mock.FileDownloadFunc == nil {
-		panic("CloudStorageMock.FileDownloadFunc: method is nil but CloudStorage.FileDownload was just called")
+// Exists calls ExistsFunc.
+func (mock *CloudStorageMock) Exists(name string) (bool, error) {
+	if mock.ExistsFunc == nil {
+		panic("CloudStorageMock.ExistsFunc: method is nil but CloudStorage.Exists was just called")
 	}
 	callInfo := struct {
-		Ctx  context.Context
-		Path string
+		Name string
 	}{
-		Ctx:  ctx,
-		Path: path,
+		Name: name,
 	}
-	mock.lockFileDownload.Lock()
-	mock.calls.FileDownload = append(mock.calls.FileDownload, callInfo)
-	mock.lockFileDownload.Unlock()
-	return mock.FileDownloadFunc(ctx, path)
+	mock.lockExists.Lock()
+	mock.calls.Exists = append(mock.calls.Exists, callInfo)
+	mock.lockExists.Unlock()
+	return mock.ExistsFunc(name)
 }
 
-// FileDownloadCalls gets all the calls that were made to FileDownload.
+// ExistsCalls gets all the calls that were made to Exists.
 // Check the length with:
 //
-//	len(mockedCloudStorage.FileDownloadCalls())
-func (mock *CloudStorageMock) FileDownloadCalls() []struct {
-	Ctx  context.Context
-	Path string
+//	len(mockedCloudStorage.ExistsCalls())
+func (mock *CloudStorageMock) ExistsCalls() []struct {
+	Name string
 } {
 	var calls []struct {
-		Ctx  context.Context
-		Path string
+		Name string
 	}
-	mock.lockFileDownload.RLock()
-	calls = mock.calls.FileDownload
-	mock.lockFileDownload.RUnlock()
+	mock.lockExists.RLock()
+	calls = mock.calls.Exists
+	mock.lockExists.RUnlock()
 	return calls
 }
 
-// FileExists calls FileExistsFunc.
-func (mock *CloudStorageMock) FileExists(ctx context.Context, path string) (bool, error) {
-	if mock.FileExistsFunc == nil {
-		panic("CloudStorageMock.FileExistsFunc: method is nil but CloudStorage.FileExists was just called")
+// Read calls ReadFunc.
+func (mock *CloudStorageMock) Read(name string) ([]byte, error) {
+	if mock.ReadFunc == nil {
+		panic("CloudStorageMock.ReadFunc: method is nil but CloudStorage.Read was just called")
 	}
 	callInfo := struct {
-		Ctx  context.Context
-		Path string
+		Name string
 	}{
-		Ctx:  ctx,
-		Path: path,
+		Name: name,
 	}
-	mock.lockFileExists.Lock()
-	mock.calls.FileExists = append(mock.calls.FileExists, callInfo)
-	mock.lockFileExists.Unlock()
-	return mock.FileExistsFunc(ctx, path)
+	mock.lockRead.Lock()
+	mock.calls.Read = append(mock.calls.Read, callInfo)
+	mock.lockRead.Unlock()
+	return mock.ReadFunc(name)
 }
 
-// FileExistsCalls gets all the calls that were made to FileExists.
+// ReadCalls gets all the calls that were made to Read.
 // Check the length with:
 //
-//	len(mockedCloudStorage.FileExistsCalls())
-func (mock *CloudStorageMock) FileExistsCalls() []struct {
-	Ctx  context.Context
-	Path string
+//	len(mockedCloudStorage.ReadCalls())
+func (mock *CloudStorageMock) ReadCalls() []struct {
+	Name string
 } {
 	var calls []struct {
-		Ctx  context.Context
-		Path string
+		Name string
 	}
-	mock.lockFileExists.RLock()
-	calls = mock.calls.FileExists
-	mock.lockFileExists.RUnlock()
+	mock.lockRead.RLock()
+	calls = mock.calls.Read
+	mock.lockRead.RUnlock()
 	return calls
 }
 
-// FileReplace calls FileReplaceFunc.
-func (mock *CloudStorageMock) FileReplace(ctx context.Context, f *os.File, path string) error {
-	if mock.FileReplaceFunc == nil {
-		panic("CloudStorageMock.FileReplaceFunc: method is nil but CloudStorage.FileReplace was just called")
+// Write calls WriteFunc.
+func (mock *CloudStorageMock) Write(name string, data []byte) error {
+	if mock.WriteFunc == nil {
+		panic("CloudStorageMock.WriteFunc: method is nil but CloudStorage.Write was just called")
 	}
 	callInfo := struct {
-		Ctx  context.Context
-		F    *os.File
-		Path string
+		Name string
+		Data []byte
 	}{
-		Ctx:  ctx,
-		F:    f,
-		Path: path,
+		Name: name,
+		Data: data,
 	}
-	mock.lockFileReplace.Lock()
-	mock.calls.FileReplace = append(mock.calls.FileReplace, callInfo)
-	mock.lockFileReplace.Unlock()
-	return mock.FileReplaceFunc(ctx, f, path)
+	mock.lockWrite.Lock()
+	mock.calls.Write = append(mock.calls.Write, callInfo)
+	mock.lockWrite.Unlock()
+	return mock.WriteFunc(name, data)
 }
 
-// FileReplaceCalls gets all the calls that were made to FileReplace.
+// WriteCalls gets all the calls that were made to Write.
 // Check the length with:
 //
-//	len(mockedCloudStorage.FileReplaceCalls())
-func (mock *CloudStorageMock) FileReplaceCalls() []struct {
-	Ctx  context.Context
-	F    *os.File
-	Path string
+//	len(mockedCloudStorage.WriteCalls())
+func (mock *CloudStorageMock) WriteCalls() []struct {
+	Name string
+	Data []byte
 } {
 	var calls []struct {
-		Ctx  context.Context
-		F    *os.File
-		Path string
+		Name string
+		Data []byte
 	}
-	mock.lockFileReplace.RLock()
-	calls = mock.calls.FileReplace
-	mock.lockFileReplace.RUnlock()
+	mock.lockWrite.RLock()
+	calls = mock.calls.Write
+	mock.lockWrite.RUnlock()
 	return calls
 }