cost-model/core/pkg/storage/storage.go

package storage

import (
	"io"
	"os"
	"strings"
	"time"

	"github.com/opencost/opencost/core/pkg/log"
	"github.com/pkg/errors"
)

// DirDelim is the delimiter used to model a directory structure in an object store bucket.
const DirDelim = "/"

// DoesNotExistError is used as a generic error to return when a target path does not
// exist in storage. Equivalent to os.ErrorNotExist such that it will work with os.IsNotExist(err)
var DoesNotExistError = os.ErrNotExist

// StorageInfo is a data object containing basic information about the path in storage.
type StorageInfo struct {
	Name    string    // base name of the file
	Size    int64     // length in bytes for regular files
	ModTime time.Time // modification time
}

// Storage provides an API for storing binary data
type Storage interface {
	// StorageType returns a string identifier for the type of storage used by the implementation.
	StorageType() StorageType

	// FullPath returns the storage working path combined with the path provided
	FullPath(path string) string

	// Stat returns the StorageStats for the specific path.
	Stat(path string) (*StorageInfo, error)

	// Read uses the relative path of the storage combined with the provided path to
	// read the contents.
	Read(path string) ([]byte, error)

	// ReadStream returns an io.ReadCloser for the specified path. Implementations should
	// stream incrementally when possible to avoid loading entire objects into memory.
	ReadStream(path string) (io.ReadCloser, error)

	// ReadToLocalFile writes the specified file at path to destPath on the local file system.
	// Implementations may stream data to minimize RAM usage, but some backends may still buffer
	// data in memory depending on their capabilities. It is up to the caller to clean up the
	// local file when finished.
	ReadToLocalFile(path, destPath string) error

	// Write uses the relative path of the storage combined with the provided path
	// to write a new file or overwrite an existing file.
	Write(path string, data []byte) error

	// WriteStream creates a new relative path and returns the `io.WriteCloser` that can be used to
	// write into the storage path. Ensure that Close() is run on the returned writer.
	WriteStream(path string) (io.WriteCloser, error)

	// Remove uses the relative path of the storage combined with the provided path to
	// remove a file from storage permanently.
	Remove(path string) error

	// Exists uses the relative path of the storage combined with the provided path to
	// determine if the file exists.
	Exists(path string) (bool, error)

	// List uses the relative path of the storage combined with the provided path to return
	// storage information for the files.
	List(path string) ([]*StorageInfo, error)

	// ListDirectories uses the relative path of the storage combined with the provided path
	// to return storage information for only directories contained along the path. This
	// functions as List, but returns storage information for only directories.
	ListDirectories(path string) ([]*StorageInfo, error)
}

// Validate uses the provided storage implementation to write a test file to the store, followed by a removal.
func Validate(storage Storage, validateWriteDelete bool) error {
	const testPath = "tmp/test.txt"
	const testContent = "test"

	log.Debug("validating storage")

	// attempt to read a path
	_, err := storage.Exists(testPath)
	if err != nil {
		return errors.Wrap(err, "Failed to check if path exists")
	}

	// attempt to list a path
	_, err = storage.List(testPath)
	if err != nil {
		return errors.Wrap(err, "Failed to list path")
	}

	if validateWriteDelete {
		// attempt to write a path
		err = storage.Write(testPath, []byte(testContent))
		if err != nil {
			return errors.Wrap(err, "Failed to write data to storage")
		}
	}

	// attempt to read the path
	// If we are not validating write and delete, the file won't exist since we never wrote it.
	// We only want to check read permissions, so ignore errors with "exist" and "404" in the error message to bypass the file not exist error.
	data, err := storage.Read(testPath)
	if err != nil && !strings.Contains(err.Error(), "exist") && !strings.Contains(err.Error(), "404") {
		return errors.Wrap(err, "Failed to read data from storage")
	}

	if validateWriteDelete {
		if string(data) != testContent {
			return errors.New("Failed to read the expected data from storage")
		}

		// delete the path
		err = storage.Remove(testPath)
		if err != nil {
			return errors.Wrap(err, "Failed to remove data from storage")
		}
	}

	return nil
}

// IsNotExist returns true if the error provided from a storage object is DoesNotExist
func IsNotExist(err error) bool {
	if err == nil {
		return false
	}

	return err.Error() == DoesNotExistError.Error()
}