package storage import ( "fmt" "io" gofs "io/fs" "os" gopath "path" "path/filepath" "github.com/opencost/opencost/core/pkg/log" "github.com/opencost/opencost/core/pkg/util/fileutil" "github.com/pkg/errors" ) // FileStorage leverages the file system to write data to disk. type FileStorage struct { baseDir string } // NewFileStorage returns a new storage API which leverages the file system. func NewFileStorage(baseDir string) Storage { return &FileStorage{baseDir} } // String returns the base directory for the file storage. func (fs *FileStorage) String() string { return fs.baseDir } // StorageType returns a string identifier for the type of storage used by the implementation. func (fs *FileStorage) StorageType() StorageType { return StorageTypeFile } // FullPath returns the storage working path combined with the path provided func (fs *FileStorage) FullPath(path string) string { return gopath.Join(fs.baseDir, path) } // Stat returns the StorageStats for the specific path. func (fs *FileStorage) Stat(path string) (*StorageInfo, error) { f := gopath.Join(fs.baseDir, path) st, err := os.Stat(f) if err != nil { if os.IsNotExist(err) { return nil, DoesNotExistError } return nil, errors.Wrap(err, "Failed to stat file") } return FileToStorageInfo(st), nil } // List uses the relative path of the storage combined with the provided path to return // storage information for the files. func (fs *FileStorage) List(path string) ([]*StorageInfo, error) { p := gopath.Join(fs.baseDir, path) // Read files in the backup path entries, err := os.ReadDir(p) if err != nil { if os.IsNotExist(err) { return []*StorageInfo{}, nil } return nil, err } files := make([]gofs.FileInfo, 0, len(entries)) for _, entry := range entries { info, err := entry.Info() if err != nil { return nil, err } files = append(files, info) } return FilesToStorageInfo(files), nil } func (fs *FileStorage) ListDirectories(path string) ([]*StorageInfo, error) { p := gopath.Join(fs.baseDir, path) // Read files in the backup path entries, err := os.ReadDir(p) if err != nil { if os.IsNotExist(err) { return []*StorageInfo{}, nil } return nil, err } files := make([]gofs.FileInfo, 0, len(entries)) for _, entry := range entries { info, err := entry.Info() if err != nil { return nil, err } files = append(files, info) } return DirFilesToStorageInfo(files, path, fs.baseDir), nil } // Read uses the relative path of the storage combined with the provided path to // read the contents. // // It takes advantage of flock() based locking to improve safety. func (fs *FileStorage) Read(path string) ([]byte, error) { f := filepath.Join(fs.baseDir, path) b, err := fileutil.ReadLocked(f) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, DoesNotExistError } return nil, fmt.Errorf("reading %s: %w", f, err) } return b, nil } // ReadStream returns a streaming reader for the specified file path. func (fs *FileStorage) ReadStream(path string) (io.ReadCloser, error) { f := filepath.Join(fs.baseDir, path) file, err := os.Open(f) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, DoesNotExistError } return nil, fmt.Errorf("opening %s: %w", f, err) } return file, nil } // ReadToLocalFile streams the specified file at path to destPath on the local file system. // // For FileStorage, this is implemented as a local file copy. func (fs *FileStorage) ReadToLocalFile(path, destPath string) error { src := filepath.Join(fs.baseDir, path) in, err := os.Open(src) if err != nil { if os.IsNotExist(err) { return DoesNotExistError } return fmt.Errorf("reading %s: %w", src, err) } defer in.Close() if err := os.MkdirAll(filepath.Dir(destPath), os.ModePerm); err != nil { return fmt.Errorf("creating destination directory: %w", err) } out, err := os.Create(destPath) if err != nil { return fmt.Errorf("creating destination file %s: %w", destPath, err) } defer out.Close() // Use 1 MB buffer for file copy operations buf := make([]byte, 1024*1024) if _, err := io.CopyBuffer(out, in, buf); err != nil { return fmt.Errorf("streaming %s to %s: %w", src, destPath, err) } return nil } // Write uses the relative path of the storage combined with the provided path // to write a new file or overwrite an existing file. // // It takes advantage of flock() based locking to improve safety. func (fs *FileStorage) Write(path string, data []byte) error { f, err := fs.prepare(path) if err != nil { return errors.Wrap(err, "Failed to prepare path") } if _, err := fileutil.WriteLocked(f, data); err != nil { return fmt.Errorf("writing %s: %w", f, err) } return nil } // Remove uses the relative path of the storage combined with the provided path to // remove a file from storage permanently. func (fs *FileStorage) Remove(path string) error { f := gopath.Join(fs.baseDir, path) err := os.Remove(f) if err != nil { if os.IsNotExist(err) { return DoesNotExistError } return errors.Wrap(err, "Failed to remove file") } return nil } // Exists uses the relative path of the storage combined with the provided path to // determine if the file exists. func (fs *FileStorage) Exists(path string) (bool, error) { f := gopath.Join(fs.baseDir, path) return fileutil.FileExists(f) } // prepare checks to see if the directory being written to should be created before writing // the file, and then returns the correct full path. func (fs *FileStorage) prepare(path string) (string, error) { f := gopath.Join(fs.baseDir, path) dir := filepath.Dir(f) if _, e := os.Stat(dir); e != nil && os.IsNotExist(e) { err := os.MkdirAll(dir, os.ModePerm) if err != nil { return "", err } } return f, nil } // FilesToStorageInfo maps a []fs.FileInfo to []*storage.StorageInfo func FilesToStorageInfo(fileInfo []gofs.FileInfo) []*StorageInfo { var stats []*StorageInfo for _, info := range fileInfo { if info.IsDir() { continue } stats = append(stats, FileToStorageInfo(info)) } return stats } // FileToStorageInfo maps a fs.FileInfo to *storage.StorageInfo func FileToStorageInfo(fileInfo gofs.FileInfo) *StorageInfo { return &StorageInfo{ Name: fileInfo.Name(), Size: fileInfo.Size(), ModTime: fileInfo.ModTime(), } } // DirFilesToStorageInfo maps a []fs.FileInfo to []*storage.StorageInfo // but only returning StorageInfo for directories func DirFilesToStorageInfo(fileInfo []gofs.FileInfo, relativePath string, basePath string) []*StorageInfo { var stats []*StorageInfo for _, info := range fileInfo { if info.IsDir() { stats = append(stats, &StorageInfo{ Name: filepath.Join(relativePath, info.Name()), Size: info.Size(), ModTime: info.ModTime(), }) } else if info.Mode()&os.ModeSymlink == os.ModeSymlink { targetPath, err := os.Readlink(filepath.Join(basePath, relativePath, info.Name())) if err != nil { log.Warnf("Converting files to storage info failed on supposed symlink that failed to be Readlink-ed (%s): %s", filepath.Join(basePath, relativePath, info.Name()), err) continue } if targetInfo, err := os.Stat(targetPath); err == nil { if targetInfo.IsDir() { stats = append(stats, &StorageInfo{ // The Name added here is the path not readlink-ed Name: filepath.Join(relativePath, info.Name()), Size: info.Size(), ModTime: info.ModTime(), }) } } else if errors.Is(err, os.ErrNotExist) { continue } else { log.Warnf("Converting files to storage info failed to stat target (%s) of symlink (%s): %s", targetPath, info.Name(), err) continue } } } return stats }