writablefs

README

writablefs

A superset of fs.FS supporting write operations.

Inspired by hackpadfs, rclone, and s3fs.

Backends

• Local directory filesystem.
• S3 compatible object storage.

Usage

To use an S3 compatible object storage as a read-write filesystem:

package main

import (
	"context"
	"log/slog"
	"os"

	"github.com/bucket-sailor/writablefs"
	"github.com/bucket-sailor/writablefs/s3fs"
	"github.com/minio/minio-go/v7/pkg/credentials"
)

func main() {
	opts := s3fs.Options{
		EndpointURL: "http://localhost:8080",
		Credentials: credentials.NewStaticV4("admin", "admin", ""),
		BucketName:  "test",
	}

	logger := slog.Default()
	fsys, err := s3fs.New(context.Background(), logger, opts)
	if err != nil {
		logger.Error("Failed to create S3 FS", "error", err)
		os.Exit(1)
	}
	defer fsys.Close()

	// Use fsys as a fs.FS or as a writablefs.FS.

	f, err := fsys.OpenFile("test.txt", writablefs.FlagReadWrite|writablefs.FlagCreate)
	if err != nil {
		logger.Error("Failed to open file", "error", err)
		os.Exit(1)
	}

	if _, err := f.Write([]byte("Hello, world!")); err != nil {
		logger.Error("Failed to write to file", "error", err)
		os.Exit(1)
	}

	if err := f.Close(); err != nil {
		logger.Error("Failed to close file", "error", err)
		os.Exit(1)
	}
}

Caveats:

• S3 objects are immutable (but versionable). This means changing a single byte in a file will result in a new object being created. To avoid this becoming a huge problem the S3 backend will only flush/upload writes when the file is closed or when Sync() is explicitly called. This means that if the program crashes or is killed pending writes will be lost. So flush as often as makes sense, also perhaps consider spreading writes across multiple smaller files.
• Not all S3 implementations are strongly consistent (but Amazon and a lot of others are). This means writes may not be immediately visible to other clients.

TODOs

• Port the Filesystem Test Suite to Go (of course almost no backends will be fully compliant).
• Add POSIX attributes to S3 objects (e.g. owner, group, permissions) via S3 object metadata.
• Most providers now offer strong read-after-write and metadata consistency. This means we can implement distributed flock()!

Documentation

Index

• Constants
• Variables
• type ArchiveFS
• type DirEntry
• type ExtendedAttributes
• type FS
  • func Sub(fsys FS, prefix string) FS
• type File
• type FileInfo
• type FileMode
• type FileOpenFlag
  • func (f FileOpenFlag) IsSet(flag FileOpenFlag) bool
• type FileReadOnly

Constants

const (
	ModeDir  = gofs.ModeDir
	ModePerm = gofs.ModePerm
)

const (
	FlagReadOnly  = FileOpenFlag(os.O_RDONLY)
	FlagWriteOnly = FileOpenFlag(os.O_WRONLY)
	FlagReadWrite = FileOpenFlag(os.O_RDWR)
	FlagCreate    = FileOpenFlag(os.O_CREATE)
)

Variables

var (
	ErrInvalid    = gofs.ErrInvalid                 // "invalid argument"
	ErrPermission = gofs.ErrPermission              // "permission denied"
	ErrExist      = gofs.ErrExist                   // "file already exists"
	ErrNotExist   = gofs.ErrNotExist                // "file does not exist"
	ErrClosed     = gofs.ErrClosed                  // "file already closed"
	ErrNoSuchAttr = fmt.Errorf("no such attribute") // "no such attribute"
)

Types

type ArchiveFS

type ArchiveFS interface {
	FS

	// Archive creates a tar archive of the directory at the given path.
	Archive(path string) (io.ReadCloser, error)
}

ArchiveFS is the interface implemented by a file system that can create tarballs. This useful for dowloading whole directories etc.

type DirEntry

type DirEntry = gofs.DirEntry

type ExtendedAttributes

type ExtendedAttributes interface {
	// Get returns the value of the extended attribute identified by name.
	Get(name string) ([]byte, error)
	// Set sets the value of the extended attribute identified by name.
	Set(name string, data []byte) error
	// Remove removes the extended attribute identified by name.
	Remove(name string) error
	// List returns the names of all extended attributes associated with the file.
	List() ([]string, error)
	// Sync flushes any metadata changes to the file system.
	Sync() error
}

ExtendedAttributes is an interface for working with file extended attributes.

type FS

type FS interface {
	io.Closer
	gofs.FS
	gofs.ReadDirFS
	gofs.StatFS

	// OpenFile opens a file using the given flags.
	// By passing O_RDWR, the file can be opened for writing.
	OpenFile(path string, flag FileOpenFlag) (File, error)

	// MkdirAll creates a directory named path, along with any necessary parents.
	MkdirAll(path string) error

	// RemoveAll removes path and any children it contains.
	RemoveAll(path string) error

	// Rename renames (moves) oldpath to newpath.
	Rename(oldPath, newPath string) error
}

FS is the interface implemented by a writeable file system.

func Sub

func Sub(fsys FS, prefix string) FS

Sub returns a writable FS that is a subdirectory of the given FS.

type File

type File interface {
	FileReadOnly
	io.Writer
	io.Seeker
	io.ReaderAt
	io.WriterAt

	// Sync flushes any changes to the file system.
	Sync() error

	// Truncate changes the size of the file.
	Truncate(size int64) error

	// XAttrs returns the extended attributes of the file.
	// If the file system does not support extended attributes,
	// an error is returned. You should call Sync() after modifying
	// the extended attributes to ensure they are persisted.
	XAttrs() (ExtendedAttributes, error)
}

File is the interface implemented by a writeable file.

type FileInfo

type FileInfo = gofs.FileInfo

type FileMode

type FileMode = gofs.FileMode

type FileOpenFlag

type FileOpenFlag int

FileOpenFlag allows configuring how a file is opened.

func (FileOpenFlag) IsSet

func (f FileOpenFlag) IsSet(flag FileOpenFlag) bool

type FileReadOnly

type FileReadOnly = gofs.File

FileReadOnly is the interface implemented by a read-only file. This is kept for compatibility with io/fs.