writablefs

README

writablefs

A minimal superset of fs.FS supporting write operations.

Inspired by hackpadfs, rclone, and s3fs.

Backends

• Local directory filesystem.
• S3 compatible object storage.

Usage

To use an S3 compatible object storage as a read-write filesystem:

package main

import (
	"context"
	"log/slog"
	"os"

	"github.com/bucket-sailor/writablefs"
	"github.com/bucket-sailor/writablefs/s3fs"
	"github.com/minio/minio-go/v7/pkg/credentials"
)

func main() {
	opts := s3fs.Options{
		EndpointURL: "http://localhost:8080",
		Credentials: credentials.NewStaticV4("admin", "admin", ""),
		BucketName:  "test",
	}

	logger := slog.Default()
	fsys, err := s3fs.New(context.Background(), logger, opts)
	if err != nil {
		logger.Error("Failed to create S3 FS", "error", err)
		os.Exit(1)
	}
	defer fsys.Close()

	// Use fsys as a fs.FS or as a writablefs.FS.

	f, err := fsys.OpenFile("test.txt", writablefs.FlagReadWrite|writablefs.FlagCreate)
	if err != nil {
		logger.Error("Failed to open file", "error", err)
		os.Exit(1)
	}

	if _, err := f.Write([]byte("Hello, world!")); err != nil {
		logger.Error("Failed to write to file", "error", err)
		os.Exit(1)
	}

	if err := f.Close(); err != nil {
		logger.Error("Failed to close file", "error", err)
		os.Exit(1)
	}
}

Caveats:

• S3 objects are immutable (but versionable). This means changing a single byte in a file will result in a new object being created. To avoid this becoming a huge problem the S3 backend will only flush/upload writes when the file is closed or when Sync() is explicitly called. This means that if the program crashes or is killed pending writes will be lost. So flush as often as makes sense, also perhaps consider spreading writes across multiple smaller files.
• Not all S3 implementations are strongly consistent (but Amazon and a lot of others are). This means writes may not be immediately visible to other clients.

TODOs

• Port the Filesystem Test Suite to Go (of course almost no backends will be fully compliant).
• Add POSIX attributes to S3 objects (e.g. owner, group, permissions) via S3 object metadata.
• Most providers now offer strong read-after-write and metadata consistency. This means we can implement distributed flock()!

Documentation

Index

• Constants
• Variables
• type DirEntry
• type FS
  • func Sub(fsys FS, prefix string) FS
• type File
• type FileExtendedAttributes
• type FileInfo
• type FileMode
• type FileOpenFlag
  • func (f FileOpenFlag) IsSet(flag FileOpenFlag) bool
• type FileReadOnly

Constants

const (
	ModeDir  = gofs.ModeDir
	ModePerm = gofs.ModePerm
)

const (
	FlagReadOnly  = FileOpenFlag(os.O_RDONLY)
	FlagWriteOnly = FileOpenFlag(os.O_WRONLY)
	FlagReadWrite = FileOpenFlag(os.O_RDWR)
	FlagCreate    = FileOpenFlag(os.O_CREATE)
)

Variables

var (
	ErrInvalid    = gofs.ErrInvalid    // "invalid argument"
	ErrPermission = gofs.ErrPermission // "permission denied"
	ErrExist      = gofs.ErrExist      // "file already exists"
	ErrNotExist   = gofs.ErrNotExist   // "file does not exist"
	ErrClosed     = gofs.ErrClosed     // "file already closed"
)

Types

type DirEntry

type DirEntry = gofs.DirEntry

type FS

type FS interface {
	io.Closer
	gofs.FS
	gofs.ReadDirFS
	gofs.StatFS

	// OpenFile opens a file using the given flags.
	// By passing O_RDWR, the file can be opened for writing.
	OpenFile(path string, flag FileOpenFlag) (File, error)

	// MkdirAll creates a directory named path, along with any necessary parents.
	MkdirAll(path string) error

	// RemoveAll removes path and any children it contains.
	RemoveAll(path string) error

	// Rename renames (moves) oldpath to newpath.
	Rename(oldPath, newPath string) error
}

FS is the interface implemented by a writeable file system.

func Sub

func Sub(fsys FS, prefix string) FS

Sub returns a writable FS that is a subdirectory of the given FS.

type File

type File interface {
	FileReadOnly
	io.Writer
	io.Seeker
	io.ReaderAt
	io.WriterAt
	Sync() error
	Truncate(size int64) error
	ExtendedAttributes() FileExtendedAttributes
}

File is the interface implemented by a writeable file.

type FileExtendedAttributes

type FileExtendedAttributes interface {
	// Get returns the value of the extended attribute identified by name.
	Get(name string) ([]byte, error)
	// Set sets the value of the extended attribute identified by name.
	Set(name string, data []byte) error
	// Remove removes the extended attribute identified by name.
	Remove(name string) error
	// List returns the names of all extended attributes associated with the file.
	List() ([]string, error)
	// Sync flushes any metadata changes to the file system.
	Sync() error
}

FileExtendedAttributes is an interface for working with file extended attributes.

type FileInfo

type FileInfo = gofs.FileInfo

type FileMode

type FileMode = gofs.FileMode

type FileOpenFlag

type FileOpenFlag int

FileOpenFlag allows configuring how a file is opened.

func (FileOpenFlag) IsSet

func (f FileOpenFlag) IsSet(flag FileOpenFlag) bool

type FileReadOnly

type FileReadOnly = gofs.File

FileReadOnly is the interface implemented by a read-only file. This is kept for compatibility with io/fs.