bfs

README

BFS

Multi-adapter bucket-based file system abstraction.

Documentation

For documentation and examples, please see https://godoc.org/github.com/bsm/bfs.

Install

go get -u github.com/bsm/bfs

Basic Usage

package main

import (
	"fmt"

	"github.com/bsm/bfs"
)

func main() {
	ctx := context.Background()
	bucket := bfs.NewInMem()

	// Write object
	o1, err := bucket.Create(ctx, "nested/file.txt", nil)
	if err != nil {
		panic(err)
	}
	defer o1.Discard()

	if _, err := o1.Write([]byte("TESTDATA")); err != nil {
		panic(err)
	}
	if err := o1.Commit(); err != nil {
		panic(err)
	}

	// Glob entries
	entries, err := bucket.Glob(ctx, "nested/**")
	if err != nil {
		panic(err)
	}
	fmt.Println("ENTRIES:", entries)

	// Read object
	o2, err := bucket.Open(ctx, "nested/file.txt")
	if err != nil {
		panic(err)
	}
	defer o2.Close()

	data, err := io.ReadAll(o2)
	if err != nil {
		panic(err)
	}
	fmt.Println("DATA:", string(data))

	// Head object
	info, err := bucket.Head(ctx, "nested/file.txt")
	if err != nil {
		panic(err)
	}
	fmt.Printf("INFO: name=%q size=%d\n", info.Name, info.Size)

	// Delete object
	if err := bucket.Remove(ctx, "nested/file.txt"); err != nil {
		panic(err)
	}
}

Documentation

Overview

Package bfs outlines an abstraction for bucket-based fyle systems with mock-implmentations.

Index

• Variables
• func CopyObject(ctx context.Context, bucket Bucket, src, dst string, dstOpts *WriteOptions) error
• func Register(scheme string, resv Resolver)
• func Unregister(scheme string)
• func WriteObject(ctx context.Context, bucket Bucket, name string, data []byte, ...) error
• type Bucket
  • func Connect(ctx context.Context, urlStr string) (Bucket, error)
  • func Resolve(ctx context.Context, u *url.URL) (Bucket, error)
• type InMem
  • func NewInMem() *InMem
  • func (*InMem) Close() error
  • func (b *InMem) Create(ctx context.Context, name string, opts *WriteOptions) (Writer, error)
  • func (b *InMem) Glob(_ context.Context, pattern string) (Iterator, error)
  • func (b *InMem) Head(_ context.Context, name string) (*MetaInfo, error)
  • func (b *InMem) ObjectSizes() map[string]int64
  • func (b *InMem) Open(_ context.Context, name string) (Reader, error)
  • func (b *InMem) Remove(_ context.Context, name string) error
• type Iterator
• type MetaInfo
• type Metadata
  • func NormMetadata(kv map[string]string) Metadata
  • func (m Metadata) Del(key string)
  • func (m Metadata) Get(key string) string
  • func (m Metadata) Set(key, value string)
• type Object
  • func NewInMemObject(name string) *Object
  • func NewObject(ctx context.Context, fullURL string) (*Object, error)
  • func NewObjectFromBucket(bucket Bucket, name string) *Object
  • func (o *Object) Close() error
  • func (o *Object) Create(ctx context.Context, opts *WriteOptions) (Writer, error)
  • func (o *Object) Head(ctx context.Context) (*MetaInfo, error)
  • func (o *Object) Name() string
  • func (o *Object) Open(ctx context.Context) (Reader, error)
  • func (o *Object) Remove(ctx context.Context) error
• type Reader
• type Resolver
• type WriteOptions
  • func (o *WriteOptions) GetContentType() string
  • func (o *WriteOptions) GetMetadata() Metadata
• type Writer

Examples

• InMem

Variables

var ErrNotFound = errors.New("bfs: object not found")

ErrNotFound must be returned by all implementations when a requested object cannot be found.

Functions

func CopyObject

func CopyObject(ctx context.Context, bucket Bucket, src, dst string, dstOpts *WriteOptions) error

CopyObject is a quick helper to copy objects within the same bucket.

func Register

func Register(scheme string, resv Resolver)

Register registers a new protocol with a scheme and a corresponding resolver. Example (from bfs/bfsfs):

bfs.Register("file", func(_ context.Context, u *url.URL) (bfs.Bucket, error) {
  return bfsfs.New(u.Path, "")
})

bucket, err := bfs.Connect(context.TODO(), "file:///home/user/Documents")
...

func Unregister

func Unregister(scheme string)

Unregister removes a registered scheme. This is only really useful in tests.

func WriteObject

func WriteObject(ctx context.Context, bucket Bucket, name string, data []byte, opts *WriteOptions) error

WriteObject is a quick write helper.

Types

type Bucket

type Bucket interface {
	// Glob lists the files matching a glob pattern. It supports
	// `*`, `**`, `?` wildcards, character classes and alternative sequences.
	// Please see https://github.com/bmatcuk/doublestar#patterns for more details.
	Glob(ctx context.Context, pattern string) (Iterator, error)

	// Head returns an object's meta info.
	Head(ctx context.Context, name string) (*MetaInfo, error)

	// Open opens an object for reading.
	Open(ctx context.Context, name string) (Reader, error)

	// Create creates/opens a object for writing.
	Create(ctx context.Context, name string, opts *WriteOptions) (Writer, error)

	// Remove removes a object.
	Remove(ctx context.Context, name string) error

	// Close closes the bucket.
	Close() error
}

Bucket is an abstract storage bucket.

func Connect

func Connect(ctx context.Context, urlStr string) (Bucket, error)

Connect connects to a bucket via URL. Example (from bfs/bfsfs):

bfs.Register("file", func(_ context.Context, u *url.URL) (bfs.Bucket, error) {
  return bfsfs.New(u.Path, "")
})

bucket, err := bfs.Connect(context.TODO(), "file:///home/user/Documents")

func Resolve

func Resolve(ctx context.Context, u *url.URL) (Bucket, error)

Resolve opens a bucket from a URL. Example (from bfs/bfsfs):

bfs.Register("file", func(_ context.Context, u *url.URL) (bfs.Bucket, error) {
  return bfsfs.New(u.Path, "")
})

u, err := url.Parse("file:///home/user/Documents")
...
bucket, err := bfs.Resolve(context.TODO(), u)
...

type InMem

type InMem struct {
	// contains filtered or unexported fields
}

InMem is an in-memory Bucket implementation which can be used for mocking.

Example

package main

import (
	"context"
	"fmt"
	"io"

	"github.com/bsm/bfs"
)

func main() {
	ctx := context.Background()
	bucket := bfs.NewInMem()

	// Write object
	o1, err := bucket.Create(ctx, "nested/file.txt", nil)
	if err != nil {
		panic(err)
	}
	defer o1.Discard()

	if _, err := o1.Write([]byte("TESTDATA")); err != nil {
		panic(err)
	}
	if err := o1.Commit(); err != nil {
		panic(err)
	}

	// Glob entries
	entries, err := bucket.Glob(ctx, "nested/**")
	if err != nil {
		panic(err)
	}
	fmt.Println("ENTRIES:", entries)

	// Read object
	o2, err := bucket.Open(ctx, "nested/file.txt")
	if err != nil {
		panic(err)
	}
	defer o2.Close()

	data, err := io.ReadAll(o2)
	if err != nil {
		panic(err)
	}
	fmt.Println("DATA:", string(data))

	// Head object
	info, err := bucket.Head(ctx, "nested/file.txt")
	if err != nil {
		panic(err)
	}
	fmt.Printf("INFO: name=%q size=%d\n", info.Name, info.Size)

	// Delete object
	if err := bucket.Remove(ctx, "nested/file.txt"); err != nil {
		panic(err)
	}
}

func NewInMem

func NewInMem() *InMem

NewInMem returns an initialised Bucket.

func (*InMem) Close

func (*InMem) Close() error

Close implements Bucket.

func (*InMem) Create

func (b *InMem) Create(ctx context.Context, name string, opts *WriteOptions) (Writer, error)

Create implements Bucket.

func (*InMem) Glob

func (b *InMem) Glob(_ context.Context, pattern string) (Iterator, error)

Glob implements Bucket.

func (*InMem) Head

func (b *InMem) Head(_ context.Context, name string) (*MetaInfo, error)

Head implements Bucket.

func (*InMem) ObjectSizes

func (b *InMem) ObjectSizes() map[string]int64

ObjectSizes return a map of object sizes by name

func (*InMem) Open

func (b *InMem) Open(_ context.Context, name string) (Reader, error)

Open implements Bucket.

func (*InMem) Remove

func (b *InMem) Remove(_ context.Context, name string) error

Remove implements Bucket.

type Iterator

type Iterator interface {
	// Next advances the cursor to the next position.
	Next() bool
	// Name returns the name at the current cursor position.
	Name() string
	// Size returns the length of the content in bytes for the current object.
	Size() int64
	// ModTime returns the modification time for the current object.
	ModTime() time.Time
	// Error returns the last iterator error, if any.
	Error() error
	// Close closes the iterator, should always be deferred.
	Close() error
}

Iterator iterates over objects

type MetaInfo

type MetaInfo struct {
	Name        string    // base name of the object
	Size        int64     // length of the content in bytes
	ModTime     time.Time // modification time
	ContentType string    // content type
	Metadata    Metadata  // metadata
}

MetaInfo contains meta information about an object.

type Metadata

type Metadata map[string]string

Metadata contains metadata values.

func NormMetadata

func NormMetadata(kv map[string]string) Metadata

NormMetadata canonicalizes kv pairs (inline) and returns the result.

func (Metadata) Del

func (m Metadata) Del(key string)

Del deletes the values associated with key. The key is case insensitive; it is canonicalized by textproto.CanonicalMIMEHeaderKey.

func (Metadata) Get

func (m Metadata) Get(key string) string

Get gets the value associated with the given key. It is case insensitive; textproto.CanonicalMIMEHeaderKey is used to canonicalize the provided key. If there are no values associated with the key, Get returns "".

func (Metadata) Set

func (m Metadata) Set(key, value string)

Set sets the header entries associated with key to the single element value. It replaces any existing values associated with key.

type Object

type Object struct {
	// contains filtered or unexported fields
}

Object is a handle for a single file/object on a Bucket.

func NewInMemObject

func NewInMemObject(name string) *Object

NewInMemObject returns a new in-memory object.

func NewObject

func NewObject(ctx context.Context, fullURL string) (*Object, error)

NewObject inits a new object from an URL string

func NewObjectFromBucket

func NewObjectFromBucket(bucket Bucket, name string) *Object

NewObjectFromBucket inits a new object from an existing bucket.

func (*Object) Close

func (o *Object) Close() error

Close closes the object.

func (*Object) Create

func (o *Object) Create(ctx context.Context, opts *WriteOptions) (Writer, error)

Create creates/opens a object for writing.

func (*Object) Head

func (o *Object) Head(ctx context.Context) (*MetaInfo, error)

Head returns an object's meta info.

func (*Object) Name

func (o *Object) Name() string

Name returns an object's name.

func (*Object) Open

func (o *Object) Open(ctx context.Context) (Reader, error)

Open opens an object for reading.

func (*Object) Remove

func (o *Object) Remove(ctx context.Context) error

Remove removes a object.

type Reader

type Reader interface {
	io.ReadCloser
}

Reader is the interface that is returned by bucket.Open.

type Resolver

type Resolver func(context.Context, *url.URL) (Bucket, error)

Resolver constructs a bucket from a URL.

type WriteOptions

type WriteOptions struct {
	ContentType string
	Metadata    Metadata
}

WriteOptions provide optional configuration when creating/writing objects.

func (*WriteOptions) GetContentType

func (o *WriteOptions) GetContentType() string

GetContentType returns a content type.

func (*WriteOptions) GetMetadata

func (o *WriteOptions) GetMetadata() Metadata

GetMetadata returns a content type.

type Writer

type Writer interface {
	io.Writer

	// Discard closes and releases the writer without writing a file.
	Discard() error

	// Commit closes and commits the content by writing a file. Calls to Commit
	// will fail when Discard was called before.
	Commit() error
}

Writer is the interface that is returned by bucket.Create.