storage

Documentation

Index

• func NameLockfile(file string) string
• type ConnectOptions
• type Filesystem
  • func (b *Filesystem) CanListFiles() bool
  • func (b *Filesystem) Close() error
  • func (b *Filesystem) Exists(pth string) (bool, error)
  • func (b *Filesystem) GetFile(pth string) (io.ReadCloser, error)
  • func (b *Filesystem) ListFiles(pth string) (chan string, chan error)
  • func (b *Filesystem) PutFile(pth string, in io.ReadCloser) error
  • func (b *Filesystem) Size(pth string) (int64, error)
• type GCSStorage
  • func (b *GCSStorage) CanListFiles() bool
  • func (b *GCSStorage) Close() error
  • func (b *GCSStorage) Exists(pth string) (bool, error)
  • func (b *GCSStorage) GetFile(pth string) (io.ReadCloser, error)
  • func (b *GCSStorage) ListFiles(pth string) (chan string, chan error)
  • func (b *GCSStorage) PutFile(pth string, in io.ReadCloser) error
  • func (b *GCSStorage) Size(pth string) (int64, error)
• type HttpStorage
  • func (b *HttpStorage) CanListFiles() bool
  • func (b *HttpStorage) Close() error
  • func (b *HttpStorage) Exists(pth string) (bool, error)
  • func (b *HttpStorage) GetFile(pth string) (io.ReadCloser, error)
  • func (b *HttpStorage) Head(pth string) (*http.Response, error)
  • func (b *HttpStorage) ListFiles(pth string) (chan string, chan error)
  • func (b *HttpStorage) PutFile(pth string, in io.ReadCloser) error
  • func (b *HttpStorage) Size(pth string) (int64, error)
• type OnDiskCache
  • func (b *OnDiskCache) Close() error
  • func (b *OnDiskCache) Evict(filepath string)
  • func (b *OnDiskCache) Exists(filepath string) (bool, error)
  • func (b *OnDiskCache) GetFile(filepath string) (io.ReadCloser, error)
  • func (b *OnDiskCache) PutFile(filepath string, in io.ReadCloser) error
  • func (b *OnDiskCache) Size(filepath string) (int64, error)
• type S3Storage
  • func (b *S3Storage) CanListFiles() bool
  • func (b *S3Storage) Close() error
  • func (b *S3Storage) Exists(pth string) (bool, error)
  • func (b *S3Storage) GetACLWriteRule() string
  • func (b *S3Storage) GetFile(pth string) (io.ReadCloser, error)
  • func (b *S3Storage) Head(pth string) (*http.Response, error)
  • func (b *S3Storage) ListFiles(pth string) (chan string, chan error)
  • func (b *S3Storage) PutFile(pth string, in io.ReadCloser) error
  • func (b *S3Storage) Size(pth string) (int64, error)
• type Storage
  • func ConnectBackend(u string, opts ConnectOptions) (Storage, error)
  • func MakeOnDiskCache(upstream Storage, dir string, maxFiles uint) (Storage, error)
  • func NewFilesystemStorage(pth string) Storage
  • func NewGCSBackend(ctx context.Context, bucketName string, prefix string, endpoint string) (Storage, error)
  • func NewHttpStorage(ctx context.Context, base *url.URL, userAgent string) Storage
  • func NewS3Storage(ctx context.Context, bucket string, prefix string, region string, ...) (Storage, error)

Functions

func NameLockfile

func NameLockfile(file string) string

Types

type ConnectOptions

type ConnectOptions struct {
	Context          context.Context
	S3Region         string
	S3Endpoint       string
	UnsignedRequests bool
	GCSEndpoint      string

	// When putting file object to s3 bucket, specify the ACL for the object.
	S3WriteACL string

	// UserAgent is the value of `User-Agent` header. Applicable only for HTTP
	// client.
	UserAgent string

	// Wrap the Storage after connection. For example, to add a caching or
	// introspection layer.
	Wrap func(Storage) (Storage, error)
}

type Filesystem

type Filesystem struct {
	// contains filtered or unexported fields
}

func (*Filesystem) CanListFiles

func (b *Filesystem) CanListFiles() bool

func (*Filesystem) Close

func (b *Filesystem) Close() error

func (*Filesystem) Exists

func (b *Filesystem) Exists(pth string) (bool, error)

func (*Filesystem) GetFile

func (b *Filesystem) GetFile(pth string) (io.ReadCloser, error)

func (*Filesystem) ListFiles

func (b *Filesystem) ListFiles(pth string) (chan string, chan error)

func (*Filesystem) PutFile

func (b *Filesystem) PutFile(pth string, in io.ReadCloser) error

func (*Filesystem) Size

func (b *Filesystem) Size(pth string) (int64, error)

type GCSStorage

type GCSStorage struct {
	// contains filtered or unexported fields
}

func (*GCSStorage) CanListFiles

func (b *GCSStorage) CanListFiles() bool

func (*GCSStorage) Close

func (b *GCSStorage) Close() error

func (*GCSStorage) Exists

func (b *GCSStorage) Exists(pth string) (bool, error)

func (*GCSStorage) GetFile

func (b *GCSStorage) GetFile(pth string) (io.ReadCloser, error)

func (*GCSStorage) ListFiles

func (b *GCSStorage) ListFiles(pth string) (chan string, chan error)

func (*GCSStorage) PutFile

func (b *GCSStorage) PutFile(pth string, in io.ReadCloser) error

func (*GCSStorage) Size

func (b *GCSStorage) Size(pth string) (int64, error)

type HttpStorage

type HttpStorage struct {
	// contains filtered or unexported fields
}

func (*HttpStorage) CanListFiles

func (b *HttpStorage) CanListFiles() bool

func (*HttpStorage) Close

func (b *HttpStorage) Close() error

func (*HttpStorage) Exists

func (b *HttpStorage) Exists(pth string) (bool, error)

func (*HttpStorage) GetFile

func (b *HttpStorage) GetFile(pth string) (io.ReadCloser, error)

func (*HttpStorage) Head

func (b *HttpStorage) Head(pth string) (*http.Response, error)

func (*HttpStorage) ListFiles

func (b *HttpStorage) ListFiles(pth string) (chan string, chan error)

func (*HttpStorage) PutFile

func (b *HttpStorage) PutFile(pth string, in io.ReadCloser) error

func (*HttpStorage) Size

func (b *HttpStorage) Size(pth string) (int64, error)

type OnDiskCache

type OnDiskCache struct {
	Storage
	// contains filtered or unexported fields
}

OnDiskCache fronts another storage with a local filesystem cache. Its thread-safe, meaning you can be actively caching a file and retrieve it at the same time without corruption, because retrieval will wait for the fetch.

func (*OnDiskCache) Close

func (b *OnDiskCache) Close() error

Close purges the cache, then forwards the call to the wrapped backend.

func (*OnDiskCache) Evict

func (b *OnDiskCache) Evict(filepath string)

Evict removes a file from the cache and the filesystem, but does not affect the upstream backend. It isn't part of the `Storage` interface.

func (*OnDiskCache) Exists

func (b *OnDiskCache) Exists(filepath string) (bool, error)

Exists shortcuts an existence check by checking if it exists in the cache. Otherwise, it returns the same result as the wrapped backend. Note that in the latter case, the cache isn't modified.

func (*OnDiskCache) GetFile

func (b *OnDiskCache) GetFile(filepath string) (io.ReadCloser, error)

GetFile retrieves the file contents from the local cache if present. Otherwise, it returns the same result that the wrapped backend returns and adds that result into the local cache, if possible.

func (*OnDiskCache) PutFile

func (b *OnDiskCache) PutFile(filepath string, in io.ReadCloser) error

PutFile writes to the given `filepath` from the given `in` reader, also writing it to the local cache if possible. It returns the same result as the wrapped backend.

func (*OnDiskCache) Size

func (b *OnDiskCache) Size(filepath string) (int64, error)

Size will return the size of the file found in the cache if possible. Otherwise, it returns the same result as the wrapped backend. Note that in the latter case, the cache isn't modified.

type S3Storage

type S3Storage struct {
	// contains filtered or unexported fields
}

func (*S3Storage) CanListFiles

func (b *S3Storage) CanListFiles() bool

func (*S3Storage) Close

func (b *S3Storage) Close() error

func (*S3Storage) Exists

func (b *S3Storage) Exists(pth string) (bool, error)

func (*S3Storage) GetACLWriteRule

func (b *S3Storage) GetACLWriteRule() string

func (*S3Storage) GetFile

func (b *S3Storage) GetFile(pth string) (io.ReadCloser, error)

func (*S3Storage) Head

func (b *S3Storage) Head(pth string) (*http.Response, error)

func (*S3Storage) ListFiles

func (b *S3Storage) ListFiles(pth string) (chan string, chan error)

func (*S3Storage) PutFile

func (b *S3Storage) PutFile(pth string, in io.ReadCloser) error

func (*S3Storage) Size

func (b *S3Storage) Size(pth string) (int64, error)

type Storage

type Storage interface {
	Exists(path string) (bool, error)
	Size(path string) (int64, error)
	GetFile(path string) (io.ReadCloser, error)
	PutFile(path string, in io.ReadCloser) error
	ListFiles(path string) (chan string, chan error)
	CanListFiles() bool
	Close() error
}

func ConnectBackend

func ConnectBackend(u string, opts ConnectOptions) (Storage, error)

func MakeOnDiskCache

func MakeOnDiskCache(upstream Storage, dir string, maxFiles uint) (Storage, error)

MakeOnDiskCache wraps an Storage with a local filesystem cache in `dir`. If dir is blank, a temporary directory will be created. If `maxFiles` is zero, a default (90 days of ledgers) is used.

func NewFilesystemStorage

func NewFilesystemStorage(pth string) Storage

func NewGCSBackend

func NewGCSBackend(
	ctx context.Context,
	bucketName string,
	prefix string,
	endpoint string,
) (Storage, error)

func NewHttpStorage

func NewHttpStorage(ctx context.Context, base *url.URL, userAgent string) Storage

func NewS3Storage

func NewS3Storage(
	ctx context.Context,
	bucket string,
	prefix string,
	region string,
	endpoint string,
	unsignedRequests bool,
	writeACLrule string,
) (Storage, error)