s3

Documentation

Index

• Constants
• func DecodeStorageMetadata(m map[string]*string) (md store.StorageMetadata, retErr error)
• func EncodeStorageMetadata(m store.StorageMetadata) map[string]*string
• type StorageClient
  • func NewStorageClient(config *lfsb.Config) *StorageClient
  • func (c *StorageClient) Close() error
  • func (c *StorageClient) Clusters(ctx context.Context) ([]string, error)
  • func (c *StorageClient) Databases(ctx context.Context, cluster string) ([]string, error)
  • func (c *StorageClient) DeleteAll(ctx context.Context) (int, error)
  • func (c *StorageClient) DeleteFiles(ctx context.Context, paths []store.StoragePath) error
  • func (c *StorageClient) Iterator(ctx context.Context, cluster, database string, level int) (store.StoragePathIterator, error)
  • func (c *StorageClient) Levels(ctx context.Context, cluster, database string) ([]int, error)
  • func (c *StorageClient) Metadata(ctx context.Context, path store.StoragePath) (md store.StorageMetadata, err error)
  • func (c *StorageClient) Open() (err error)
  • func (c *StorageClient) OpenFile(ctx context.Context, path store.StoragePath) (io.ReadCloser, error)
  • func (c *StorageClient) Type() string
  • func (c *StorageClient) WriteFile(ctx context.Context, path store.StoragePath, r io.Reader) error
• type StoragePathIterator
  • func (itr *StoragePathIterator) Close() error
  • func (itr *StoragePathIterator) NextStoragePath(ctx context.Context) (store.StoragePath, error)

Constants

const PathSeparator = '/'

PathSeparator is the character used to separate parts of an S3 path.

Functions

func DecodeStorageMetadata

func DecodeStorageMetadata(m map[string]*string) (md store.StorageMetadata, retErr error)

DecodeStorageMetadata decodes m into a metadata structure from S3 object metadata.

func EncodeStorageMetadata

func EncodeStorageMetadata(m store.StorageMetadata) map[string]*string

EncodeStorageMetadata encodes m into S3 object metadata.

Types

type StorageClient

type StorageClient struct {
	// contains filtered or unexported fields
}

StorageClient represents a client for long-term storage on the file system.

func NewStorageClient

func NewStorageClient(config *lfsb.Config) *StorageClient

NewStorageClient returns a new instance of StorageClient.

func (*StorageClient) Close

func (c *StorageClient) Close() error

Close is a no-op.

func (*StorageClient) Clusters

func (c *StorageClient) Clusters(ctx context.Context) ([]string, error)

Clusters returns a list of clusters within an org.

func (*StorageClient) Databases

func (c *StorageClient) Databases(ctx context.Context, cluster string) ([]string, error)

Databases returns a list of databases within a cluster.

func (*StorageClient) DeleteAll

func (c *StorageClient) DeleteAll(ctx context.Context) (int, error)

DeleteAll removes all objects from the cluster. Returns number deleted. Used for testing.

func (*StorageClient) DeleteFiles

func (c *StorageClient) DeleteFiles(ctx context.Context, paths []store.StoragePath) error

DeleteFiles removes a list of paths from the cluster.

func (*StorageClient) Iterator

func (c *StorageClient) Iterator(ctx context.Context, cluster, database string, level int) (store.StoragePathIterator, error)

Returns an iterator over a given database's compaction level.

func (*StorageClient) Levels

func (c *StorageClient) Levels(ctx context.Context, cluster, database string) ([]int, error)

Levels returns a list of levels within a cluster.

func (*StorageClient) Metadata

func (c *StorageClient) Metadata(ctx context.Context, path store.StoragePath) (md store.StorageMetadata, err error)

Metadata returns metadata for a specific file. Returns os.NotExist if not found.

func (*StorageClient) Open

func (c *StorageClient) Open() (err error)

Open initializes the s3 client.

See AWS session docs for details: https://docs.aws.amazon.com/sdk-for-go/api/aws/session/

func (*StorageClient) OpenFile

func (c *StorageClient) OpenFile(ctx context.Context, path store.StoragePath) (io.ReadCloser, error)

OpenFile returns a reader for a specific file. Returns os.NotExist if not found.

func (*StorageClient) Type

func (c *StorageClient) Type() string

func (*StorageClient) WriteFile

func (c *StorageClient) WriteFile(ctx context.Context, path store.StoragePath, r io.Reader) error

WriteFile writes the contents of r to a path in long-term storage.

type StoragePathIterator

type StoragePathIterator struct {
	// contains filtered or unexported fields
}

StoragePathIterator returns an iterator over a given level.

func (*StoragePathIterator) Close

func (itr *StoragePathIterator) Close() error

Close cancels the iterator and waits for the background goroutine to finish.

func (*StoragePathIterator) NextStoragePath

func (itr *StoragePathIterator) NextStoragePath(ctx context.Context) (store.StoragePath, error)

NextStoragePath returns the next available storage path. Returns io.EOF when no more paths are available.