objstore

Documentation

Index

• Constants
• func AcceptanceTest(t *testing.T, bkt Bucket)
• func BucketWithMetrics(name string, b Bucket, reg prometheus.Registerer) *metricBucket
• func CreateTemporaryTestBucketName(t testing.TB) string
• func DownloadDir(ctx context.Context, logger log.Logger, bkt BucketReader, src, dst string) error
• func DownloadFile(ctx context.Context, logger log.Logger, bkt BucketReader, src, dst string) (err error)
• func EmptyBucket(t testing.TB, ctx context.Context, bkt Bucket)
• func TryToGetSize(r io.Reader) (int64, error)
• func UploadDir(ctx context.Context, logger log.Logger, bkt Bucket, srcdir, dstdir string) error
• func UploadFile(ctx context.Context, logger log.Logger, bkt Bucket, src, dst string) error
• type Bucket
• type BucketReader
• type InMemBucket
  • func NewInMemBucket() *InMemBucket
  • func (b *InMemBucket) Close() error
  • func (b *InMemBucket) Delete(_ context.Context, name string) error
  • func (b *InMemBucket) Exists(_ context.Context, name string) (bool, error)
  • func (b *InMemBucket) Get(_ context.Context, name string) (io.ReadCloser, error)
  • func (b *InMemBucket) GetRange(_ context.Context, name string, off, length int64) (io.ReadCloser, error)
  • func (b *InMemBucket) IsObjNotFoundErr(err error) bool
  • func (b *InMemBucket) Iter(_ context.Context, dir string, f func(string) error) error
  • func (b *InMemBucket) Name() string
  • func (b *InMemBucket) ObjectSize(_ context.Context, name string) (uint64, error)
  • func (b *InMemBucket) Objects() map[string][]byte
  • func (b *InMemBucket) Upload(_ context.Context, name string, r io.Reader) error
• type InstrumentedBucket
  • func WithNoopInstr(bkt Bucket) InstrumentedBucket
• type InstrumentedBucketReader
• type IsOpFailureExpectedFunc

Constants

const DirDelim = "/"

DirDelim is the delimiter used to model a directory structure in an object store bucket.

Functions

func AcceptanceTest

func AcceptanceTest(t *testing.T, bkt Bucket)

func BucketWithMetrics

func BucketWithMetrics(name string, b Bucket, reg prometheus.Registerer) *metricBucket

BucketWithMetrics takes a bucket and registers metrics with the given registry for operations run against the bucket.

func CreateTemporaryTestBucketName

func CreateTemporaryTestBucketName(t testing.TB) string

func DownloadDir

func DownloadDir(ctx context.Context, logger log.Logger, bkt BucketReader, src, dst string) error

DownloadDir downloads all object found in the directory into the local directory.

func DownloadFile

func DownloadFile(ctx context.Context, logger log.Logger, bkt BucketReader, src, dst string) (err error)

DownloadFile downloads the src file from the bucket to dst. If dst is an existing directory, a file with the same name as the source is created in dst. If destination file is already existing, download file will overwrite it.

func EmptyBucket

func EmptyBucket(t testing.TB, ctx context.Context, bkt Bucket)

EmptyBucket deletes all objects from bucket. This operation is required to properly delete bucket as a whole. It is used for testing only. TODO(bplotka): Add retries.

func TryToGetSize

func TryToGetSize(r io.Reader) (int64, error)

TryToGetSize tries to get upfront size from reader. TODO(https://github.com/thanos-io/thanos/issues/678): Remove guessing length when minio provider will support multipart upload without this.

func UploadDir

func UploadDir(ctx context.Context, logger log.Logger, bkt Bucket, srcdir, dstdir string) error

UploadDir uploads all files in srcdir to the bucket with into a top-level directory named dstdir. It is a caller responsibility to clean partial upload in case of failure.

func UploadFile

func UploadFile(ctx context.Context, logger log.Logger, bkt Bucket, src, dst string) error

UploadFile uploads the file with the given name to the bucket. It is a caller responsibility to clean partial upload in case of failure.

Types

type Bucket

type Bucket interface {
	io.Closer
	BucketReader

	// Upload the contents of the reader as an object into the bucket.
	// Upload should be idempotent.
	Upload(ctx context.Context, name string, r io.Reader) error

	// Delete removes the object with the given name.
	// If object does not exists in the moment of deletion, Delete should throw error.
	Delete(ctx context.Context, name string) error

	// Name returns the bucket name for the provider.
	Name() string
}

Bucket provides read and write access to an object storage bucket. NOTE: We assume strong consistency for write-read flow.

type BucketReader

type BucketReader interface {
	// Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full
	// object name including the prefix of the inspected directory.
	Iter(ctx context.Context, dir string, f func(string) error) error

	// Get returns a reader for the given object name.
	Get(ctx context.Context, name string) (io.ReadCloser, error)

	// GetRange returns a new range reader for the given object name and range.
	GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

	// Exists checks if the given object exists in the bucket.
	Exists(ctx context.Context, name string) (bool, error)

	// IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations.
	IsObjNotFoundErr(err error) bool

	// ObjectSize returns the size of the specified object.
	ObjectSize(ctx context.Context, name string) (uint64, error)
}

BucketReader provides read access to an object storage bucket.

type InMemBucket

type InMemBucket struct {
	// contains filtered or unexported fields
}

InMemBucket implements the objstore.Bucket interfaces against local memory. Methods from Bucket interface are thread-safe. Objects are assumed to be immutable.

func NewInMemBucket

func NewInMemBucket() *InMemBucket

NewInMemBucket returns a new in memory Bucket. NOTE: Returned bucket is just a naive in memory bucket implementation. For test use cases only.

func (*InMemBucket) Close

func (b *InMemBucket) Close() error

func (*InMemBucket) Delete

func (b *InMemBucket) Delete(_ context.Context, name string) error

Delete removes all data prefixed with the dir.

func (*InMemBucket) Exists

func (b *InMemBucket) Exists(_ context.Context, name string) (bool, error)

Exists checks if the given directory exists in memory.

func (*InMemBucket) Get

func (b *InMemBucket) Get(_ context.Context, name string) (io.ReadCloser, error)

Get returns a reader for the given object name.

func (*InMemBucket) GetRange

func (b *InMemBucket) GetRange(_ context.Context, name string, off, length int64) (io.ReadCloser, error)

GetRange returns a new range reader for the given object name and range.

func (*InMemBucket) IsObjNotFoundErr

func (b *InMemBucket) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations.

func (*InMemBucket) Iter

func (b *InMemBucket) Iter(_ context.Context, dir string, f func(string) error) error

Iter calls f for each entry in the given directory. The argument to f is the full object name including the prefix of the inspected directory.

func (*InMemBucket) Name

func (b *InMemBucket) Name() string

Name returns the bucket name.

func (*InMemBucket) ObjectSize

func (b *InMemBucket) ObjectSize(_ context.Context, name string) (uint64, error)

ObjectSize returns the size of the specified object.

func (*InMemBucket) Objects

func (b *InMemBucket) Objects() map[string][]byte

Objects returns internally stored objects. NOTE: For assert purposes.

func (*InMemBucket) Upload

func (b *InMemBucket) Upload(_ context.Context, name string, r io.Reader) error

Upload writes the file specified in src to into the memory.

type InstrumentedBucket

type InstrumentedBucket interface {
	Bucket

	// WithExpectedErrs allows to specify a filter that marks certain errors as expected, so it will not increment
	// thanos_objstore_bucket_operation_failures_total metric.
	WithExpectedErrs(IsOpFailureExpectedFunc) Bucket

	// ReaderWithExpectedErrs allows to specify a filter that marks certain errors as expected, so it will not increment
	// thanos_objstore_bucket_operation_failures_total metric.
	// TODO(bwplotka): Remove this when moved to Go 1.14 and replace with InstrumentedBucketReader.
	ReaderWithExpectedErrs(IsOpFailureExpectedFunc) BucketReader
}

InstrumentedBucket is a Bucket with optional instrumentation control on reader.

func WithNoopInstr

func WithNoopInstr(bkt Bucket) InstrumentedBucket

type InstrumentedBucketReader

type InstrumentedBucketReader interface {
	BucketReader

	// ReaderWithExpectedErrs allows to specify a filter that marks certain errors as expected, so it will not increment
	// thanos_objstore_bucket_operation_failures_total metric.
	ReaderWithExpectedErrs(IsOpFailureExpectedFunc) BucketReader
}

InstrumentedBucket is a BucketReader with optional instrumentation control.

type IsOpFailureExpectedFunc

type IsOpFailureExpectedFunc func(error) bool

IsOpFailureExpectedFunc allows to mark certain errors as expected, so they will not increment thanos_objstore_bucket_operation_failures_total metric.