objstore

package
v1.6.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 30, 2024 License: AGPL-3.0 Imports: 15 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var ErrObjectDoesNotExist = errors.New("object does not exist")

ErrObjectDoesNotExist is used in tests to simulate objstore.Bucket.IsObjNotFoundErr().

Functions

func DeletePrefix

func DeletePrefix(ctx context.Context, bkt objstore.Bucket, prefix string, logger log.Logger) (int, error)

DeletePrefix removes all objects with given prefix, recursively. It returns number of deleted objects. If deletion of any object fails, it returns error and stops.

func NewDelayedBucketClient

func NewDelayedBucketClient(wrapped objstore.Bucket, minDelay, maxDelay time.Duration) objstore.Bucket

Types

type Bucket

type Bucket interface {
	objstore.Bucket
	ReaderAt(ctx context.Context, filename string) (ReaderAtCloser, error)
}

func NewBucket

func NewBucket(bkt objstore.Bucket) Bucket

func NewPrefixedBucket

func NewPrefixedBucket(bkt Bucket, prefix string) Bucket

type BucketReader

type BucketReader interface {
	objstore.BucketReader
	ReaderAt(ctx context.Context, filename string) (ReaderAtCloser, error)
}

type ClientMock added in v1.1.0

type ClientMock struct {
	mock.Mock
}

ClientMock mocks objstore.Bucket

func (*ClientMock) Attributes added in v1.1.0

func (m *ClientMock) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)

ObjectSize mocks objstore.Bucket.Attributes()

func (*ClientMock) Close added in v1.1.0

func (m *ClientMock) Close() error

Close mocks objstore.Bucket.Close()

func (*ClientMock) Delete added in v1.1.0

func (m *ClientMock) Delete(ctx context.Context, name string) error

Delete mocks objstore.Bucket.Delete()

func (*ClientMock) Exists added in v1.1.0

func (m *ClientMock) Exists(ctx context.Context, name string) (bool, error)

Exists mocks objstore.Bucket.Exists()

func (*ClientMock) Get added in v1.1.0

func (m *ClientMock) Get(ctx context.Context, name string) (io.ReadCloser, error)

Get mocks objstore.Bucket.Get()

func (*ClientMock) GetRange added in v1.1.0

func (m *ClientMock) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

GetRange mocks objstore.Bucket.GetRange()

func (*ClientMock) IsAccessDeniedErr added in v1.2.1

func (m *ClientMock) IsAccessDeniedErr(_ error) bool

func (*ClientMock) IsObjNotFoundErr added in v1.1.0

func (m *ClientMock) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr mocks objstore.Bucket.IsObjNotFoundErr()

func (*ClientMock) Iter added in v1.1.0

func (m *ClientMock) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error

Iter mocks objstore.Bucket.Iter()

func (*ClientMock) MockAttributes added in v1.1.0

func (m *ClientMock) MockAttributes(name string, attrs objstore.ObjectAttributes, err error)

func (*ClientMock) MockDelete added in v1.1.0

func (m *ClientMock) MockDelete(name string, err error)

func (*ClientMock) MockExists added in v1.1.0

func (m *ClientMock) MockExists(name string, exists bool, err error)

func (*ClientMock) MockGet added in v1.1.0

func (m *ClientMock) MockGet(name, content string, err error)

MockGet is a convenient method to mock Get() and Exists()

func (*ClientMock) MockGetAndLastModified added in v1.1.0

func (m *ClientMock) MockGetAndLastModified(name, content string, lastModified time.Time, err error)

func (*ClientMock) MockIter added in v1.1.0

func (m *ClientMock) MockIter(prefix string, objects []string, err error)

MockIter is a convenient method to mock Iter()

func (*ClientMock) MockIterWithCallback added in v1.1.0

func (m *ClientMock) MockIterWithCallback(prefix string, objects []string, err error, cb func())

MockIterWithCallback is a convenient method to mock Iter() and get a callback called when the Iter API is called.

func (*ClientMock) MockUpload added in v1.1.0

func (m *ClientMock) MockUpload(name string, err error)

func (*ClientMock) Name added in v1.1.0

func (m *ClientMock) Name() string

Name mocks objstore.Bucket.Name()

func (*ClientMock) ReaderAt added in v1.1.0

func (m *ClientMock) ReaderAt(ctx context.Context, name string) (ReaderAtCloser, error)

func (*ClientMock) Upload added in v1.1.0

func (m *ClientMock) Upload(ctx context.Context, name string, r io.Reader) error

Upload mocks objstore.Bucket.Upload()

type DelayedBucketClient

type DelayedBucketClient struct {
	// contains filtered or unexported fields
}

DelayedBucketClient wraps objstore.InstrumentedBucket and add a random delay to each API call. This client is intended to be used only for testing purposes.

func (*DelayedBucketClient) Attributes

func (*DelayedBucketClient) Close

func (m *DelayedBucketClient) Close() error

func (*DelayedBucketClient) Delete

func (m *DelayedBucketClient) Delete(ctx context.Context, name string) error

func (*DelayedBucketClient) Exists

func (m *DelayedBucketClient) Exists(ctx context.Context, name string) (bool, error)

func (*DelayedBucketClient) Get

func (*DelayedBucketClient) GetRange

func (m *DelayedBucketClient) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

func (*DelayedBucketClient) IsAccessDeniedErr added in v1.2.1

func (m *DelayedBucketClient) IsAccessDeniedErr(err error) bool

IsAccessDeniedErr returns true if acces to object is denied.

func (*DelayedBucketClient) IsObjNotFoundErr

func (m *DelayedBucketClient) IsObjNotFoundErr(err error) bool

func (*DelayedBucketClient) Iter

func (m *DelayedBucketClient) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error

func (*DelayedBucketClient) Name

func (m *DelayedBucketClient) Name() string

func (*DelayedBucketClient) Upload

func (m *DelayedBucketClient) Upload(ctx context.Context, name string, r io.Reader) error

type GetRangeReader

type GetRangeReader interface {
	GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)
}

type InstrumentedBucket added in v1.1.0

type InstrumentedBucket interface {
	Bucket
	// WithExpectedErrs allows to specify a filter that marks certain errors as expected, so it will not increment
	// objstore_bucket_operation_failures_total metric.
	WithExpectedErrs(IsOpFailureExpectedFunc) Bucket

	// ReaderWithExpectedErrs allows to specify a filter that marks certain errors as expected, so it will not increment
	// objstore_bucket_operation_failures_total metric.
	// TODO(bwplotka): Remove this when moved to Go 1.14 and replace with InstrumentedBucketReader.
	ReaderWithExpectedErrs(IsOpFailureExpectedFunc) BucketReader
}

func NewSSEBucketClient

func NewSSEBucketClient(userID string, bucket Bucket, cfgProvider TenantConfigProvider) InstrumentedBucket

NewSSEBucketClient makes a new SSEBucketClient. The cfgProvider can be nil.

func NewTenantBucketClient added in v1.2.0

func NewTenantBucketClient(tenantID string, bucket Bucket, cfgProvider TenantConfigProvider) InstrumentedBucket

NewTenantBucketClient returns a bucket client to use to access the storage on behalf of the provided user. The cfgProvider can be nil.

type IsOpFailureExpectedFunc added in v1.1.0

type IsOpFailureExpectedFunc func(error) bool

IsOpFailureExpectedFunc allows to mark certain errors as expected, so they will not increment objstore_bucket_operation_failures_total metric.

type PrefixedBucket

type PrefixedBucket struct {
	Bucket
	// contains filtered or unexported fields
}

func (PrefixedBucket) Attributes

func (p PrefixedBucket) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)

Attributes returns information about the specified object.

func (*PrefixedBucket) Close

func (p *PrefixedBucket) Close() error

func (*PrefixedBucket) Delete

func (p *PrefixedBucket) Delete(ctx context.Context, name string) error

Delete removes the object with the given name. If object does not exists in the moment of deletion, Delete should throw error.

func (*PrefixedBucket) Exists

func (p *PrefixedBucket) Exists(ctx context.Context, name string) (bool, error)

Exists checks if the given object exists in the bucket.

func (*PrefixedBucket) Get

func (p *PrefixedBucket) Get(ctx context.Context, name string) (io.ReadCloser, error)

Get returns a reader for the given object name.

func (*PrefixedBucket) GetRange

func (p *PrefixedBucket) GetRange(ctx context.Context, name string, off int64, length int64) (io.ReadCloser, error)

GetRange returns a new range reader for the given object name and range.

func (*PrefixedBucket) IsObjNotFoundErr

func (p *PrefixedBucket) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations.

func (*PrefixedBucket) Iter

func (p *PrefixedBucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error

Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full object name including the prefix of the inspected directory. Entries are passed to function in sorted order.

func (*PrefixedBucket) Name

func (p *PrefixedBucket) Name() string

Name returns the bucket name for the provider.

func (*PrefixedBucket) Prefix

func (p *PrefixedBucket) Prefix() string

func (*PrefixedBucket) ReaderAt

func (p *PrefixedBucket) ReaderAt(ctx context.Context, name string) (ReaderAtCloser, error)

func (*PrefixedBucket) ReaderWithExpectedErrs added in v1.1.0

func (p *PrefixedBucket) ReaderWithExpectedErrs(fn IsOpFailureExpectedFunc) BucketReader

ReaderWithExpectedErrs implements objstore.Bucket.

func (*PrefixedBucket) Upload

func (p *PrefixedBucket) Upload(ctx context.Context, name string, r io.Reader) error

Upload the contents of the reader as an object into the bucket. Upload should be idempotent.

func (*PrefixedBucket) WithExpectedErrs added in v1.1.0

func (p *PrefixedBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket

WithExpectedErrs implements objstore.Bucket.

type ReaderAt

type ReaderAt struct {
	GetRangeReader
	// contains filtered or unexported fields
}

func (*ReaderAt) Close

func (b *ReaderAt) Close() error

func (*ReaderAt) ReadAt

func (b *ReaderAt) ReadAt(p []byte, off int64) (int, error)

type ReaderAtBucket

type ReaderAtBucket struct {
	objstore.Bucket
}

func (*ReaderAtBucket) ReaderAt

func (b *ReaderAtBucket) ReaderAt(ctx context.Context, name string) (ReaderAtCloser, error)

func (*ReaderAtBucket) ReaderWithExpectedErrs added in v1.1.0

func (b *ReaderAtBucket) ReaderWithExpectedErrs(fn IsOpFailureExpectedFunc) BucketReader

ReaderWithExpectedErrs implements objstore.Bucket.

func (*ReaderAtBucket) WithExpectedErrs added in v1.1.0

func (b *ReaderAtBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket

WithExpectedErrs implements objstore.Bucket.

type ReaderAtCloser

type ReaderAtCloser interface {
	io.ReaderAt
	io.Closer
}

type SSEBucketClient

type SSEBucketClient struct {
	// contains filtered or unexported fields
}

SSEBucketClient is a wrapper around a objstore.BucketReader that configures the object storage server-side encryption (SSE) for a given user.

func (*SSEBucketClient) Attributes

func (b *SSEBucketClient) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)

Attributes implements objstore.Bucket.

func (*SSEBucketClient) Close

func (b *SSEBucketClient) Close() error

Close implements objstore.Bucket.

func (*SSEBucketClient) Delete

func (b *SSEBucketClient) Delete(ctx context.Context, name string) error

Delete implements objstore.Bucket.

func (*SSEBucketClient) Exists

func (b *SSEBucketClient) Exists(ctx context.Context, name string) (bool, error)

Exists implements objstore.Bucket.

func (*SSEBucketClient) Get

func (b *SSEBucketClient) Get(ctx context.Context, name string) (io.ReadCloser, error)

Get implements objstore.Bucket.

func (*SSEBucketClient) GetRange

func (b *SSEBucketClient) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

GetRange implements objstore.Bucket.

func (*SSEBucketClient) IsAccessDeniedErr added in v1.2.1

func (b *SSEBucketClient) IsAccessDeniedErr(err error) bool

IsAccessDeniedErr returns true if acces to object is denied.

func (*SSEBucketClient) IsObjNotFoundErr

func (b *SSEBucketClient) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr implements objstore.Bucket.

func (*SSEBucketClient) Iter

func (b *SSEBucketClient) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error

Iter implements objstore.Bucket.

func (*SSEBucketClient) Name

func (b *SSEBucketClient) Name() string

Name implements objstore.Bucket.

func (*SSEBucketClient) ReaderAt added in v1.1.0

func (b *SSEBucketClient) ReaderAt(ctx context.Context, name string) (ReaderAtCloser, error)

func (*SSEBucketClient) ReaderWithExpectedErrs

func (b *SSEBucketClient) ReaderWithExpectedErrs(fn IsOpFailureExpectedFunc) BucketReader

ReaderWithExpectedErrs implements objstore.Bucket.

func (*SSEBucketClient) Upload

func (b *SSEBucketClient) Upload(ctx context.Context, name string, r io.Reader) error

Upload the contents of the reader as an object into the bucket.

func (*SSEBucketClient) WithExpectedErrs

func (b *SSEBucketClient) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket

WithExpectedErrs implements objstore.Bucket.

type TenantConfigProvider

type TenantConfigProvider interface {
	// S3SSEType returns the per-tenant S3 SSE type.
	S3SSEType(userID string) string

	// S3SSEKMSKeyID returns the per-tenant S3 KMS-SSE key id or an empty string if not set.
	S3SSEKMSKeyID(userID string) string

	// S3SSEKMSEncryptionContext returns the per-tenant S3 KMS-SSE key id or an empty string if not set.
	S3SSEKMSEncryptionContext(userID string) string
}

TenantConfigProvider defines a per-tenant config provider.

Directories

Path Synopsis
providers
cos
gcs
s3

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL