bucket

package
v0.0.0-...-aa375f8 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 19, 2024 License: AGPL-3.0 Imports: 25 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// S3 is the value for the S3 storage backend.
	S3 = "s3"

	// GCS is the value for the GCS storage backend.
	GCS = "gcs"

	// Azure is the value for the Azure storage backend.
	Azure = "azure"

	// Swift is the value for the Openstack Swift storage backend.
	Swift = "swift"

	// Filesystem is the value for the filesystem storage backend.
	Filesystem = "filesystem"

	// MimirInternalsPrefix is the bucket prefix under which all Mimir internal cluster-wide objects are stored.
	// The object storage path delimiter (/) is appended to this prefix when building the full object path.
	MimirInternalsPrefix = "__mimir_cluster"
)
View Source 
const (
	OpGet = 1 << iota
	OpExists
	OpUpload
	OpDelete
	OpIter
	OpAttributes
)

Variables

View Source 
var (
	SupportedBackends = []string{S3, GCS, Azure, Swift, Filesystem}

	ErrUnsupportedStorageBackend        = errors.New("unsupported storage backend")
	ErrInvalidCharactersInStoragePrefix = errors.New("storage prefix contains invalid characters, it may only contain digits and English alphabet letters")
)
View Source 
var ErrObjectDoesNotExist = errors.New("object does not exist")

ErrObjectDoesNotExist is used in tests to simulate objstore.Bucket.IsObjNotFoundErr().

Functions

func DeletePrefix 

func DeletePrefix(ctx context.Context, bkt objstore.Bucket, prefix string, logger log.Logger) (int, error)

DeletePrefix removes all objects with given prefix, recursively. It returns number of deleted objects. If deletion of any object fails, it returns error and stops.

func InjectErrorOn 

func InjectErrorOn(mask Operation, target string, err error) func(Operation, string) error

InjectErrorOn injects the provided error if the operation matches the provided mask and target

func NewClient 

func NewClient(ctx context.Context, cfg Config, name string, logger log.Logger, reg prometheus.Registerer) (objstore.InstrumentedBucket, error)

NewClient creates a new bucket client based on the configured backend

func NewDelayedBucketClient 

func NewDelayedBucketClient(wrapped objstore.Bucket, minDelay, maxDelay time.Duration) objstore.Bucket

func NewUserBucketClient 

func NewUserBucketClient(userID string, bucket objstore.Bucket, cfgProvider TenantConfigProvider) objstore.InstrumentedBucket

NewUserBucketClient returns a bucket client to use to access the storage on behalf of the provided user. The cfgProvider can be nil.

Types

type ClientMock 

type ClientMock struct {
	mock.Mock
}

ClientMock mocks objstore.Bucket

func (*ClientMock) Attributes 

func (m *ClientMock) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)

ObjectSize mocks objstore.Bucket.Attributes()

func (*ClientMock) Close 

func (m *ClientMock) Close() error

Close mocks objstore.Bucket.Close()

func (*ClientMock) Delete 

func (m *ClientMock) Delete(ctx context.Context, name string) error

Delete mocks objstore.Bucket.Delete()

func (*ClientMock) Exists 

func (m *ClientMock) Exists(ctx context.Context, name string) (bool, error)

Exists mocks objstore.Bucket.Exists()

func (*ClientMock) Get 

func (m *ClientMock) Get(ctx context.Context, name string) (io.ReadCloser, error)

Get mocks objstore.Bucket.Get()

func (*ClientMock) GetRange 

func (m *ClientMock) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

GetRange mocks objstore.Bucket.GetRange()

func (*ClientMock) IsAccessDeniedErr 

func (m *ClientMock) IsAccessDeniedErr(_ error) bool

func (*ClientMock) IsObjNotFoundErr 

func (m *ClientMock) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr mocks objstore.Bucket.IsObjNotFoundErr()

func (*ClientMock) Iter 

func (m *ClientMock) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error

Iter mocks objstore.Bucket.Iter()

func (*ClientMock) IterWithAttributes 

func (m *ClientMock) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error

IterWithAttributes mocks objstore.Bucket.IterWithAttributes().

func (*ClientMock) MockAttributes 

func (m *ClientMock) MockAttributes(name string, attrs objstore.ObjectAttributes, err error)

func (*ClientMock) MockDelete 

func (m *ClientMock) MockDelete(name string, err error)

func (*ClientMock) MockExists 

func (m *ClientMock) MockExists(name string, exists bool, err error)

func (*ClientMock) MockGet 

func (m *ClientMock) MockGet(name, content string, err error)

MockGet is a convenient method to mock Get() and Exists()

func (*ClientMock) MockGetAndLastModified 

func (m *ClientMock) MockGetAndLastModified(name, content string, lastModified time.Time, err error)

func (*ClientMock) MockIter 

func (m *ClientMock) MockIter(prefix string, objects []string, err error)

MockIter is a convenient method to mock Iter()

func (*ClientMock) MockIterWithCallback 

func (m *ClientMock) MockIterWithCallback(prefix string, objects []string, err error, cb func())

MockIterWithCallback is a convenient method to mock Iter() and get a callback called when the Iter API is called.

func (*ClientMock) MockUpload 

func (m *ClientMock) MockUpload(name string, err error)

func (*ClientMock) Name 

func (m *ClientMock) Name() string

Name mocks objstore.Bucket.Name()

func (*ClientMock) Provider 

func (m *ClientMock) Provider() objstore.ObjProvider

func (*ClientMock) SupportedIterOptions 

func (m *ClientMock) SupportedIterOptions() []objstore.IterOptionType

SupportedIterOptions mocks objstore.Bucket.SupportedIterOptions().

func (*ClientMock) Upload 

func (m *ClientMock) Upload(ctx context.Context, name string, r io.Reader) error

Upload mocks objstore.Bucket.Upload()

type Config 

type Config struct {
	StorageBackendConfig `yaml:",inline"`

	StoragePrefix string `yaml:"storage_prefix"`

	// Not used internally, meant to allow callers to wrap Buckets
	// created using this config
	Middlewares []func(objstore.InstrumentedBucket) (objstore.InstrumentedBucket, error) `yaml:"-"`
}

Config holds configuration for accessing long-term storage.

func (*Config) RegisterFlags 

func (cfg *Config) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers the backend storage config.

func (*Config) RegisterFlagsWithPrefix 

func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

func (*Config) RegisterFlagsWithPrefixAndDefaultDirectory 

func (cfg *Config) RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir string, f *flag.FlagSet)

func (*Config) Validate 

func (cfg *Config) Validate() error

type DelayedBucketClient 

type DelayedBucketClient struct {
	// contains filtered or unexported fields
}

DelayedBucketClient wraps objstore.InstrumentedBucket and add a random delay to each API call. This client is intended to be used only for testing purposes.

func (*DelayedBucketClient) Attributes 

func (m *DelayedBucketClient) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)

func (*DelayedBucketClient) Close 

func (m *DelayedBucketClient) Close() error

func (*DelayedBucketClient) Delete 

func (m *DelayedBucketClient) Delete(ctx context.Context, name string) error

func (*DelayedBucketClient) Exists 

func (m *DelayedBucketClient) Exists(ctx context.Context, name string) (bool, error)

func (*DelayedBucketClient) Get 

func (m *DelayedBucketClient) Get(ctx context.Context, name string) (io.ReadCloser, error)

func (*DelayedBucketClient) GetRange 

func (m *DelayedBucketClient) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

func (*DelayedBucketClient) IsAccessDeniedErr 

func (m *DelayedBucketClient) IsAccessDeniedErr(err error) bool

func (*DelayedBucketClient) IsObjNotFoundErr 

func (m *DelayedBucketClient) IsObjNotFoundErr(err error) bool

func (*DelayedBucketClient) Iter 

func (m *DelayedBucketClient) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error

func (*DelayedBucketClient) IterWithAttributes 

func (m *DelayedBucketClient) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error

func (*DelayedBucketClient) Name 

func (m *DelayedBucketClient) Name() string

func (*DelayedBucketClient) Provider 

func (m *DelayedBucketClient) Provider() objstore.ObjProvider

func (*DelayedBucketClient) SupportedIterOptions 

func (m *DelayedBucketClient) SupportedIterOptions() []objstore.IterOptionType

func (*DelayedBucketClient) Upload 

func (m *DelayedBucketClient) Upload(ctx context.Context, name string, r io.Reader) error

type ErrorInjectedBucketClient 

type ErrorInjectedBucketClient struct {
	objstore.Bucket
	Injector func(Operation, string) error
}

ErrorInjectedBucketClient facilitates injecting errors into a bucket client

func (*ErrorInjectedBucketClient) Attributes 

func (b *ErrorInjectedBucketClient) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)

func (*ErrorInjectedBucketClient) Delete 

func (b *ErrorInjectedBucketClient) Delete(ctx context.Context, name string) error

func (*ErrorInjectedBucketClient) Exists 

func (b *ErrorInjectedBucketClient) Exists(ctx context.Context, name string) (bool, error)

func (*ErrorInjectedBucketClient) Get 

func (b *ErrorInjectedBucketClient) Get(ctx context.Context, name string) (io.ReadCloser, error)

func (*ErrorInjectedBucketClient) Iter 

func (b *ErrorInjectedBucketClient) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error

func (*ErrorInjectedBucketClient) Upload 

func (b *ErrorInjectedBucketClient) Upload(ctx context.Context, name string, r io.Reader) error

type Operation 

type Operation uint8

type PrefixedBucketClient 

type PrefixedBucketClient struct {
	// contains filtered or unexported fields
}

func NewPrefixedBucketClient 

func NewPrefixedBucketClient(bucket objstore.Bucket, prefix string) *PrefixedBucketClient

NewPrefixedBucketClient returns a new PrefixedBucketClient.

func (*PrefixedBucketClient) Attributes 

func (b *PrefixedBucketClient) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)

Attributes returns attributes of the specified object.

func (*PrefixedBucketClient) Close 

func (b *PrefixedBucketClient) Close() error

Close implements io.Closer

func (*PrefixedBucketClient) Delete 

func (b *PrefixedBucketClient) Delete(ctx context.Context, name string) error

Delete removes the object with the given name.

func (*PrefixedBucketClient) Exists 

func (b *PrefixedBucketClient) Exists(ctx context.Context, name string) (bool, error)

Exists checks if the given object exists in the bucket.

func (*PrefixedBucketClient) Get 

func (b *PrefixedBucketClient) Get(ctx context.Context, name string) (io.ReadCloser, error)

Get returns a reader for the given object name.

func (*PrefixedBucketClient) GetRange 

func (b *PrefixedBucketClient) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

GetRange returns a new range reader for the given object name and range.

func (*PrefixedBucketClient) IsAccessDeniedErr 

func (b *PrefixedBucketClient) IsAccessDeniedErr(err error) bool

IsAccessDeniedErr returns true if access to an operation is denied

func (*PrefixedBucketClient) IsObjNotFoundErr 

func (b *PrefixedBucketClient) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations.

func (*PrefixedBucketClient) Iter 

func (b *PrefixedBucketClient) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error

Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full object name including the prefix of the inspected directory. The configured prefix will be stripped before supplied function is applied.

func (*PrefixedBucketClient) IterWithAttributes 

func (b *PrefixedBucketClient) IterWithAttributes(ctx context.Context, dir string, f func(objstore.IterObjectAttributes) error, options ...objstore.IterOption) error

IterWithAttributes calls f for each entry in the given directory similar to Iter. In addition to Name, it also includes requested object attributes in the argument to f.

Attributes can be requested using IterOption. Not all IterOptions are supported by all providers, requesting for an unsupported option will fail with ErrOptionNotSupported.

func (*PrefixedBucketClient) Name 

func (b *PrefixedBucketClient) Name() string

Name returns the bucket name for the provider.

func (*PrefixedBucketClient) Provider 

func (b *PrefixedBucketClient) Provider() objstore.ObjProvider

func (*PrefixedBucketClient) ReaderWithExpectedErrs 

func (b *PrefixedBucketClient) ReaderWithExpectedErrs(fn objstore.IsOpFailureExpectedFunc) objstore.BucketReader

ReaderWithExpectedErrs allows to specify a filter that marks certain errors as expected, so it will not increment thanos_objstore_bucket_operation_failures_total metric.

func (*PrefixedBucketClient) SupportedIterOptions 

func (b *PrefixedBucketClient) SupportedIterOptions() []objstore.IterOptionType

SupportedIterOptions returns a list of supported IterOptions by the underlying provider.

func (*PrefixedBucketClient) Upload 

func (b *PrefixedBucketClient) Upload(ctx context.Context, name string, r io.Reader) (err error)

Upload the contents of the reader as an object into the bucket.

func (*PrefixedBucketClient) WithExpectedErrs 

func (b *PrefixedBucketClient) WithExpectedErrs(fn objstore.IsOpFailureExpectedFunc) objstore.Bucket

WithExpectedErrs allows to specify a filter that marks certain errors as expected, so it will not increment thanos_objstore_bucket_operation_failures_total metric.

type SSEBucketClient 

type SSEBucketClient struct {
	// contains filtered or unexported fields
}

SSEBucketClient is a wrapper around a objstore.BucketReader that configures the object storage server-side encryption (SSE) for a given user.

func NewSSEBucketClient 

func NewSSEBucketClient(userID string, bucket objstore.Bucket, cfgProvider TenantConfigProvider) *SSEBucketClient

NewSSEBucketClient makes a new SSEBucketClient. The cfgProvider can be nil.

func (*SSEBucketClient) Attributes 

func (b *SSEBucketClient) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)

Attributes implements objstore.Bucket.

func (*SSEBucketClient) Close 

func (b *SSEBucketClient) Close() error

Close implements objstore.Bucket.

func (*SSEBucketClient) Delete 

func (b *SSEBucketClient) Delete(ctx context.Context, name string) error

Delete implements objstore.Bucket.

func (*SSEBucketClient) Exists 

func (b *SSEBucketClient) Exists(ctx context.Context, name string) (bool, error)

Exists implements objstore.Bucket.

func (*SSEBucketClient) Get 

func (b *SSEBucketClient) Get(ctx context.Context, name string) (io.ReadCloser, error)

Get implements objstore.Bucket.

func (*SSEBucketClient) GetRange 

func (b *SSEBucketClient) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

GetRange implements objstore.Bucket.

func (*SSEBucketClient) IsAccessDeniedErr 

func (b *SSEBucketClient) IsAccessDeniedErr(err error) bool

IsAccessDeniedErr implements objstore.Bucket.

func (*SSEBucketClient) IsObjNotFoundErr 

func (b *SSEBucketClient) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr implements objstore.Bucket.

func (*SSEBucketClient) Iter 

func (b *SSEBucketClient) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error

Iter implements objstore.Bucket.

func (*SSEBucketClient) IterWithAttributes 

func (b *SSEBucketClient) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error

IterWithAttributes implements objstore.Bucket.

func (*SSEBucketClient) Name 

func (b *SSEBucketClient) Name() string

Name implements objstore.Bucket.

func (*SSEBucketClient) Provider 

func (b *SSEBucketClient) Provider() objstore.ObjProvider

func (*SSEBucketClient) ReaderWithExpectedErrs 

func (b *SSEBucketClient) ReaderWithExpectedErrs(fn objstore.IsOpFailureExpectedFunc) objstore.BucketReader

ReaderWithExpectedErrs implements objstore.Bucket.

func (*SSEBucketClient) SupportedIterOptions 

func (b *SSEBucketClient) SupportedIterOptions() []objstore.IterOptionType

SupportedIterOptions implements objstore.Bucket.

func (*SSEBucketClient) Upload 

func (b *SSEBucketClient) Upload(ctx context.Context, name string, r io.Reader) error

Upload the contents of the reader as an object into the bucket.

func (*SSEBucketClient) WithExpectedErrs 

func (b *SSEBucketClient) WithExpectedErrs(fn objstore.IsOpFailureExpectedFunc) objstore.Bucket

WithExpectedErrs implements objstore.Bucket.

type StorageBackendConfig 

type StorageBackendConfig struct {
	Backend string `yaml:"backend"`

	// Backends
	S3         s3.Config         `yaml:"s3"`
	GCS        gcs.Config        `yaml:"gcs"`
	Azure      azure.Config      `yaml:"azure"`
	Swift      swift.Config      `yaml:"swift"`
	Filesystem filesystem.Config `yaml:"filesystem"`

	// Used to inject additional backends into the config. Allows for this config to
	// be embedded in multiple contexts and support non-object storage based backends.
	ExtraBackends []string `yaml:"-"`

	// Used to keep track of the flag names registered in this config, to be able to overwrite them later properly.
	RegisteredFlags util.RegisteredFlags `yaml:"-"`
}

func (*StorageBackendConfig) RegisterFlags 

func (cfg *StorageBackendConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers the backend storage config.

func (*StorageBackendConfig) RegisterFlagsWithPrefix 

func (cfg *StorageBackendConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

func (*StorageBackendConfig) RegisterFlagsWithPrefixAndDefaultDirectory 

func (cfg *StorageBackendConfig) RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir string, f *flag.FlagSet)

func (*StorageBackendConfig) Validate 

func (cfg *StorageBackendConfig) Validate() error

type TenantConfigProvider 

type TenantConfigProvider interface {
	// S3SSEType returns the per-tenant S3 SSE type.
	S3SSEType(userID string) string

	// S3SSEKMSKeyID returns the per-tenant S3 KMS-SSE key id or an empty string if not set.
	S3SSEKMSKeyID(userID string) string

	// S3SSEKMSEncryptionContext returns the per-tenant S3 KMS-SSE key id or an empty string if not set.
	S3SSEKMSEncryptionContext(userID string) string
}

TenantConfigProvider defines a per-tenant config provider.

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.