s3

Documentation

Overview

Package s3 implements common object storage abstractions against s3-compatible APIs.

Index

• Constants
• Variables
• func ContextWithSSEConfig(ctx context.Context, value encrypt.ServerSide) context.Context
• func NewAWSSDKAuth(region string) *credentials.Credentials
• func NewTestBucket(t testing.TB, location string) (objstore.Bucket, func(), error)
• func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucket bool) (objstore.Bucket, func(), error)
• func ValidateForTests(conf Config) error
• type AWSSDKAuth
  • func (a *AWSSDKAuth) IsExpired() bool
  • func (a *AWSSDKAuth) Retrieve() (credentials.Value, error)
• type Bucket
  • func NewBucket(logger log.Logger, conf []byte, component string, ...) (*Bucket, error)
  • func NewBucketWithConfig(logger log.Logger, config Config, component string, ...) (*Bucket, error)
  • func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)
  • func (b *Bucket) Close() error
  • func (b *Bucket) Delete(ctx context.Context, name string) error
  • func (b *Bucket) Exists(ctx context.Context, name string) (bool, error)
  • func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error)
  • func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)
  • func (b *Bucket) IsAccessDeniedErr(err error) bool
  • func (b *Bucket) IsObjNotFoundErr(err error) bool
  • func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, ...) error
  • func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, ...) error
  • func (b *Bucket) Name() string
  • func (b *Bucket) Provider() objstore.ObjProvider
  • func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType
  • func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error
• type BucketLookupType
  • func (blt BucketLookupType) MarshalYAML() (interface{}, error)
  • func (blt BucketLookupType) MinioType() minio.BucketLookupType
  • func (blt BucketLookupType) String() string
  • func (blt *BucketLookupType) UnmarshalYAML(unmarshal func(interface{}) error) error
• type Config
• type HTTPConfig
• type SSEConfig
• type TraceConfig

Constants

const (
	AutoLookup BucketLookupType = iota
	VirtualHostLookup
	PathLookup

	// DirDelim is the delimiter used to model a directory structure in an object store bucket.
	DirDelim = "/"

	// SSEKMS is the name of the SSE-KMS method for objectstore encryption.
	SSEKMS = "SSE-KMS"

	// SSEC is the name of the SSE-C method for objstore encryption.
	SSEC = "SSE-C"

	// SSES3 is the name of the SSE-S3 method for objstore encryption.
	SSES3 = "SSE-S3"
)

Variables

var DefaultConfig = Config{
	PutUserMetadata:  map[string]string{},
	HTTPConfig:       exthttp.DefaultHTTPConfig,
	DisableMultipart: false,
	PartSize:         1024 * 1024 * 64,
	BucketLookupType: AutoLookup,
	SendContentMd5:   true,
}

Functions

func ContextWithSSEConfig

func ContextWithSSEConfig(ctx context.Context, value encrypt.ServerSide) context.Context

ContextWithSSEConfig returns a context with a custom SSE config set. The returned context should be provided to S3 objstore client functions to override the default SSE config.

func NewAWSSDKAuth

func NewAWSSDKAuth(region string) *credentials.Credentials

NewAWSSDKAuth returns a pointer to a new Credentials object wrapping the environment variable provider.

func NewTestBucket

func NewTestBucket(t testing.TB, location string) (objstore.Bucket, func(), error)

NewTestBucket creates test bkt client that before returning creates temporary bucket. In a close function it empties and deletes the bucket.

func NewTestBucketFromConfig

func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucket bool) (objstore.Bucket, func(), error)

func ValidateForTests

func ValidateForTests(conf Config) error

ValidateForTests checks to see the config options for tests are set.

Types

type AWSSDKAuth

type AWSSDKAuth struct {
	Region string
	// contains filtered or unexported fields
}

AWSSDKAuth retrieves credentials from the aws-sdk-go.

func (*AWSSDKAuth) IsExpired

func (a *AWSSDKAuth) IsExpired() bool

IsExpired returns if the credentials have been retrieved.

func (*AWSSDKAuth) Retrieve

func (a *AWSSDKAuth) Retrieve() (credentials.Value, error)

Retrieve retrieves the keys from the environment.

type Bucket

type Bucket struct {
	// contains filtered or unexported fields
}

Bucket implements the store.Bucket interface against s3-compatible APIs.

func NewBucket

func NewBucket(logger log.Logger, conf []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error)

NewBucket returns a new Bucket using the provided s3 config values.

func NewBucketWithConfig

func NewBucketWithConfig(logger log.Logger, config Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error)

NewBucketWithConfig returns a new Bucket using the provided s3 config values.

func (*Bucket) Attributes

func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)

Attributes returns information about the specified object.

func (*Bucket) Close

func (b *Bucket) Close() error

func (*Bucket) Delete

func (b *Bucket) Delete(ctx context.Context, name string) error

Delete removes the object with the given name.

func (*Bucket) Exists

func (b *Bucket) Exists(ctx context.Context, name string) (bool, error)

Exists checks if the given object exists.

func (*Bucket) Get

func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error)

Get returns a reader for the given object name.

func (*Bucket) GetRange

func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

GetRange returns a new range reader for the given object name and range.

func (*Bucket) IsAccessDeniedErr

func (b *Bucket) IsAccessDeniedErr(err error) bool

IsAccessDeniedErr returns true if access to object is denied.

func (*Bucket) IsObjNotFoundErr

func (b *Bucket) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations.

func (*Bucket) Iter

func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opts ...objstore.IterOption) error

func (*Bucket) IterWithAttributes

func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error

func (*Bucket) Name

func (b *Bucket) Name() string

Name returns the bucket name for s3.

func (*Bucket) Provider

func (b *Bucket) Provider() objstore.ObjProvider

func (*Bucket) SupportedIterOptions

func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType

func (*Bucket) Upload

func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error

Upload the contents of the reader as an object into the bucket.

type BucketLookupType

type BucketLookupType int

func (BucketLookupType) MarshalYAML

func (blt BucketLookupType) MarshalYAML() (interface{}, error)

func (BucketLookupType) MinioType

func (blt BucketLookupType) MinioType() minio.BucketLookupType

func (BucketLookupType) String

func (blt BucketLookupType) String() string

func (*BucketLookupType) UnmarshalYAML

func (blt *BucketLookupType) UnmarshalYAML(unmarshal func(interface{}) error) error

type Config

type Config struct {
	Bucket             string             `yaml:"bucket"`
	Endpoint           string             `yaml:"endpoint"`
	Region             string             `yaml:"region"`
	DisableDualstack   bool               `yaml:"disable_dualstack"`
	AWSSDKAuth         bool               `yaml:"aws_sdk_auth"`
	AccessKey          string             `yaml:"access_key"`
	Insecure           bool               `yaml:"insecure"`
	SignatureV2        bool               `yaml:"signature_version2"`
	SecretKey          string             `yaml:"secret_key"`
	SessionToken       string             `yaml:"session_token"`
	PutUserMetadata    map[string]string  `yaml:"put_user_metadata"`
	HTTPConfig         exthttp.HTTPConfig `yaml:"http_config"`
	TraceConfig        TraceConfig        `yaml:"trace"`
	ListObjectsVersion string             `yaml:"list_objects_version"`
	BucketLookupType   BucketLookupType   `yaml:"bucket_lookup_type"`
	SendContentMd5     bool               `yaml:"send_content_md5"`
	DisableMultipart   bool               `yaml:"disable_multipart"`
	// PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize.
	// NOTE we need to make sure this number does not produce more parts than 10 000.
	PartSize    uint64    `yaml:"part_size"`
	SSEConfig   SSEConfig `yaml:"sse_config"`
	STSEndpoint string    `yaml:"sts_endpoint"`
	MaxRetries  int       `yaml:"max_retries"`
}

Config stores the configuration for s3 bucket.

type HTTPConfig

type HTTPConfig = exthttp.HTTPConfig

HTTPConfig exists here only because Cortex depends on it, and we depend on Cortex. Deprecated. TODO(bwplotka): Remove it, once we remove Cortex cycle dep, or Cortex stops using this.

type SSEConfig

type SSEConfig struct {
	Type                 string            `yaml:"type"`
	KMSKeyID             string            `yaml:"kms_key_id"`
	KMSEncryptionContext map[string]string `yaml:"kms_encryption_context"`
	EncryptionKey        string            `yaml:"encryption_key"`
}

SSEConfig deals with the configuration of SSE for Minio. The following options are valid: KMSEncryptionContext == https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html#s3-encryption-context

type TraceConfig

type TraceConfig struct {
	Enable bool `yaml:"enable"`
}