s3

Documentation

Overview

Package s3 implements common object storage abstractions against s3-compatible APIs.

Index

• Constants
• Variables
• func ContextWithSSEConfig(ctx context.Context, value encrypt.ServerSide) context.Context
• func DefaultTransport(config Config) (*http.Transport, error)
• func NewAWSSDKAuth(region string) *credentials.Credentials
• func NewTestBucket(t testing.TB, location string) (objstore.Bucket, func(), error)
• func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucket bool) (objstore.Bucket, func(), error)
• func ValidateForTests(conf Config) error
• type AWSSDKAuth
  • func (a *AWSSDKAuth) IsExpired() bool
  • func (a *AWSSDKAuth) Retrieve() (credentials.Value, error)
• type Bucket
  • func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error)
  • func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error)
  • func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)
  • func (b *Bucket) Close() error
  • func (b *Bucket) Delete(ctx context.Context, name string) error
  • func (b *Bucket) Exists(ctx context.Context, name string) (bool, error)
  • func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error)
  • func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)
  • func (b *Bucket) IsObjNotFoundErr(err error) bool
  • func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, ...) error
  • func (b *Bucket) Name() string
  • func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error
• type BucketLookupType
  • func (blt BucketLookupType) MarshalYAML() (interface{}, error)
  • func (blt BucketLookupType) MinioType() minio.BucketLookupType
  • func (blt BucketLookupType) String() string
  • func (blt *BucketLookupType) UnmarshalYAML(unmarshal func(interface{}) error) error
• type Config
• type HTTPConfig
• type SSEConfig
• type TraceConfig

Constants

const (
	AutoLookup BucketLookupType = iota
	VirtualHostLookup
	PathLookup

	// DirDelim is the delimiter used to model a directory structure in an object store bucket.
	DirDelim = "/"

	// SSEKMS is the name of the SSE-KMS method for objectstore encryption.
	SSEKMS = "SSE-KMS"

	// SSEC is the name of the SSE-C method for objstore encryption.
	SSEC = "SSE-C"

	// SSES3 is the name of the SSE-S3 method for objstore encryption.
	SSES3 = "SSE-S3"
)

Variables

var DefaultConfig = Config{
	PutUserMetadata: map[string]string{},
	HTTPConfig: HTTPConfig{
		IdleConnTimeout:       model.Duration(90 * time.Second),
		ResponseHeaderTimeout: model.Duration(2 * time.Minute),
		TLSHandshakeTimeout:   model.Duration(10 * time.Second),
		ExpectContinueTimeout: model.Duration(1 * time.Second),
		MaxIdleConns:          100,
		MaxIdleConnsPerHost:   100,
		MaxConnsPerHost:       0,
	},
	PartSize:         1024 * 1024 * 64,
	BucketLookupType: AutoLookup,
}

Functions

func ContextWithSSEConfig

func ContextWithSSEConfig(ctx context.Context, value encrypt.ServerSide) context.Context

ContextWithSSEConfig returns a context with a custom SSE config set. The returned context should be provided to S3 objstore client functions to override the default SSE config.

func DefaultTransport

func DefaultTransport(config Config) (*http.Transport, error)

DefaultTransport - this default transport is based on the Minio DefaultTransport up until the following commit: https://github.com/minio/minio-go/commit/008c7aa71fc17e11bf980c209a4f8c4d687fc884 The values have since diverged.

func NewAWSSDKAuth

func NewAWSSDKAuth(region string) *credentials.Credentials

NewAWSSDKAuth returns a pointer to a new Credentials object wrapping the environment variable provider.

func NewTestBucket

func NewTestBucket(t testing.TB, location string) (objstore.Bucket, func(), error)

NewTestBucket creates test bkt client that before returning creates temporary bucket. In a close function it empties and deletes the bucket.

func NewTestBucketFromConfig

func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucket bool) (objstore.Bucket, func(), error)

func ValidateForTests

func ValidateForTests(conf Config) error

ValidateForTests checks to see the config options for tests are set.

Types

type AWSSDKAuth

type AWSSDKAuth struct {
	Region string
	// contains filtered or unexported fields
}

AWSSDKAuth retrieves credentials from the aws-sdk-go.

func (*AWSSDKAuth) IsExpired

func (a *AWSSDKAuth) IsExpired() bool

IsExpired returns if the credentials have been retrieved.

func (*AWSSDKAuth) Retrieve

func (a *AWSSDKAuth) Retrieve() (credentials.Value, error)

Retrieve retrieves the keys from the environment.

type Bucket

type Bucket struct {
	// contains filtered or unexported fields
}

Bucket implements the store.Bucket interface against s3-compatible APIs.

func NewBucket

func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error)

NewBucket returns a new Bucket using the provided s3 config values.

func NewBucketWithConfig

func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error)

NewBucketWithConfig returns a new Bucket using the provided s3 config values.

func (*Bucket) Attributes

func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)

Attributes returns information about the specified object.

func (*Bucket) Close

func (b *Bucket) Close() error

func (*Bucket) Delete

func (b *Bucket) Delete(ctx context.Context, name string) error

Delete removes the object with the given name.

func (*Bucket) Exists

func (b *Bucket) Exists(ctx context.Context, name string) (bool, error)

Exists checks if the given object exists.

func (*Bucket) Get

func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error)

Get returns a reader for the given object name.

func (*Bucket) GetRange

func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

GetRange returns a new range reader for the given object name and range.

func (*Bucket) IsObjNotFoundErr

func (b *Bucket) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations.

func (*Bucket) Iter

func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error

Iter calls f for each entry in the given directory. The argument to f is the full object name including the prefix of the inspected directory.

func (*Bucket) Name

func (b *Bucket) Name() string

Name returns the bucket name for s3.

func (*Bucket) Upload

func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error

Upload the contents of the reader as an object into the bucket.

type BucketLookupType

type BucketLookupType int

func (BucketLookupType) MarshalYAML

func (blt BucketLookupType) MarshalYAML() (interface{}, error)

func (BucketLookupType) MinioType

func (blt BucketLookupType) MinioType() minio.BucketLookupType

func (BucketLookupType) String

func (blt BucketLookupType) String() string

func (*BucketLookupType) UnmarshalYAML

func (blt *BucketLookupType) UnmarshalYAML(unmarshal func(interface{}) error) error

type Config

type Config struct {
	Bucket             string            `yaml:"bucket"`
	Endpoint           string            `yaml:"endpoint"`
	Region             string            `yaml:"region"`
	AWSSDKAuth         bool              `yaml:"aws_sdk_auth"`
	AccessKey          string            `yaml:"access_key"`
	Insecure           bool              `yaml:"insecure"`
	SignatureV2        bool              `yaml:"signature_version2"`
	SecretKey          string            `yaml:"secret_key"`
	PutUserMetadata    map[string]string `yaml:"put_user_metadata"`
	HTTPConfig         HTTPConfig        `yaml:"http_config"`
	TraceConfig        TraceConfig       `yaml:"trace"`
	ListObjectsVersion string            `yaml:"list_objects_version"`
	BucketLookupType   BucketLookupType  `yaml:"bucket_lookup_type"`
	// PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize.
	// NOTE we need to make sure this number does not produce more parts than 10 000.
	PartSize    uint64    `yaml:"part_size"`
	SSEConfig   SSEConfig `yaml:"sse_config"`
	STSEndpoint string    `yaml:"sts_endpoint"`
}

Config stores the configuration for s3 bucket.

type HTTPConfig

type HTTPConfig struct {
	IdleConnTimeout       model.Duration `yaml:"idle_conn_timeout"`
	ResponseHeaderTimeout model.Duration `yaml:"response_header_timeout"`
	InsecureSkipVerify    bool           `yaml:"insecure_skip_verify"`

	TLSHandshakeTimeout   model.Duration `yaml:"tls_handshake_timeout"`
	ExpectContinueTimeout model.Duration `yaml:"expect_continue_timeout"`
	MaxIdleConns          int            `yaml:"max_idle_conns"`
	MaxIdleConnsPerHost   int            `yaml:"max_idle_conns_per_host"`
	MaxConnsPerHost       int            `yaml:"max_conns_per_host"`

	// Allow upstream callers to inject a round tripper
	Transport http.RoundTripper `yaml:"-"`

	TLSConfig objstore.TLSConfig `yaml:"tls_config"`
}

HTTPConfig stores the http.Transport configuration for the s3 minio client.

type SSEConfig

type SSEConfig struct {
	Type                 string            `yaml:"type"`
	KMSKeyID             string            `yaml:"kms_key_id"`
	KMSEncryptionContext map[string]string `yaml:"kms_encryption_context"`
	EncryptionKey        string            `yaml:"encryption_key"`
}

SSEConfig deals with the configuration of SSE for Minio. The following options are valid: kmsencryptioncontext == https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html#s3-encryption-context

type TraceConfig

type TraceConfig struct {
	Enable bool `yaml:"enable"`
}