s3

Documentation

Overview

Package s3 implements common object storage abstractions against s3-compatible APIs.

Index

• Constants
• Variables
• func NewTestBucket(t testing.TB, location string) (objstore.Bucket, func(), error)
• func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucket bool) (objstore.Bucket, func(), error)
• func ValidateForTests(conf Config) error
• type Bucket
  • func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error)
  • func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error)
  • func (b *Bucket) Close() error
  • func (b *Bucket) Delete(_ context.Context, name string) error
  • func (b *Bucket) Exists(_ context.Context, name string) (bool, error)
  • func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error)
  • func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)
  • func (b *Bucket) IsObjNotFoundErr(err error) bool
  • func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error) error
  • func (b *Bucket) Name() string
  • func (b *Bucket) ObjectSize(_ context.Context, name string) (uint64, error)
  • func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error
• type Config
• type HTTPConfig
• type TraceConfig

Constants

const DirDelim = "/"

DirDelim is the delimiter used to model a directory structure in an object store bucket.

Variables

var DefaultConfig = Config{
	PutUserMetadata: map[string]string{},
	HTTPConfig: HTTPConfig{
		IdleConnTimeout:       model.Duration(90 * time.Second),
		ResponseHeaderTimeout: model.Duration(2 * time.Minute),
	},

	PartSize: 1024 * 1024 * 128,
}

Functions

func NewTestBucket

func NewTestBucket(t testing.TB, location string) (objstore.Bucket, func(), error)

NewTestBucket creates test bkt client that before returning creates temporary bucket. In a close function it empties and deletes the bucket.

func NewTestBucketFromConfig

func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucket bool) (objstore.Bucket, func(), error)

func ValidateForTests

func ValidateForTests(conf Config) error

ValidateForTests checks to see the config options for tests are set.

Types

type Bucket

type Bucket struct {
	// contains filtered or unexported fields
}

Bucket implements the store.Bucket interface against s3-compatible APIs.

func NewBucket

func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error)

NewBucket returns a new Bucket using the provided s3 config values.

func NewBucketWithConfig

func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error)

NewBucketWithConfig returns a new Bucket using the provided s3 config values.

func (*Bucket) Close

func (b *Bucket) Close() error

func (*Bucket) Delete

func (b *Bucket) Delete(_ context.Context, name string) error

Delete removes the object with the given name.

func (*Bucket) Exists

func (b *Bucket) Exists(_ context.Context, name string) (bool, error)

Exists checks if the given object exists.

func (*Bucket) Get

func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error)

Get returns a reader for the given object name.

func (*Bucket) GetRange

func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

GetRange returns a new range reader for the given object name and range.

func (*Bucket) IsObjNotFoundErr

func (b *Bucket) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations.

func (*Bucket) Iter

func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error) error

Iter calls f for each entry in the given directory. The argument to f is the full object name including the prefix of the inspected directory.

func (*Bucket) Name

func (b *Bucket) Name() string

Name returns the bucket name for s3.

func (*Bucket) ObjectSize

func (b *Bucket) ObjectSize(_ context.Context, name string) (uint64, error)

ObjectSize returns the size of the specified object.

func (*Bucket) Upload

func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error

Upload the contents of the reader as an object into the bucket.

type Config

type Config struct {
	Bucket          string            `yaml:"bucket"`
	Endpoint        string            `yaml:"endpoint"`
	Region          string            `yaml:"region"`
	AccessKey       string            `yaml:"access_key"`
	Insecure        bool              `yaml:"insecure"`
	SignatureV2     bool              `yaml:"signature_version2"`
	SSEEncryption   bool              `yaml:"encrypt_sse"`
	SecretKey       string            `yaml:"secret_key"`
	PutUserMetadata map[string]string `yaml:"put_user_metadata"`
	HTTPConfig      HTTPConfig        `yaml:"http_config"`
	TraceConfig     TraceConfig       `yaml:"trace"`
	// PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize.
	PartSize uint64 `yaml:"part_size"`
}

Config stores the configuration for s3 bucket.

type HTTPConfig

type HTTPConfig struct {
	IdleConnTimeout       model.Duration `yaml:"idle_conn_timeout"`
	ResponseHeaderTimeout model.Duration `yaml:"response_header_timeout"`
	InsecureSkipVerify    bool           `yaml:"insecure_skip_verify"`
}

HTTPConfig stores the http.Transport configuration for the s3 minio client.

type TraceConfig

type TraceConfig struct {
	Enable bool `yaml:"enable"`
}