s3

package
v0.1.1-test Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 26, 2021 License: Apache-2.0 Imports: 32 Imported by: 0

Documentation

Index

Constants

View Source
const (
	DefaultStreamingChunkSize    = 2 << 19         // 1MiB by default per chunk
	DefaultStreamingChunkTimeout = time.Second * 1 // if we haven't read DefaultStreamingChunkSize by this duration, write whatever we have as a chunk

	ExpireObjectS3Tag = "lakefs_expire_object"
)
View Source
const (
	StreamingSha256          = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
	StreamingContentEncoding = "aws-chunked"
)

Variables

View Source
var (
	ErrS3          = errors.New("s3 error")
	ErrMissingETag = fmt.Errorf("%w: missing ETag", ErrS3)
)
View Source
var ErrInventoryFilesRangesOverlap = errors.New("got s3 inventory with files covering overlapping ranges")
View Source
var ErrInventoryNotSorted = errors.New("got unsorted s3 inventory")
View Source
var ErrReaderTimeout = errors.New("reader timeout")

Functions

func GenerateInventory

func GenerateInventory(logger logging.Logger, m *Manifest, inventoryReader s3inventory.IReader, shouldSort bool, prefixes []string) (block.Inventory, error)

func ReadAllWithTimeout

func ReadAllWithTimeout(r io.Reader, buf []byte, timeout time.Duration, minSize int) (n int, err error)

ReadAllWithTimeout is taken from io.ReadAtLeast and adapted to support a timeout and a minimum read size

func WithDiscoverBucketRegion added in v0.55.0

func WithDiscoverBucketRegion(b bool) func(a *Adapter)

func WithHTTPClient

func WithHTTPClient(c *http.Client) func(a *Adapter)

func WithStatsCollector added in v0.48.0

func WithStatsCollector(s stats.Collector) func(a *Adapter)

func WithStreamingChunkSize

func WithStreamingChunkSize(sz int) func(a *Adapter)

func WithStreamingChunkTimeout

func WithStreamingChunkTimeout(d time.Duration) func(a *Adapter)

func WithTranslator

func WithTranslator(t block.UploadIDTranslator) func(a *Adapter)

Types

type Adapter

type Adapter struct {
	// contains filtered or unexported fields
}

func NewAdapter

func NewAdapter(awsSession *session.Session, opts ...func(a *Adapter)) *Adapter

func (*Adapter) AbortMultiPartUpload

func (a *Adapter) AbortMultiPartUpload(ctx context.Context, obj block.ObjectPointer, uploadID string) error

func (*Adapter) BlockstoreType

func (a *Adapter) BlockstoreType() string

func (*Adapter) CompleteMultiPartUpload

func (a *Adapter) CompleteMultiPartUpload(ctx context.Context, obj block.ObjectPointer, uploadID string, multipartList *block.MultipartUploadCompletion) (*block.CompleteMultiPartUploadResponse, error)

func (*Adapter) Copy

func (a *Adapter) Copy(ctx context.Context, sourceObj, destinationObj block.ObjectPointer) error

func (*Adapter) Exists

func (a *Adapter) Exists(ctx context.Context, obj block.ObjectPointer) (bool, error)

func (*Adapter) GenerateInventory

func (a *Adapter) GenerateInventory(ctx context.Context, logger logging.Logger, manifestURL string, shouldSort bool, prefixes []string) (block.Inventory, error)

func (*Adapter) Get

func (*Adapter) GetProperties

func (a *Adapter) GetProperties(ctx context.Context, obj block.ObjectPointer) (block.Properties, error)

func (*Adapter) GetRange

func (a *Adapter) GetRange(ctx context.Context, obj block.ObjectPointer, startPosition int64, endPosition int64) (io.ReadCloser, error)

func (*Adapter) GetStorageNamespaceInfo

func (a *Adapter) GetStorageNamespaceInfo() block.StorageNamespaceInfo

func (*Adapter) Put

func (a *Adapter) Put(ctx context.Context, obj block.ObjectPointer, sizeBytes int64, reader io.Reader, opts block.PutOpts) error

func (*Adapter) Remove

func (a *Adapter) Remove(ctx context.Context, obj block.ObjectPointer) error

func (*Adapter) RuntimeStats added in v0.41.1

func (a *Adapter) RuntimeStats() map[string]string

func (*Adapter) UploadCopyPart

func (a *Adapter) UploadCopyPart(ctx context.Context, sourceObj, destinationObj block.ObjectPointer, uploadID string, partNumber int) (*block.UploadPartResponse, error)

func (*Adapter) UploadCopyPartRange

func (a *Adapter) UploadCopyPartRange(ctx context.Context, sourceObj, destinationObj block.ObjectPointer, uploadID string, partNumber int, startPosition, endPosition int64) (*block.UploadPartResponse, error)

func (*Adapter) UploadPart

func (a *Adapter) UploadPart(ctx context.Context, obj block.ObjectPointer, sizeBytes int64, reader io.Reader, uploadID string, partNumber int) (*block.UploadPartResponse, error)

func (*Adapter) Walk

func (a *Adapter) Walk(ctx context.Context, walkOpt block.WalkOpts, walkFn block.WalkFunc) error

type ClientCache added in v0.48.0

type ClientCache struct {
	// contains filtered or unexported fields
}

func NewClientCache added in v0.48.0

func NewClientCache(awsSession *session.Session) *ClientCache

func (*ClientCache) DiscoverBucketRegion added in v0.55.0

func (c *ClientCache) DiscoverBucketRegion(b bool)

func (*ClientCache) Get added in v0.48.0

func (c *ClientCache) Get(ctx context.Context, bucket string) s3iface.S3API

Get returns an AWS client configured to the region of the given bucket.

func (*ClientCache) SetClientFactory added in v0.55.0

func (c *ClientCache) SetClientFactory(clientFactory clientFactory)

func (*ClientCache) SetS3RegionGetter added in v0.55.0

func (c *ClientCache) SetS3RegionGetter(s3RegionGetter s3RegionGetter)

func (*ClientCache) SetStatsCollector added in v0.55.0

func (c *ClientCache) SetStatsCollector(statsCollector stats.Collector)

type Inventory

type Inventory struct {
	Manifest *Manifest
	// contains filtered or unexported fields
}

func (*Inventory) InventoryURL

func (inv *Inventory) InventoryURL() string

func (*Inventory) Iterator

func (inv *Inventory) Iterator() block.InventoryIterator

func (*Inventory) SourceName

func (inv *Inventory) SourceName() string

type InventoryFile added in v0.48.0

type InventoryFile struct {
	Key string `json:"key"` // an s3 key for an inventory list file
	// contains filtered or unexported fields
}

type InventoryIterator

type InventoryIterator struct {
	*Inventory
	// contains filtered or unexported fields
}

func NewInventoryIterator

func NewInventoryIterator(inv *Inventory) *InventoryIterator

func (*InventoryIterator) Err

func (it *InventoryIterator) Err() error

func (*InventoryIterator) Get

func (*InventoryIterator) Next

func (it *InventoryIterator) Next() bool

func (*InventoryIterator) Progress

func (it *InventoryIterator) Progress() []*cmdutils.Progress

type Manifest

type Manifest struct {
	URL                string          `json:"-"`
	InventoryBucketArn string          `json:"destinationBucket"`
	SourceBucket       string          `json:"sourceBucket"`
	Files              []InventoryFile `json:"files"` // inventory list files, each contains a list of objects
	Format             string          `json:"fileFormat"`
	CreationTimestamp  string          `json:"creationTimestamp"`
	// contains filtered or unexported fields
}

type StreamingReader

type StreamingReader struct {
	Reader       io.Reader
	Size         int64
	StreamSigner *v4.StreamSigner
	Time         time.Time
	ChunkSize    int
	ChunkTimeout time.Duration
	// contains filtered or unexported fields
}

func (*StreamingReader) GetLastChunk

func (s *StreamingReader) GetLastChunk() []byte

func (*StreamingReader) Read

func (s *StreamingReader) Read(p []byte) (int, error)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL