s3

package
v0.54.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 8, 2021 License: Apache-2.0 Imports: 32 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	DefaultStreamingChunkSize    = 2 << 19         // 1MiB by default per chunk
	DefaultStreamingChunkTimeout = time.Second * 1 // if we haven't read DefaultStreamingChunkSize by this duration, write whatever we have as a chunk

	ExpireObjectS3Tag = "lakefs_expire_object"
)
View Source 
const (
	StreamingSha256          = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
	StreamingContentEncoding = "aws-chunked"
)

Variables

View Source 
var (
	ErrS3          = errors.New("s3 error")
	ErrMissingETag = fmt.Errorf("%w: missing ETag", ErrS3)
)
View Source 
var ErrInventoryFilesRangesOverlap = errors.New("got s3 inventory with files covering overlapping ranges")
View Source 
var ErrInventoryNotSorted = errors.New("got unsorted s3 inventory")
View Source 
var ErrReaderTimeout = errors.New("reader timeout")

Functions

func GenerateInventory 

func GenerateInventory(logger logging.Logger, m *Manifest, inventoryReader s3inventory.IReader, shouldSort bool, prefixes []string) (block.Inventory, error)

func ReadAllWithTimeout 

func ReadAllWithTimeout(r io.Reader, buf []byte, timeout time.Duration, minSize int) (n int, err error)

ReadAllWithTimeout is taken from io.ReadAtLeast and adapted to support a timeout and a minimum read size

func WithHTTPClient 

func WithHTTPClient(c *http.Client) func(a *Adapter)

func WithStatsCollector added in v0.48.0 

func WithStatsCollector(s stats.Collector) func(a *Adapter)

func WithStreamingChunkSize 

func WithStreamingChunkSize(sz int) func(a *Adapter)

func WithStreamingChunkTimeout 

func WithStreamingChunkTimeout(d time.Duration) func(a *Adapter)

func WithTranslator 

func WithTranslator(t block.UploadIDTranslator) func(a *Adapter)

Types

type Adapter 

type Adapter struct {
	// contains filtered or unexported fields
}

func NewAdapter 

func NewAdapter(awsSession *session.Session, opts ...func(a *Adapter)) *Adapter

func (*Adapter) AbortMultiPartUpload 

func (a *Adapter) AbortMultiPartUpload(ctx context.Context, obj block.ObjectPointer, uploadID string) error

func (*Adapter) BlockstoreType 

func (a *Adapter) BlockstoreType() string

func (*Adapter) CompleteMultiPartUpload 

func (a *Adapter) CompleteMultiPartUpload(ctx context.Context, obj block.ObjectPointer, uploadID string, multipartList *block.MultipartUploadCompletion) (*block.CompleteMultiPartUploadResponse, error)

func (*Adapter) Copy 

func (a *Adapter) Copy(ctx context.Context, sourceObj, destinationObj block.ObjectPointer) error

func (*Adapter) CreateMultiPartUpload 

func (a *Adapter) CreateMultiPartUpload(ctx context.Context, obj block.ObjectPointer, r *http.Request, opts block.CreateMultiPartUploadOpts) (*block.CreateMultiPartUploadResponse, error)

func (*Adapter) Exists 

func (a *Adapter) Exists(ctx context.Context, obj block.ObjectPointer) (bool, error)

func (*Adapter) GenerateInventory 

func (a *Adapter) GenerateInventory(ctx context.Context, logger logging.Logger, manifestURL string, shouldSort bool, prefixes []string) (block.Inventory, error)

func (*Adapter) Get 

func (a *Adapter) Get(ctx context.Context, obj block.ObjectPointer, _ int64) (io.ReadCloser, error)

func (*Adapter) GetProperties 

func (a *Adapter) GetProperties(ctx context.Context, obj block.ObjectPointer) (block.Properties, error)

func (*Adapter) GetRange 

func (a *Adapter) GetRange(ctx context.Context, obj block.ObjectPointer, startPosition int64, endPosition int64) (io.ReadCloser, error)

func (*Adapter) GetStorageNamespaceInfo 

func (a *Adapter) GetStorageNamespaceInfo() block.StorageNamespaceInfo

func (*Adapter) Put 

func (a *Adapter) Put(ctx context.Context, obj block.ObjectPointer, sizeBytes int64, reader io.Reader, opts block.PutOpts) error

func (*Adapter) Remove 

func (a *Adapter) Remove(ctx context.Context, obj block.ObjectPointer) error

func (*Adapter) RuntimeStats added in v0.41.1 

func (a *Adapter) RuntimeStats() map[string]string

func (*Adapter) UploadCopyPart 

func (a *Adapter) UploadCopyPart(ctx context.Context, sourceObj, destinationObj block.ObjectPointer, uploadID string, partNumber int64) (*block.UploadPartResponse, error)

func (*Adapter) UploadCopyPartRange 

func (a *Adapter) UploadCopyPartRange(ctx context.Context, sourceObj, destinationObj block.ObjectPointer, uploadID string, partNumber, startPosition, endPosition int64) (*block.UploadPartResponse, error)

func (*Adapter) UploadPart 

func (a *Adapter) UploadPart(ctx context.Context, obj block.ObjectPointer, sizeBytes int64, reader io.Reader, uploadID string, partNumber int64) (*block.UploadPartResponse, error)

func (*Adapter) ValidateConfiguration 

func (a *Adapter) ValidateConfiguration(ctx context.Context, storageNamespace string) error

ValidateConfiguration on an S3 adapter checks for a usable bucket lifecycle policy: the storageNamespace bucket should expire objects marked with ExpireObjectS3Tag (with _some_ duration, even if nonzero).

func (*Adapter) Walk 

func (a *Adapter) Walk(ctx context.Context, walkOpt block.WalkOpts, walkFn block.WalkFunc) error

type ClientCache added in v0.48.0 

type ClientCache struct {
	// contains filtered or unexported fields
}

func NewClientCache added in v0.48.0 

func NewClientCache(awsSession *session.Session) *ClientCache

func (*ClientCache) Get added in v0.48.0 

func (c *ClientCache) Get(ctx context.Context, bucket string) s3iface.S3API

Get returns an AWS client configured to the region of the given bucket.

func (*ClientCache) WithClientFactory added in v0.48.0 

func (c *ClientCache) WithClientFactory(clientFactory clientFactory) *ClientCache

func (*ClientCache) WithS3RegionGetter added in v0.48.0 

func (c *ClientCache) WithS3RegionGetter(s3RegionGetter s3RegionGetter) *ClientCache

func (*ClientCache) WithStatsCollector added in v0.48.0 

func (c *ClientCache) WithStatsCollector(statsCollector stats.Collector) *ClientCache

type Inventory 

type Inventory struct {
	Manifest *Manifest
	// contains filtered or unexported fields
}

func (*Inventory) InventoryURL 

func (inv *Inventory) InventoryURL() string

func (*Inventory) Iterator 

func (inv *Inventory) Iterator() block.InventoryIterator

func (*Inventory) SourceName 

func (inv *Inventory) SourceName() string

type InventoryFile added in v0.48.0 

type InventoryFile struct {
	Key string `json:"key"` // an s3 key for an inventory list file
	// contains filtered or unexported fields
}

type InventoryIterator 

type InventoryIterator struct {
	*Inventory
	// contains filtered or unexported fields
}

func NewInventoryIterator 

func NewInventoryIterator(inv *Inventory) *InventoryIterator

func (*InventoryIterator) Err 

func (it *InventoryIterator) Err() error

func (*InventoryIterator) Get 

func (it *InventoryIterator) Get() *block.InventoryObject

func (*InventoryIterator) Next 

func (it *InventoryIterator) Next() bool

func (*InventoryIterator) Progress 

func (it *InventoryIterator) Progress() []*cmdutils.Progress

type Manifest 

type Manifest struct {
	URL                string          `json:"-"`
	InventoryBucketArn string          `json:"destinationBucket"`
	SourceBucket       string          `json:"sourceBucket"`
	Files              []InventoryFile `json:"files"` // inventory list files, each contains a list of objects
	Format             string          `json:"fileFormat"`
	CreationTimestamp  string          `json:"creationTimestamp"`
	// contains filtered or unexported fields
}

type StreamingReader 

type StreamingReader struct {
	Reader       io.Reader
	Size         int
	StreamSigner *v4.StreamSigner
	Time         time.Time
	ChunkSize    int
	ChunkTimeout time.Duration
	// contains filtered or unexported fields
}

func (*StreamingReader) GetLastChunk 

func (s *StreamingReader) GetLastChunk() []byte

func (*StreamingReader) Read 

func (s *StreamingReader) Read(p []byte) (int, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.