s3

package
v0.106.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 16, 2023 License: Apache-2.0 Imports: 31 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	DefaultStreamingChunkSize    = 2 << 19         // 1MiB by default per chunk
	DefaultStreamingChunkTimeout = time.Second * 1 // if we haven't read DefaultStreamingChunkSize by this duration, write whatever we have as a chunk

)
View Source 
const (
	StreamingSha256          = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
	StreamingContentEncoding = "aws-chunked"
)

Variables

View Source 
var (
	ErrS3          = errors.New("s3 error")
	ErrMissingETag = fmt.Errorf("%w: missing ETag", ErrS3)
)
View Source 
var ErrDoesntExpire = errors.New("no access expiry")

ErrDoesntExpire is returned by an Expirer if expiry times cannot be determined. For instance, if AWS is configured using an access key then Expirer cannot determine expiry.

View Source 
var ErrInventoryFilesRangesOverlap = errors.New("got s3 inventory with files covering overlapping ranges")
View Source 
var ErrInventoryNotSorted = errors.New("got unsorted s3 inventory")
View Source 
var ErrReaderTimeout = errors.New("reader timeout")

Functions

func ExtractParamsFromQK added in v0.97.0 

func ExtractParamsFromQK(qk block.QualifiedKey) (string, string)

func GenerateInventory 

func GenerateInventory(ctx context.Context, logger logging.Logger, m *Manifest, inventoryReader s3inventory.IReader, shouldSort bool, prefixes []string) (block.Inventory, error)

func ReadAllWithTimeout 

func ReadAllWithTimeout(r io.Reader, buf []byte, timeout time.Duration, minSize int) (n int, err error)

ReadAllWithTimeout is taken from io.ReadAtLeast and adapted to support a timeout and a minimum read size

func WithDisablePreSigned added in v0.94.0 

func WithDisablePreSigned(b bool) func(a *Adapter)

func WithDisablePreSignedUI added in v0.97.0 

func WithDisablePreSignedUI(b bool) func(a *Adapter)

func WithDiscoverBucketRegion added in v0.55.0 

func WithDiscoverBucketRegion(b bool) func(a *Adapter)

func WithPreSignedExpiry added in v0.92.0 

func WithPreSignedExpiry(v time.Duration) func(a *Adapter)

func WithPreSignedRefreshWindow added in v0.106.2 

func WithPreSignedRefreshWindow(v time.Duration) func(a *Adapter)

func WithServerSideEncryption added in v0.87.0 

func WithServerSideEncryption(s string) func(a *Adapter)

func WithServerSideEncryptionKmsKeyID added in v0.87.0 

func WithServerSideEncryptionKmsKeyID(s string) func(a *Adapter)

func WithStatsCollector added in v0.48.0 

func WithStatsCollector(s stats.Collector) func(a *Adapter)

func WithStreamingChunkSize 

func WithStreamingChunkSize(sz int) func(a *Adapter)

func WithStreamingChunkTimeout 

func WithStreamingChunkTimeout(d time.Duration) func(a *Adapter)

Types

type Adapter 

type Adapter struct {
	ServerSideEncryption         string
	ServerSideEncryptionKmsKeyID string
	// contains filtered or unexported fields
}

func NewAdapter 

func NewAdapter(awsSession *session.Session, opts ...AdapterOption) *Adapter

func (*Adapter) AbortMultiPartUpload 

func (a *Adapter) AbortMultiPartUpload(ctx context.Context, obj block.ObjectPointer, uploadID string) error

func (*Adapter) BlockstoreType 

func (a *Adapter) BlockstoreType() string

func (*Adapter) CompleteMultiPartUpload 

func (a *Adapter) CompleteMultiPartUpload(ctx context.Context, obj block.ObjectPointer, uploadID string, multipartList *block.MultipartUploadCompletion) (*block.CompleteMultiPartUploadResponse, error)

func (*Adapter) Copy 

func (a *Adapter) Copy(ctx context.Context, sourceObj, destinationObj block.ObjectPointer) error

func (*Adapter) CreateMultiPartUpload 

func (a *Adapter) CreateMultiPartUpload(ctx context.Context, obj block.ObjectPointer, _ *http.Request, opts block.CreateMultiPartUploadOpts) (*block.CreateMultiPartUploadResponse, error)

func (*Adapter) Exists 

func (a *Adapter) Exists(ctx context.Context, obj block.ObjectPointer) (bool, error)

func (*Adapter) GenerateInventory 

func (a *Adapter) GenerateInventory(ctx context.Context, logger logging.Logger, manifestURL string, shouldSort bool, prefixes []string) (block.Inventory, error)

func (*Adapter) Get 

func (a *Adapter) Get(ctx context.Context, obj block.ObjectPointer, _ int64) (io.ReadCloser, error)

func (*Adapter) GetPreSignedURL added in v0.90.0 

func (a *Adapter) GetPreSignedURL(ctx context.Context, obj block.ObjectPointer, mode block.PreSignMode) (string, time.Time, error)

func (*Adapter) GetProperties 

func (a *Adapter) GetProperties(ctx context.Context, obj block.ObjectPointer) (block.Properties, error)

func (*Adapter) GetRange 

func (a *Adapter) GetRange(ctx context.Context, obj block.ObjectPointer, startPosition int64, endPosition int64) (io.ReadCloser, error)

func (*Adapter) GetStorageNamespaceInfo 

func (a *Adapter) GetStorageNamespaceInfo() block.StorageNamespaceInfo

func (*Adapter) GetWalker added in v0.97.0 

func (a *Adapter) GetWalker(uri *url.URL) (block.Walker, error)

func (*Adapter) Put 

func (a *Adapter) Put(ctx context.Context, obj block.ObjectPointer, sizeBytes int64, reader io.Reader, opts block.PutOpts) error

func (*Adapter) Remove 

func (a *Adapter) Remove(ctx context.Context, obj block.ObjectPointer) error

func (*Adapter) ResolveNamespace added in v0.97.0 

func (a *Adapter) ResolveNamespace(storageNamespace, key string, identifierType block.IdentifierType) (block.QualifiedKey, error)

func (*Adapter) RuntimeStats added in v0.41.1 

func (a *Adapter) RuntimeStats() map[string]string

func (*Adapter) UploadCopyPart 

func (a *Adapter) UploadCopyPart(ctx context.Context, sourceObj, destinationObj block.ObjectPointer, uploadID string, partNumber int) (*block.UploadPartResponse, error)

func (*Adapter) UploadCopyPartRange 

func (a *Adapter) UploadCopyPartRange(ctx context.Context, sourceObj, destinationObj block.ObjectPointer, uploadID string, partNumber int, startPosition, endPosition int64) (*block.UploadPartResponse, error)

func (*Adapter) UploadPart 

func (a *Adapter) UploadPart(ctx context.Context, obj block.ObjectPointer, sizeBytes int64, reader io.Reader, uploadID string, partNumber int) (*block.UploadPartResponse, error)

type AdapterOption added in v0.87.0 

type AdapterOption func(a *Adapter)

type ClientCache added in v0.48.0 

type ClientCache struct {
	// contains filtered or unexported fields
}

func NewClientCache added in v0.48.0 

func NewClientCache(awsSession *session.Session) *ClientCache

func (*ClientCache) DiscoverBucketRegion added in v0.55.0 

func (c *ClientCache) DiscoverBucketRegion(b bool)

func (*ClientCache) Get added in v0.48.0 

func (c *ClientCache) Get(ctx context.Context, bucket string) S3APIWithExpirer

Get returns an AWS client configured to the region of the given bucket.

func (*ClientCache) SetClientFactory added in v0.55.0 

func (c *ClientCache) SetClientFactory(clientFactory clientFactory)

func (*ClientCache) SetS3RegionGetter added in v0.55.0 

func (c *ClientCache) SetS3RegionGetter(s3RegionGetter s3RegionGetter)

func (*ClientCache) SetStatsCollector added in v0.55.0 

func (c *ClientCache) SetStatsCollector(statsCollector stats.Collector)

type Expirer added in v0.105.0 

type Expirer interface {
	// ExpiresAt returns an expiry time or an error.  It returns
	// a ErrDoesntExpire if it cannot determine expiry times -- for
	// instance, if AWS is configured using an access key.
	ExpiresAt() (time.Time, error)
	// Refresh attempts to refresh and returns ExpiresAt().
	Refresh() (time.Time, error)
}

type Inventory 

type Inventory struct {
	Manifest *Manifest
	// contains filtered or unexported fields
}

func (*Inventory) InventoryURL 

func (inv *Inventory) InventoryURL() string

func (*Inventory) Iterator 

func (inv *Inventory) Iterator() block.InventoryIterator

func (*Inventory) SourceName 

func (inv *Inventory) SourceName() string

type InventoryFile added in v0.48.0 

type InventoryFile struct {
	Key string `json:"key"` // an s3 key for an inventory list file
	// contains filtered or unexported fields
}

type InventoryIterator 

type InventoryIterator struct {
	*Inventory
	// contains filtered or unexported fields
}

func NewInventoryIterator 

func NewInventoryIterator(inv *Inventory) *InventoryIterator

func (*InventoryIterator) Err 

func (it *InventoryIterator) Err() error

func (*InventoryIterator) Get 

func (it *InventoryIterator) Get() *block.InventoryObject

func (*InventoryIterator) Next 

func (it *InventoryIterator) Next() bool

func (*InventoryIterator) Progress 

func (it *InventoryIterator) Progress() []*cmdutils.Progress

type Manifest 

type Manifest struct {
	URL                string          `json:"-"`
	InventoryBucketArn string          `json:"destinationBucket"`
	SourceBucket       string          `json:"sourceBucket"`
	Files              []InventoryFile `json:"files"` // inventory list files, each contains a list of objects
	Format             string          `json:"fileFormat"`
	CreationTimestamp  string          `json:"creationTimestamp"`
	// contains filtered or unexported fields
}

type S3APIWithExpirer added in v0.105.0 

type S3APIWithExpirer interface {
	s3iface.S3API
	Expirer
}

type StreamingReader 

type StreamingReader struct {
	Reader       io.Reader
	Size         int64
	StreamSigner *v4.StreamSigner
	Time         time.Time
	ChunkSize    int
	ChunkTimeout time.Duration
	// contains filtered or unexported fields
}

func (*StreamingReader) GetLastChunk 

func (s *StreamingReader) GetLastChunk() []byte

func (*StreamingReader) Read 

func (s *StreamingReader) Read(p []byte) (int, error)

type Walker added in v0.97.0 

type Walker struct {
	// contains filtered or unexported fields
}

func NewS3Walker added in v0.97.0 

func NewS3Walker(sess *session.Session) *Walker

func (*Walker) GetSkippedEntries added in v0.98.0 

func (s *Walker) GetSkippedEntries() []block.ObjectStoreEntry

func (*Walker) Marker added in v0.97.0 

func (s *Walker) Marker() block.Mark

func (*Walker) Walk added in v0.97.0 

func (s *Walker) Walk(ctx context.Context, storageURI *url.URL, op block.WalkOptions, walkFn func(e block.ObjectStoreEntry) error) error

Notes

Bugs

  • This is an inherent race. urlLifetime is computed 

    relative to the local clock. If expiry was shortened because
of clientExpiry then AWS will determine _remotely_ whether
the URL expired. So this URL can expire before the client or
even lakeFS think that it has.

This is a limitation of the AWS SDK, which signs locally, and
of the AWS S3 API, which does not allow a meaningful
workaround.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.