Documentation
¶
Index ¶
- Constants
- Variables
- func ExtractParamsFromQK(qk block.QualifiedKey) (string, string)
- func GenerateInventory(ctx context.Context, logger logging.Logger, m *Manifest, ...) (block.Inventory, error)
- func ReadAllWithTimeout(r io.Reader, buf []byte, timeout time.Duration, minSize int) (n int, err error)
- func WithDisablePreSigned(b bool) func(a *Adapter)
- func WithDisablePreSignedUI(b bool) func(a *Adapter)
- func WithDiscoverBucketRegion(b bool) func(a *Adapter)
- func WithPreSignedExpiry(v time.Duration) func(a *Adapter)
- func WithPreSignedRefreshWindow(v time.Duration) func(a *Adapter)
- func WithServerSideEncryption(s string) func(a *Adapter)
- func WithServerSideEncryptionKmsKeyID(s string) func(a *Adapter)
- func WithStatsCollector(s stats.Collector) func(a *Adapter)
- func WithStreamingChunkSize(sz int) func(a *Adapter)
- func WithStreamingChunkTimeout(d time.Duration) func(a *Adapter)
- type Adapter
- func (a *Adapter) AbortMultiPartUpload(ctx context.Context, obj block.ObjectPointer, uploadID string) error
- func (a *Adapter) BlockstoreType() string
- func (a *Adapter) CompleteMultiPartUpload(ctx context.Context, obj block.ObjectPointer, uploadID string, ...) (*block.CompleteMultiPartUploadResponse, error)
- func (a *Adapter) Copy(ctx context.Context, sourceObj, destinationObj block.ObjectPointer) error
- func (a *Adapter) CreateMultiPartUpload(ctx context.Context, obj block.ObjectPointer, _ *http.Request, ...) (*block.CreateMultiPartUploadResponse, error)
- func (a *Adapter) Exists(ctx context.Context, obj block.ObjectPointer) (bool, error)
- func (a *Adapter) GenerateInventory(ctx context.Context, logger logging.Logger, manifestURL string, ...) (block.Inventory, error)
- func (a *Adapter) Get(ctx context.Context, obj block.ObjectPointer, _ int64) (io.ReadCloser, error)
- func (a *Adapter) GetPreSignedURL(ctx context.Context, obj block.ObjectPointer, mode block.PreSignMode) (string, time.Time, error)
- func (a *Adapter) GetProperties(ctx context.Context, obj block.ObjectPointer) (block.Properties, error)
- func (a *Adapter) GetRange(ctx context.Context, obj block.ObjectPointer, startPosition int64, ...) (io.ReadCloser, error)
- func (a *Adapter) GetStorageNamespaceInfo() block.StorageNamespaceInfo
- func (a *Adapter) GetWalker(uri *url.URL) (block.Walker, error)
- func (a *Adapter) Put(ctx context.Context, obj block.ObjectPointer, sizeBytes int64, ...) error
- func (a *Adapter) Remove(ctx context.Context, obj block.ObjectPointer) error
- func (a *Adapter) ResolveNamespace(storageNamespace, key string, identifierType block.IdentifierType) (block.QualifiedKey, error)
- func (a *Adapter) RuntimeStats() map[string]string
- func (a *Adapter) UploadCopyPart(ctx context.Context, sourceObj, destinationObj block.ObjectPointer, ...) (*block.UploadPartResponse, error)
- func (a *Adapter) UploadCopyPartRange(ctx context.Context, sourceObj, destinationObj block.ObjectPointer, ...) (*block.UploadPartResponse, error)
- func (a *Adapter) UploadPart(ctx context.Context, obj block.ObjectPointer, sizeBytes int64, ...) (*block.UploadPartResponse, error)
- type AdapterOption
- type ClientCache
- func (c *ClientCache) DiscoverBucketRegion(b bool)
- func (c *ClientCache) Get(ctx context.Context, bucket string) S3APIWithExpirer
- func (c *ClientCache) SetClientFactory(clientFactory clientFactory)
- func (c *ClientCache) SetS3RegionGetter(s3RegionGetter s3RegionGetter)
- func (c *ClientCache) SetStatsCollector(statsCollector stats.Collector)
- type Expirer
- type Inventory
- type InventoryFile
- type InventoryIterator
- type Manifest
- type S3APIWithExpirer
- type StreamingReader
- type Walker
- Bugs
Constants ¶
View Source
const ( DefaultStreamingChunkSize = 2 << 19 // 1MiB by default per chunk DefaultStreamingChunkTimeout = time.Second * 1 // if we haven't read DefaultStreamingChunkSize by this duration, write whatever we have as a chunk )
View Source
const ( StreamingSha256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" StreamingContentEncoding = "aws-chunked" )
Variables ¶
View Source
var ( ErrS3 = errors.New("s3 error") ErrMissingETag = fmt.Errorf("%w: missing ETag", ErrS3) )
View Source
var ErrDoesntExpire = errors.New("no access expiry")
ErrDoesntExpire is returned by an Expirer if expiry times cannot be determined. For instance, if AWS is configured using an access key then Expirer cannot determine expiry.
View Source
var ErrInventoryFilesRangesOverlap = errors.New("got s3 inventory with files covering overlapping ranges")
View Source
var ErrInventoryNotSorted = errors.New("got unsorted s3 inventory")
View Source
var ErrReaderTimeout = errors.New("reader timeout")
Functions ¶
func ExtractParamsFromQK ¶ added in v0.97.0
func ExtractParamsFromQK(qk block.QualifiedKey) (string, string)
func GenerateInventory ¶
func ReadAllWithTimeout ¶
func ReadAllWithTimeout(r io.Reader, buf []byte, timeout time.Duration, minSize int) (n int, err error)
ReadAllWithTimeout is taken from io.ReadAtLeast and adapted to support a timeout and a minimum read size
func WithDisablePreSigned ¶ added in v0.94.0
func WithDisablePreSignedUI ¶ added in v0.97.0
func WithDiscoverBucketRegion ¶ added in v0.55.0
func WithPreSignedExpiry ¶ added in v0.92.0
func WithPreSignedRefreshWindow ¶ added in v0.106.2
func WithServerSideEncryption ¶ added in v0.87.0
func WithServerSideEncryptionKmsKeyID ¶ added in v0.87.0
func WithStatsCollector ¶ added in v0.48.0
func WithStreamingChunkSize ¶
Types ¶
type Adapter ¶
type Adapter struct { ServerSideEncryption string ServerSideEncryptionKmsKeyID string // contains filtered or unexported fields }
func NewAdapter ¶
func NewAdapter(awsSession *session.Session, opts ...AdapterOption) *Adapter
func (*Adapter) AbortMultiPartUpload ¶
func (*Adapter) BlockstoreType ¶
func (*Adapter) CompleteMultiPartUpload ¶
func (a *Adapter) CompleteMultiPartUpload(ctx context.Context, obj block.ObjectPointer, uploadID string, multipartList *block.MultipartUploadCompletion) (*block.CompleteMultiPartUploadResponse, error)
func (*Adapter) CreateMultiPartUpload ¶
func (a *Adapter) CreateMultiPartUpload(ctx context.Context, obj block.ObjectPointer, _ *http.Request, opts block.CreateMultiPartUploadOpts) (*block.CreateMultiPartUploadResponse, error)
func (*Adapter) GenerateInventory ¶
func (*Adapter) Get ¶
func (a *Adapter) Get(ctx context.Context, obj block.ObjectPointer, _ int64) (io.ReadCloser, error)
func (*Adapter) GetPreSignedURL ¶ added in v0.90.0
func (*Adapter) GetProperties ¶
func (a *Adapter) GetProperties(ctx context.Context, obj block.ObjectPointer) (block.Properties, error)
func (*Adapter) GetRange ¶
func (a *Adapter) GetRange(ctx context.Context, obj block.ObjectPointer, startPosition int64, endPosition int64) (io.ReadCloser, error)
func (*Adapter) GetStorageNamespaceInfo ¶
func (a *Adapter) GetStorageNamespaceInfo() block.StorageNamespaceInfo
func (*Adapter) ResolveNamespace ¶ added in v0.97.0
func (a *Adapter) ResolveNamespace(storageNamespace, key string, identifierType block.IdentifierType) (block.QualifiedKey, error)
func (*Adapter) RuntimeStats ¶ added in v0.41.1
func (*Adapter) UploadCopyPart ¶
func (a *Adapter) UploadCopyPart(ctx context.Context, sourceObj, destinationObj block.ObjectPointer, uploadID string, partNumber int) (*block.UploadPartResponse, error)
func (*Adapter) UploadCopyPartRange ¶
func (*Adapter) UploadPart ¶
type AdapterOption ¶ added in v0.87.0
type AdapterOption func(a *Adapter)
type ClientCache ¶ added in v0.48.0
type ClientCache struct {
// contains filtered or unexported fields
}
func NewClientCache ¶ added in v0.48.0
func NewClientCache(awsSession *session.Session) *ClientCache
func (*ClientCache) DiscoverBucketRegion ¶ added in v0.55.0
func (c *ClientCache) DiscoverBucketRegion(b bool)
func (*ClientCache) Get ¶ added in v0.48.0
func (c *ClientCache) Get(ctx context.Context, bucket string) S3APIWithExpirer
Get returns an AWS client configured to the region of the given bucket.
func (*ClientCache) SetClientFactory ¶ added in v0.55.0
func (c *ClientCache) SetClientFactory(clientFactory clientFactory)
func (*ClientCache) SetS3RegionGetter ¶ added in v0.55.0
func (c *ClientCache) SetS3RegionGetter(s3RegionGetter s3RegionGetter)
func (*ClientCache) SetStatsCollector ¶ added in v0.55.0
func (c *ClientCache) SetStatsCollector(statsCollector stats.Collector)
type Expirer ¶ added in v0.105.0
type Expirer interface { // ExpiresAt returns an expiry time or an error. It returns // a ErrDoesntExpire if it cannot determine expiry times -- for // instance, if AWS is configured using an access key. ExpiresAt() (time.Time, error) // Refresh attempts to refresh and returns ExpiresAt(). Refresh() (time.Time, error) }
type Inventory ¶
type Inventory struct { Manifest *Manifest // contains filtered or unexported fields }
func (*Inventory) InventoryURL ¶
func (*Inventory) Iterator ¶
func (inv *Inventory) Iterator() block.InventoryIterator
func (*Inventory) SourceName ¶
type InventoryFile ¶ added in v0.48.0
type InventoryFile struct { Key string `json:"key"` // an s3 key for an inventory list file // contains filtered or unexported fields }
type InventoryIterator ¶
type InventoryIterator struct { *Inventory // contains filtered or unexported fields }
func NewInventoryIterator ¶
func NewInventoryIterator(inv *Inventory) *InventoryIterator
func (*InventoryIterator) Err ¶
func (it *InventoryIterator) Err() error
func (*InventoryIterator) Get ¶
func (it *InventoryIterator) Get() *block.InventoryObject
func (*InventoryIterator) Next ¶
func (it *InventoryIterator) Next() bool
func (*InventoryIterator) Progress ¶
func (it *InventoryIterator) Progress() []*cmdutils.Progress
type Manifest ¶
type Manifest struct { URL string `json:"-"` InventoryBucketArn string `json:"destinationBucket"` SourceBucket string `json:"sourceBucket"` Files []InventoryFile `json:"files"` // inventory list files, each contains a list of objects Format string `json:"fileFormat"` CreationTimestamp string `json:"creationTimestamp"` // contains filtered or unexported fields }
type S3APIWithExpirer ¶ added in v0.105.0
type StreamingReader ¶
type StreamingReader struct { Reader io.Reader Size int64 StreamSigner *v4.StreamSigner Time time.Time ChunkSize int ChunkTimeout time.Duration // contains filtered or unexported fields }
func (*StreamingReader) GetLastChunk ¶
func (s *StreamingReader) GetLastChunk() []byte
type Walker ¶ added in v0.97.0
type Walker struct {
// contains filtered or unexported fields
}
func NewS3Walker ¶ added in v0.97.0
func (*Walker) GetSkippedEntries ¶ added in v0.98.0
func (s *Walker) GetSkippedEntries() []block.ObjectStoreEntry
Notes ¶
Bugs ¶
This is an inherent race. urlLifetime is computed
relative to the local clock. If expiry was shortened because of clientExpiry then AWS will determine _remotely_ whether the URL expired. So this URL can expire before the client or even lakeFS think that it has. This is a limitation of the AWS SDK, which signs locally, and of the AWS S3 API, which does not allow a meaningful workaround.
Click to show internal directories.
Click to hide internal directories.