Documentation
¶
Index ¶
- Constants
- Variables
- func GenerateInventory(logger logging.Logger, m *Manifest, inventoryReader s3inventory.IReader, ...) (block.Inventory, error)
- func ReadAllWithTimeout(r io.Reader, buf []byte, timeout time.Duration, minSize int) (n int, err error)
- func WithDisablePreSigned(b bool) func(a *Adapter)
- func WithDiscoverBucketRegion(b bool) func(a *Adapter)
- func WithPreSignedExpiry(v time.Duration) func(a *Adapter)
- func WithServerSideEncryption(s string) func(a *Adapter)
- func WithServerSideEncryptionKmsKeyID(s string) func(a *Adapter)
- func WithStatsCollector(s stats.Collector) func(a *Adapter)
- func WithStreamingChunkSize(sz int) func(a *Adapter)
- func WithStreamingChunkTimeout(d time.Duration) func(a *Adapter)
- type Adapter
- func (a *Adapter) AbortMultiPartUpload(ctx context.Context, obj block.ObjectPointer, uploadID string) error
- func (a *Adapter) BlockstoreType() string
- func (a *Adapter) CompleteMultiPartUpload(ctx context.Context, obj block.ObjectPointer, uploadID string, ...) (*block.CompleteMultiPartUploadResponse, error)
- func (a *Adapter) Copy(ctx context.Context, sourceObj, destinationObj block.ObjectPointer) error
- func (a *Adapter) CreateMultiPartUpload(ctx context.Context, obj block.ObjectPointer, _ *http.Request, ...) (*block.CreateMultiPartUploadResponse, error)
- func (a *Adapter) Exists(ctx context.Context, obj block.ObjectPointer) (bool, error)
- func (a *Adapter) GenerateInventory(ctx context.Context, logger logging.Logger, manifestURL string, ...) (block.Inventory, error)
- func (a *Adapter) Get(ctx context.Context, obj block.ObjectPointer, _ int64) (io.ReadCloser, error)
- func (a *Adapter) GetPreSignedURL(ctx context.Context, obj block.ObjectPointer, mode block.PreSignMode) (string, error)
- func (a *Adapter) GetProperties(ctx context.Context, obj block.ObjectPointer) (block.Properties, error)
- func (a *Adapter) GetRange(ctx context.Context, obj block.ObjectPointer, startPosition int64, ...) (io.ReadCloser, error)
- func (a *Adapter) GetStorageNamespaceInfo() block.StorageNamespaceInfo
- func (a *Adapter) Put(ctx context.Context, obj block.ObjectPointer, sizeBytes int64, ...) error
- func (a *Adapter) Remove(ctx context.Context, obj block.ObjectPointer) error
- func (a *Adapter) RuntimeStats() map[string]string
- func (a *Adapter) UploadCopyPart(ctx context.Context, sourceObj, destinationObj block.ObjectPointer, ...) (*block.UploadPartResponse, error)
- func (a *Adapter) UploadCopyPartRange(ctx context.Context, sourceObj, destinationObj block.ObjectPointer, ...) (*block.UploadPartResponse, error)
- func (a *Adapter) UploadPart(ctx context.Context, obj block.ObjectPointer, sizeBytes int64, ...) (*block.UploadPartResponse, error)
- func (a *Adapter) Walk(ctx context.Context, walkOpt block.WalkOpts, walkFn block.WalkFunc) error
- type AdapterOption
- type ClientCache
- func (c *ClientCache) DiscoverBucketRegion(b bool)
- func (c *ClientCache) Get(ctx context.Context, bucket string) s3iface.S3API
- func (c *ClientCache) SetClientFactory(clientFactory clientFactory)
- func (c *ClientCache) SetS3RegionGetter(s3RegionGetter s3RegionGetter)
- func (c *ClientCache) SetStatsCollector(statsCollector stats.Collector)
- type Inventory
- type InventoryFile
- type InventoryIterator
- type Manifest
- type StreamingReader
Constants ¶
View Source
const ( DefaultStreamingChunkSize = 2 << 19 // 1MiB by default per chunk DefaultStreamingChunkTimeout = time.Second * 1 // if we haven't read DefaultStreamingChunkSize by this duration, write whatever we have as a chunk )
View Source
const ( StreamingSha256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" StreamingContentEncoding = "aws-chunked" )
Variables ¶
View Source
var ( ErrS3 = errors.New("s3 error") ErrMissingETag = fmt.Errorf("%w: missing ETag", ErrS3) )
View Source
var ErrInventoryFilesRangesOverlap = errors.New("got s3 inventory with files covering overlapping ranges")
View Source
var ErrInventoryNotSorted = errors.New("got unsorted s3 inventory")
View Source
var ErrReaderTimeout = errors.New("reader timeout")
Functions ¶
func GenerateInventory ¶
func ReadAllWithTimeout ¶
func ReadAllWithTimeout(r io.Reader, buf []byte, timeout time.Duration, minSize int) (n int, err error)
ReadAllWithTimeout is taken from io.ReadAtLeast and adapted to support a timeout and a minimum read size
func WithDisablePreSigned ¶ added in v0.94.0
func WithDiscoverBucketRegion ¶ added in v0.55.0
func WithPreSignedExpiry ¶ added in v0.92.0
func WithServerSideEncryption ¶ added in v0.87.0
func WithServerSideEncryptionKmsKeyID ¶ added in v0.87.0
func WithStatsCollector ¶ added in v0.48.0
func WithStreamingChunkSize ¶
Types ¶
type Adapter ¶
type Adapter struct { ServerSideEncryption string ServerSideEncryptionKmsKeyID string // contains filtered or unexported fields }
func NewAdapter ¶
func NewAdapter(awsSession *session.Session, opts ...AdapterOption) *Adapter
func (*Adapter) AbortMultiPartUpload ¶
func (*Adapter) BlockstoreType ¶
func (*Adapter) CompleteMultiPartUpload ¶
func (a *Adapter) CompleteMultiPartUpload(ctx context.Context, obj block.ObjectPointer, uploadID string, multipartList *block.MultipartUploadCompletion) (*block.CompleteMultiPartUploadResponse, error)
func (*Adapter) CreateMultiPartUpload ¶
func (a *Adapter) CreateMultiPartUpload(ctx context.Context, obj block.ObjectPointer, _ *http.Request, opts block.CreateMultiPartUploadOpts) (*block.CreateMultiPartUploadResponse, error)
func (*Adapter) GenerateInventory ¶
func (*Adapter) Get ¶
func (a *Adapter) Get(ctx context.Context, obj block.ObjectPointer, _ int64) (io.ReadCloser, error)
func (*Adapter) GetPreSignedURL ¶ added in v0.90.0
func (a *Adapter) GetPreSignedURL(ctx context.Context, obj block.ObjectPointer, mode block.PreSignMode) (string, error)
func (*Adapter) GetProperties ¶
func (a *Adapter) GetProperties(ctx context.Context, obj block.ObjectPointer) (block.Properties, error)
func (*Adapter) GetRange ¶
func (a *Adapter) GetRange(ctx context.Context, obj block.ObjectPointer, startPosition int64, endPosition int64) (io.ReadCloser, error)
func (*Adapter) GetStorageNamespaceInfo ¶
func (a *Adapter) GetStorageNamespaceInfo() block.StorageNamespaceInfo
func (*Adapter) RuntimeStats ¶ added in v0.41.1
func (*Adapter) UploadCopyPart ¶
func (a *Adapter) UploadCopyPart(ctx context.Context, sourceObj, destinationObj block.ObjectPointer, uploadID string, partNumber int) (*block.UploadPartResponse, error)
func (*Adapter) UploadCopyPartRange ¶
func (*Adapter) UploadPart ¶
type AdapterOption ¶ added in v0.87.0
type AdapterOption func(a *Adapter)
type ClientCache ¶ added in v0.48.0
type ClientCache struct {
// contains filtered or unexported fields
}
func NewClientCache ¶ added in v0.48.0
func NewClientCache(awsSession *session.Session) *ClientCache
func (*ClientCache) DiscoverBucketRegion ¶ added in v0.55.0
func (c *ClientCache) DiscoverBucketRegion(b bool)
func (*ClientCache) Get ¶ added in v0.48.0
Get returns an AWS client configured to the region of the given bucket.
func (*ClientCache) SetClientFactory ¶ added in v0.55.0
func (c *ClientCache) SetClientFactory(clientFactory clientFactory)
func (*ClientCache) SetS3RegionGetter ¶ added in v0.55.0
func (c *ClientCache) SetS3RegionGetter(s3RegionGetter s3RegionGetter)
func (*ClientCache) SetStatsCollector ¶ added in v0.55.0
func (c *ClientCache) SetStatsCollector(statsCollector stats.Collector)
type Inventory ¶
type Inventory struct { Manifest *Manifest // contains filtered or unexported fields }
func (*Inventory) InventoryURL ¶
func (*Inventory) Iterator ¶
func (inv *Inventory) Iterator() block.InventoryIterator
func (*Inventory) SourceName ¶
type InventoryFile ¶ added in v0.48.0
type InventoryFile struct { Key string `json:"key"` // an s3 key for an inventory list file // contains filtered or unexported fields }
type InventoryIterator ¶
type InventoryIterator struct { *Inventory // contains filtered or unexported fields }
func NewInventoryIterator ¶
func NewInventoryIterator(inv *Inventory) *InventoryIterator
func (*InventoryIterator) Err ¶
func (it *InventoryIterator) Err() error
func (*InventoryIterator) Get ¶
func (it *InventoryIterator) Get() *block.InventoryObject
func (*InventoryIterator) Next ¶
func (it *InventoryIterator) Next() bool
func (*InventoryIterator) Progress ¶
func (it *InventoryIterator) Progress() []*cmdutils.Progress
type Manifest ¶
type Manifest struct { URL string `json:"-"` InventoryBucketArn string `json:"destinationBucket"` SourceBucket string `json:"sourceBucket"` Files []InventoryFile `json:"files"` // inventory list files, each contains a list of objects Format string `json:"fileFormat"` CreationTimestamp string `json:"creationTimestamp"` // contains filtered or unexported fields }
type StreamingReader ¶
type StreamingReader struct { Reader io.Reader Size int64 StreamSigner *v4.StreamSigner Time time.Time ChunkSize int ChunkTimeout time.Duration // contains filtered or unexported fields }
func (*StreamingReader) GetLastChunk ¶
func (s *StreamingReader) GetLastChunk() []byte
Click to show internal directories.
Click to hide internal directories.