s3

Documentation

Index

• func Uint64Max(a uint64, b uint64) uint64
• func Uint64Min(a uint64, b uint64) uint64
• type BatchDownloadObject
  • func (bdo *BatchDownloadObject) Bytes() []byte
• type BufferedReadSeekerWriteToPool
  • func NewBufferedReadSeekerWriteToPool(size int) *BufferedReadSeekerWriteToPool
  • func (p *BufferedReadSeekerWriteToPool) Close()
  • func (p *BufferedReadSeekerWriteToPool) GetWriteTo(seeker io.ReadSeeker) (r s3manager.ReadSeekerWriteTo, cleanup func())
• type Downloader
  • func NewDownloader(sess *session.Session, options ...func(*Downloader)) *Downloader
  • func (d *Downloader) Close()
  • func (d *Downloader) Done(input *BatchDownloadObject)
  • func (d *Downloader) Download(ctx aws.Context, builder func(*BatchDownloadObject)) error
  • func (d *Downloader) DownloadWithIterator(ctx aws.Context, iter chan *BatchDownloadObject) error
  • func (d *Downloader) GetDownloadPartSize() uint64
  • func (d *Downloader) Schedule(iter chan *BatchDownloadObject, builder func(*BatchDownloadObject)) (int, error)
• type PooledBufferedReadFromProvider
  • func NewPooledBufferedWriterReadFromProvider(size int) *PooledBufferedReadFromProvider
  • func (p *PooledBufferedReadFromProvider) Close()
  • func (p *PooledBufferedReadFromProvider) GetReadFrom(writer io.Writer) (r s3manager.WriterReadFrom, cleanup func())

Functions

func Uint64Max

func Uint64Max(a uint64, b uint64) uint64

func Uint64Min

func Uint64Min(a uint64, b uint64) uint64

Types

type BatchDownloadObject

type BatchDownloadObject struct {
	Object     *s3.GetObjectInput
	Size       uint64
	Writer     io.WriterAt
	After      func() error
	Meta       interface{}
	Downloaded int64
	Error      error
}

Customized version of s3manager.BatchDownloadObject

func (*BatchDownloadObject) Bytes

func (bdo *BatchDownloadObject) Bytes() []byte

type BufferedReadSeekerWriteToPool

type BufferedReadSeekerWriteToPool struct {
	// contains filtered or unexported fields
}

BufferedReadSeekerWriteToPool uses a sync.Pool to create and reuse []byte slices for buffering parts in memory

func NewBufferedReadSeekerWriteToPool

func NewBufferedReadSeekerWriteToPool(size int) *BufferedReadSeekerWriteToPool

NewBufferedReadSeekerWriteToPool will return a new BufferedReadSeekerWriteToPool that will create a pool of reusable buffers . If size is less then < 64 KiB then the buffer will default to 64 KiB. Reason: io.Copy from writers or readers that don't support io.WriteTo or io.ReadFrom respectively will default to copying 32 KiB.

func (*BufferedReadSeekerWriteToPool) Close

func (p *BufferedReadSeekerWriteToPool) Close()

func (*BufferedReadSeekerWriteToPool) GetWriteTo

func (p *BufferedReadSeekerWriteToPool) GetWriteTo(seeker io.ReadSeeker) (r s3manager.ReadSeekerWriteTo, cleanup func())

GetWriteTo will wrap the provided io.ReadSeeker with a BufferedReadSeekerWriteTo. The provided cleanup must be called after operations have been completed on the returned io.ReadSeekerWriteTo in order to signal the return of resources to the pool.

type Downloader

type Downloader struct {
	Concurrency    int
	BufferProvider s3manager.WriterReadFromProvider
	RequestOptions []request.Option
	// contains filtered or unexported fields
}

S3 Downloader Optimization Because the minimum concurrent download unit for aws download api is 5M, this optimization will effectively lower this limit for batch downloading

func NewDownloader

func NewDownloader(sess *session.Session, options ...func(*Downloader)) *Downloader

func (*Downloader) Close

func (d *Downloader) Close()

func (*Downloader) Done

func (d *Downloader) Done(input *BatchDownloadObject)

func (*Downloader) Download

func (d *Downloader) Download(ctx aws.Context, builder func(*BatchDownloadObject)) error

func (*Downloader) DownloadWithIterator

func (d *Downloader) DownloadWithIterator(ctx aws.Context, iter chan *BatchDownloadObject) error

func (*Downloader) GetDownloadPartSize

func (d *Downloader) GetDownloadPartSize() uint64

func (*Downloader) Schedule

func (d *Downloader) Schedule(iter chan *BatchDownloadObject, builder func(*BatchDownloadObject)) (int, error)

type PooledBufferedReadFromProvider

type PooledBufferedReadFromProvider struct {
	// contains filtered or unexported fields
}

PooledBufferedReadFromProvider is a WriterReadFromProvider that uses a sync.Pool to manage allocation and reuse of *bufio.Writer structures.

func NewPooledBufferedWriterReadFromProvider

func NewPooledBufferedWriterReadFromProvider(size int) *PooledBufferedReadFromProvider

NewPooledBufferedWriterReadFromProvider returns a new PooledBufferedReadFromProvider Size is used to control the size of the underlying *bufio.Writer created for calls to GetReadFrom.

func (*PooledBufferedReadFromProvider) Close

func (p *PooledBufferedReadFromProvider) Close()

func (*PooledBufferedReadFromProvider) GetReadFrom

func (p *PooledBufferedReadFromProvider) GetReadFrom(writer io.Writer) (r s3manager.WriterReadFrom, cleanup func())

GetReadFrom takes an io.Writer and wraps it with a type which satisfies the WriterReadFrom interface/ Additionally a cleanup function is provided which must be called after usage of the WriterReadFrom has been completed in order to allow the reuse of the *bufio.Writer