barf

Documentation

Index

• Constants
• Variables
• type Barfer
  • func New(strm Stream) Barfer
• type S3Barfer
  • func (b *S3Barfer) Barf() <-chan string
  • func (b *S3Barfer) Close()
• type S3Lister
• type S3Reader
• type Stream

Constants

const MaxFileListings = 1000

Variables

var (
	// DefaultFlow is the default rate limit at which we can make AWS S3 API calls.
	// It is set to a conservative value for most modern hardware.
	DefaultFlow = 1.0
	// DefaultS3Region is the default AWS region for making AWS S3 API calls.
	// For more info, http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html
	DefaultS3Region = aws.USEast
	// DefaultWokers is the default number of concurrent wokers for reading s3 files.
	// By default it is set to the num of available cpus - 1
	DefaultWorkers = runtime.NumCPU() - 1
)

Types

type Barfer

type Barfer interface {
	Barf() <-chan string
	Close()
}

func New

func New(strm Stream) Barfer

type S3Barfer

type S3Barfer struct {
	Lister  *S3Lister
	Reader  *S3Reader
	Readers int
	// contains filtered or unexported fields
}

func (*S3Barfer) Barf

func (b *S3Barfer) Barf() <-chan string

func (*S3Barfer) Close

func (b *S3Barfer) Close()

type S3Lister

type S3Lister struct {
	Bucket s3bucketLister
	Path   string

	Limiter  <-chan struct{}
	DoneChan chan struct{}
	// contains filtered or unexported fields
}

type S3Reader

type S3Reader struct {
	Bucket  s3bucketReader
	Limiter <-chan struct{}
}

type Stream

type Stream struct {
	Src    string     // Src is the source of your file on S3. Example, s3://bucket/prefix/
	Flow   float64    // Flow is the rate limit of http calls per second.
	Region aws.Region // Region of AWS bucket.
}