s3

Documentation

Index

• Constants
• type S3Collector
  • func NewS3Collector(cfg S3CollectorConfig) *S3Collector
  • func (s *S3Collector) RetrieveArtifacts(ctx context.Context, docChannel chan<- *processor.Document) error
  • func (s *S3Collector) Type() string
• type S3CollectorConfig

Constants

const (
	S3CollectorType = "S3CollectorType"
)

Types

type S3Collector

type S3Collector struct {
	// contains filtered or unexported fields
}

func NewS3Collector

func NewS3Collector(cfg S3CollectorConfig) *S3Collector

func (*S3Collector) RetrieveArtifacts

func (s *S3Collector) RetrieveArtifacts(ctx context.Context, docChannel chan<- *processor.Document) error

func (*S3Collector) Type

func (s *S3Collector) Type() string

type S3CollectorConfig

type S3CollectorConfig struct {
	MessageProvider         string
	MessageProviderEndpoint string                           // optional if using the sqs message provider
	S3Url                   string                           // optional (uses aws sdk defaults)
	S3Bucket                string                           // bucket name to collect from
	S3Path                  string                           // optional (only for non-polling) s3 folder path to collect from
	S3Item                  string                           // optional (only for non-polling behaviour)
	S3Region                string                           // optional (defaults to us-east-1, assumes same region for s3 and sqs)
	Queues                  string                           // optional (comma-separated list of queues/topics)
	MpBuilder               messaging.MessageProviderBuilder // optional
	BucketBuilder           bucket.BuildBucket               // optional
	Poll                    bool
}