uploader

Documentation

Index

• func BuildUploaderForBlueprint(numWorkers int, sns snsiface.SNSAPI, s3Uploader s3manageriface.UploaderAPI, ...) *uploader.UploaderPool
• func BuildUploaderForRedshift(numWorkers int, sns snsiface.SNSAPI, s3Uploader s3manageriface.UploaderAPI, ...) *uploader.UploaderPool
• func ClearEventsFolder(uploaderPool *uploader.UploaderPool, eventsDir string) error
• func SafeGzipUpload(uploaderPool *uploader.UploaderPool, path string)
• func SalvageCorruptedEvents(eventsDir string) error
• type BlueprintSNSNotifierHarness
  • func (s *BlueprintSNSNotifierHarness) SendMessage(message *uploader.UploadReceipt) error
• type NullErrorHandler
  • func (n *NullErrorHandler) SendError(err error)
• type NullNotifierHarness
  • func (n *NullNotifierHarness) SendMessage(_ *uploader.UploadReceipt) error
• type ProcessorErrorHandler
  • func (p *ProcessorErrorHandler) SendError(err error)
• type RedshiftSNSNotifierHarness
  • func (s *RedshiftSNSNotifierHarness) SendMessage(message *uploader.UploadReceipt) error

Functions

func BuildUploaderForBlueprint

func BuildUploaderForBlueprint(numWorkers int, sns snsiface.SNSAPI, s3Uploader s3manageriface.UploaderAPI,
	nonTrackedBucketName, nonTrackedTopicARN, nonTrackedErrorTopicARN string, replay bool) *uploader.UploaderPool

BuildUploaderForBlueprint builds an Uploader that uploads non-tracked events to s3 and notifies sns.

func BuildUploaderForRedshift

func BuildUploaderForRedshift(numWorkers int, sns snsiface.SNSAPI, s3Uploader s3manageriface.UploaderAPI,
	aceBucketName, aceTopicARN, aceErrorTopicARN, runTag string, replay bool) *uploader.UploaderPool

BuildUploaderForRedshift builds an Uploader that uploads files to s3 and notifies sns.

func ClearEventsFolder

func ClearEventsFolder(uploaderPool *uploader.UploaderPool, eventsDir string) error

ClearEventsFolder uploads all files in the eventsDir.

func SafeGzipUpload

func SafeGzipUpload(uploaderPool *uploader.UploaderPool, path string)

SafeGzipUpload validates a file is a valid gzip file and then uploads it.

func SalvageCorruptedEvents

func SalvageCorruptedEvents(eventsDir string) error

SalvageCorruptedEvents salvages in place all invalid gzip files in the eventsDir

Types

type BlueprintSNSNotifierHarness

type BlueprintSNSNotifierHarness struct {
	// contains filtered or unexported fields
}

BlueprintSNSNotifierHarness is an SNS client that writes messages about uploaded nontracked event files

func (*BlueprintSNSNotifierHarness) SendMessage

func (s *BlueprintSNSNotifierHarness) SendMessage(message *uploader.UploadReceipt) error

SendMessage sends information to SNS about file uploaded to S3.

type NullErrorHandler

type NullErrorHandler struct{}

NullErrorHandler logs errors but does not send an SNS message.

func (*NullErrorHandler) SendError

func (n *NullErrorHandler) SendError(err error)

SendError logs the given error.

type NullNotifierHarness

type NullNotifierHarness struct{}

NullNotifierHarness is a stub SNS client that does nothing. In replay mode, we don't need to notify anyone of the newly uploaded files.

func (*NullNotifierHarness) SendMessage

func (n *NullNotifierHarness) SendMessage(_ *uploader.UploadReceipt) error

SendMessage is a noop

type ProcessorErrorHandler

type ProcessorErrorHandler struct {
	// contains filtered or unexported fields
}

ProcessorErrorHandler sends messages about errors sending SNS messages to another topic.

func (*ProcessorErrorHandler) SendError

func (p *ProcessorErrorHandler) SendError(err error)

SendError sends the sending error to an topic.

type RedshiftSNSNotifierHarness

type RedshiftSNSNotifierHarness struct {
	// contains filtered or unexported fields
}

RedshiftSNSNotifierHarness is an SNS client that writes messages about uploaded event files

func (*RedshiftSNSNotifierHarness) SendMessage

func (s *RedshiftSNSNotifierHarness) SendMessage(message *uploader.UploadReceipt) error

SendMessage sends information to SNS about file uploaded to S3.