redshiftloader

Documentation

Index

• Variables
• func NewHandler(ctx context.Context, ready chan bool, consumerGroupID string, ...) *loaderHandler
• func ToExtraMaskSchemaMap(r string) map[string]serializer.ExtraMaskInfo
• func ToExtraMaskSchemaString(m map[string]serializer.ExtraMaskInfo) string
• func ToMaskSchemaMap(r string) map[string]serializer.MaskInfo
• func ToMaskSchemaString(m map[string]serializer.MaskInfo) string
• type Job
  • func NewJob(upstreamTopic string, startOffset int64, endOffset int64, csvDialect string, ...) Job
  • func StringMapToJob(data map[string]interface{}) Job
  • func (c Job) ToStringMap() map[string]interface{}
• type LoaderConfig

Variables

var (
	DefaultMaxWaitSeconds    int     = 1800
	DefaultMaxProcessingTime int32   = 600000
	MaxRunningLoaders        float64 = 10
	ThrottlingBudget         int     = 10
	FirstThrottlingBudget    int     = 120
)

var JobAvroSchema string = `` /* 867-byte string literal not displayed */

Functions

func NewHandler

func NewHandler(
	ctx context.Context,
	ready chan bool,
	consumerGroupID string,
	loaderConfig LoaderConfig,
	saramaConfig kafka.SaramaConfig,
	redshifter *redshift.Redshift,
	redshiftSchema string,
	redshiftGroup *string,
	redshiftMetrics bool,
	prometheusClient prometheus.Client,
	schemaQueries *model.Vector,
) *loaderHandler

func ToExtraMaskSchemaMap

func ToExtraMaskSchemaMap(r string) map[string]serializer.ExtraMaskInfo

TODO: hack, to release fast, found unwanted complications in using map[string]interface in goavro(will revisit)

func ToExtraMaskSchemaString

func ToExtraMaskSchemaString(m map[string]serializer.ExtraMaskInfo) string

TODO: hack, to release fast, found unwanted complications in using map[string]interface in goavro (may revisit if required)

func ToMaskSchemaMap

func ToMaskSchemaMap(r string) map[string]serializer.MaskInfo

TODO: hack, to release fast, found unwanted complications in using map[string]interface in goavro(will revisit)

func ToMaskSchemaString

func ToMaskSchemaString(m map[string]serializer.MaskInfo) string

TODO: hack, to release fast, found unwanted complications in using map[string]interface in goavro (may revisit if required)

Types

type Job

type Job struct {
	UpstreamTopic   string                              `json:"upstreamTopic"` // batcher topic
	StartOffset     int64                               `json:"startOffset"`
	EndOffset       int64                               `json:"endOffset"`
	CsvDialect      string                              `json:"csvDialect"`
	S3Path          string                              `json:"s3Path"`
	SchemaId        int                                 `json:"schemaId"`    // schema id of debezium event for the value for upstream topic (batcher topic)
	SchemaIdKey     int                                 `json:"schemaIdKey"` // schema id of debezium event for the key for upstream topic (batcher topic)
	MaskSchema      map[string]serializer.MaskInfo      `json:"maskSchema"`
	ExtraMaskSchema map[string]serializer.ExtraMaskInfo `json:"extraMaskSchema"`
	SkipMerge       bool                                `json:"skipMerge"`    // deprecated in favour of createEvents, updateEvents and deleteEvents
	BatchBytes      int64                               `json:"batchBytes"`   // batch bytes store sum of all message bytes in this batch
	CreateEvents    int64                               `json:"createEvents"` // stores count of create events
	UpdateEvents    int64                               `json:"updateEvents"` // stores count of update events
	DeleteEvents    int64                               `json:"deleteEvents"` // stores count of delete events
}

func NewJob

func NewJob(
	upstreamTopic string, startOffset int64, endOffset int64,
	csvDialect string, s3Path string, schemaId int, schemaIdKey int,
	maskSchema map[string]serializer.MaskInfo,
	extraMaskSchema map[string]serializer.ExtraMaskInfo,
	skipMerge bool,
	batchBytes, createEvents, updateEvents, deleteEvents int64) Job

func StringMapToJob

func StringMapToJob(data map[string]interface{}) Job

StringMapToUser returns a User from a map representation of the User.

func (Job) ToStringMap

func (c Job) ToStringMap() map[string]interface{}

ToStringMap returns a map representation of the Job

type LoaderConfig

type LoaderConfig struct {
	// Maximum size of a batch, on exceeding this batch is pushed
	// regarless of the wait time.
	// Deprecated: in favour of MaxBytesPerBatch
	MaxSize int `yaml:"maxSize,omitempty"`

	// MaxBytesPerBatch is the maximum bytes per batch. It is not the size
	// of kafka message but the size of all the messages that would be
	// loaded in the batch. Default is there
	// if the user has not specified a default will be applied.
	// If this is specified, maxSize specification is not considered.
	// Default would be specified after MaxSize is gone
	MaxBytesPerBatch *int64 `yaml:"maxBytesPerBatch,omitempty"`

	// MaxWaitSeconds after which the bash would be pushed regardless of its size.
	MaxWaitSeconds *int `yaml:"maxWaitSeconds,omitempty"`
}