redshiftbatcher

Documentation

Index

• Variables
• func NewHandler(ready chan bool, ctx context.Context, consumerGroupID string, ...) *batcherHandler
• type BatcherConfig

Variables

var (
	DefaultMaxWaitSeconds    int   = 30
	DefaultMaxConcurrency    int   = 10
	DefaultMaxProcessingTime int32 = 180000
)

Functions

func NewHandler

func NewHandler(
	ready chan bool,
	ctx context.Context,
	consumerGroupID string,
	batcherConfig BatcherConfig,
	kafkaConfig kafka.KafkaConfig,
	saramaConfig kafka.SaramaConfig,
	maskConfig masker.MaskConfig,
	loaderPrefix string,
) *batcherHandler

Types

type BatcherConfig

type BatcherConfig struct {
	// Mask should be turned on or off
	Mask bool `yaml:"mask,omitempty"`
	// MaskSalt specifies the salt to be used for masking
	MaskSalt string `yaml:"maskSalt,omitempty"`
	// MaskFile can be the either of the two:
	// 1. Absolute path of the mask configuration file. This file needs to be
	// be mounted as config map when the batcher starts.
	// 2. Git File or Folder. Examples:
	// https://github.com/practo/tipoca-stream/pkg/database.yaml
	// then this file or repo be cloned and kept at / when batcher starts.
	MaskFile string `yaml:"maskFile,omitempty"`
	// MaskFileVersion is the git version of the MaskFile
	// It is useful when the MaskFile is specified is a Git File.
	// otherwise when MaskFile is an abosolute mounted file, it is not used.
	MaskFileVersion string `yaml:"maskFileVersion,omitempty"`

	// MaxSize is the maximum size of a batch, on exceeding this batch is pushed
	// regarless of the wait time.
	// Deprecated: in favour of MaxBytesPerBatch
	MaxSize int `yaml:"maxSize,omitempty"`

	// MaxWaitSeconds after which the batch would be flushed
	// Defaults to 30
	MaxWaitSeconds *int `yaml:"maxWaitSeconds,omitempty"`
	// MaxConcurrency is the maximum number of concurrent processing to run
	// Defaults to 10
	MaxConcurrency *int `yaml:"maxConcurrency,omitempty"`
	// MaxBytesPerBatch is the maximum bytes per batch. Default is there
	// if the user has not specified a default will be applied.
	// If this is specified, maxSize specification is not considered.
	// Default would be specified after MaxSize is gone
	MaxBytesPerBatch *int64 `yaml:"maxBytesPerBatch,omitempty"`
}