config

Documentation

Index

• type Config
  • func LoadConfig(configFilename string, replay bool) (*Config, error)

Types

type Config

type Config struct {
	// Directory for all spade output
	SpadeDir string
	// ConfigBucket is the bucket that the blueprint published config lives in
	ConfigBucket string
	// SchemasKey is the s3 key to the blueprint published schemas config
	SchemasKey string
	// KinesisConfigKey is the s3 key to the blueprint published schemas config
	KinesisConfigKey string
	// MetadataConfigKey is the s3 key to the blueprint published metadata config
	MetadataConfigKey string
	// ProcessorErrorTopicARN is the arn of the SNS topic for processor errors
	ProcessorErrorTopicARN string
	// AceTopicARN is the arn of the SNS topic for events going to Ace
	AceTopicARN string
	// AceErrorTopicARN is the arn of the SNS topic for errors when sending events to Ace
	AceErrorTopicARN string
	// NonTrackedTopicARN is the arn of the SNS topic for events not tracked in blueprint
	NonTrackedTopicARN string
	// NonTrackedErrorTopicARN is the arn of the SNS topic for errors when sending nontracked events
	NonTrackedErrorTopicARN string
	// AceBucketName is the name of the s3 bucket to put processed events into
	AceBucketName string
	// NonTrackedBucketName is the name of the s3 bucket to put nontracked events into
	NonTrackedBucketName string
	// MaxLogBytes is the max number of log bytes before file rotation
	MaxLogBytes int64
	// MaxLogAgeSecs is the max number of seconds between log rotations
	MaxLogAgeSecs int64
	// NontrackedMaxLogAgeSecs is the max number of seconds between nontracked log rotations
	NontrackedMaxLogAgeSecs int64
	// Consumer is the config for the kinesis based event consumer
	Consumer consumer.Config
	// Geoip is the config for the geoip updater
	Geoip *geoip.Config
	// RollbarToken is our token to authenticate with Rollbar
	RollbarToken string
	// RollbarEnvironment is the environment we report we are running in to Rollbar
	RollbarEnvironment string

	// KinesisOutputs contains configs for KinesisWriters.
	KinesisOutputs []scoop_protocol.KinesisWriterConfig
	// KinesisWriterErrorsBeforeThrottling is the number of errors each Kinesis writer is permitted
	// to write to Rollbar before throttling kicks in.  Set to 0 to start throttling immediately.
	KinesisWriterErrorsBeforeThrottling int
	// KinesisWriterErrorThrottlePeriodSeconds is the error throttle period for each Kinesis writer
	// (i.e. once throttling switches on, only one error will be sent to Rollbar per period).
	// Set to 0 to turn throttling off and send all errors to Rollbar.
	KinesisWriterErrorThrottlePeriodSeconds int64
	// KinesisFilters is a set of filters that can be used by Kinesis streams.
	KinesisFilters map[string]*scoop_protocol.TestableKinesisEventFilter
	// KinesisFilterFuncs is the instantiation of KinesisFilters.
	KinesisFilterFuncs map[string]scoop_protocol.EventFilterFunc `json:"-"`
	// KinesisDefaultFilter is a default filter applied to all events in all streams.
	KinesisDefaultFilter *scoop_protocol.TestableKinesisEventFilter
	// KinesisFilterFuncs is the instantiation of KinesisDefaultFilter.
	KinesisDefaultFilterFunc scoop_protocol.EventFilterFunc `json:"-"`

	// JSONValueFetchers is a map of id to JSONValueFetcherConfigs
	JSONValueFetchers map[string]lookup.JSONValueFetcherConfig

	// TransformerCacheCluster contains the config required to instantiate a cache for transformers
	TransformerCacheCluster elastimemcache.Config

	// TransformerFetchers is a map of transformer id to value fetcher id
	TransformerFetchers map[string]string

	// LRULifetimeSeconds is the lifetime of an item in the local cache, in seconds.
	LRULifetimeSeconds int64

	// How often to load table schemas from Blueprint.
	SchemaReloadFrequency jsonutil.Duration
	// How long to sleep if there's an error loading table schemas from Blueprint.
	SchemaRetryDelay jsonutil.Duration
	// How often to load kinesis configs from Blueprint.
	KinesisConfigReloadFrequency jsonutil.Duration
	// How long to sleep if there's an error loading kinesis configs from Blueprint.
	KinesisConfigRetryDelay jsonutil.Duration
	// How often to load event metadata from Blueprint.
	EventMetadataReloadFrequency jsonutil.Duration
	// How long to sleep if there's an error loading event metadata from Blueprint.
	EventMetadataRetryDelay jsonutil.Duration

	// Host:port to send statsd events to
	StatsdHostport string
	// Prefix for statsd metrics
	StatsdPrefix string
}

Config controls the processor's behavior.

func LoadConfig

func LoadConfig(configFilename string, replay bool) (*Config, error)

LoadConfig loads the config object from the file, performing validations.