config

Documentation

Index

• type AzureSink
• type Badger
• type BadgerDefault
• type BigQuerySink
• type Compaction
• type Computed
• type Config
• type Configurer
• type FileSink
• type Func
  • func Load(ctx context.Context, d time.Duration, configurers ...Configurer) Func
• type GCSSink
• type GRPC
• type K8s
• type Presto
• type PubSubSink
• type Readers
• type S3SQS
• type S3Sink
• type Sinks
• type StatsD
• type Storage
• type Streams
• type Table
• type Tables
• type TalariaSink
• type Writers

Types

type AzureSink

type AzureSink struct {
	Container       string   `json:"container" yaml:"container" env:"CONTAINER"`                   // The container name
	Prefix          string   `json:"prefix" yaml:"prefix" env:"PREFIX"`                            // The prefix to add
	Parallelism     uint16   `json:"parallelism" yaml:"parallelism" env:"PARALLELISM"`             // The BlockBlob upload parallelism
	BlockSize       int64    `json:"blockSize" yaml:"blockSize" env:"BLOCKSIZE"`                   // The Block Size for upload
	BlobServiceURL  string   `json:"blobServiceURL" yaml:"blobServiceURL" env:"BLOBSERVICEURL"`    // The blob service URL
	StorageAccounts []string `json:"storageAccounts" yaml:"storageAccounts" env:"STORAGEACCOUNTS"` // The list of storage accounts
}

AzureSink represents a sink to Azure

type Badger

type Badger struct {
	SyncWrites          *bool         `json:"syncWrites" yaml:"syncWrites" env:"SYNCWRITES"`                            // Whether to sync writes to disk before ack. defaults to true
	ValueLogMaxEntries  *uint32       `json:"valueLogMaxEntries" yaml:"valueLogMaxEntries" env:"VALUELOGMAXENTRIES"`    // Maximum number of entries a value log file can hold approximately. defaults to 5000
	MaxTableSize        *int64        `json:"maxTableSize" yaml:"maxTableSize" env:"MAXTABLESIZE"`                      // Maximum size in bytes for each LSM table or file.
	LevelOneSize        *int64        `json:"levelOneSize" yaml:"levelOneSize" env:"LEVELONESIZE"`                      // Maximum total size in bytes for Level 1. defaults to 1 Million
	LevelSizeMultiplier *int          `json:"levelSizeMultiplier" yaml:"levelSizeMultiplier" env:"LEVELSIZEMULTIPLIER"` // The ratio between the maximum sizes of contiguous levels in the LSM. defaults to 10
	MaxLevels           *int          `json:"maxLevels" yaml:"maxLevels" env:"MAXLEVELS"`                               // Maximum number of levels of compaction allowed in the LSM. defaults to 7
	Default             BadgerDefault `json:"default" yaml:"default" env:"DEFAULT"`                                     // default badger option to optimize for storage, ingestion or default that badger provides
}

Badger configures badger K-V store that we use underlying. This will help to tune the options to optimize for various uses cases like bigger files, point query or range queries.

type BadgerDefault

type BadgerDefault string

BadgerDefault represents a default way of configuring badger

const (
	BadgerStorage       BadgerDefault = "storage"
	BadgerIngestion     BadgerDefault = "ingestion"
	BadgerDefaultOption BadgerDefault = "default"
)

Various Badger options

type BigQuerySink

type BigQuerySink struct {
	Project string `json:"project" yaml:"project" env:"PROJECT"` // The project ID
	Dataset string `json:"dataset" yaml:"dataset" env:"DATASET"` // The dataset ID
	Table   string `json:"table" yaml:"table" env:"TABLE"`       // The table ID
}

BigQuerySink represents a sink to Google Big Query

type Compaction

type Compaction struct {
	Sinks    `yaml:",inline"`
	Encoder  string `json:"encoder" yaml:"encoder"`                  // The default encoder for the compaction
	NameFunc string `json:"nameFunc" yaml:"nameFunc" env:"NAMEFUNC"` // The lua script to compute file name given a row
	Interval int    `json:"interval" yaml:"interval" env:"INTERVAL"` // The compaction interval, in seconds
}

Compaction represents a configuration for compaction sinks

type Computed

type Computed struct {
	Name string      `json:"name"`
	Type typeof.Type `json:"type"`
	Func string      `json:"func"`
}

Computed represents a computed column

type Config

type Config struct {
	URI      string     `json:"uri" yaml:"uri" env:"URI"`
	Env      string     `json:"env" yaml:"env" env:"ENV"`             // The environment (eg: prd, stg)
	AppName  string     `json:"appName" yaml:"appName" env:"APPNAME"` // app name used for monitoring
	Domain   string     `json:"domain" yaml:"domain" env:"DOMAIN"`
	Readers  Readers    `json:"readers" yaml:"readers" env:"READERS"`
	Writers  Writers    `json:"writers" yaml:"writers" env:"WRITERS"`
	Storage  Storage    `json:"storage" yaml:"storage" env:"STORAGE"`
	Tables   Tables     `json:"tables" yaml:"tables"`
	Statsd   *StatsD    `json:"statsd,omitempty" yaml:"statsd" env:"STATSD"`
	Computed []Computed `json:"computed" yaml:"computed" env:"COMPUTED"`
	K8s      *K8s       `json:"k8s,omitempty" yaml:"k8s" env:"K8S"`
}

Config global

type Configurer

type Configurer interface {
	Configure(*Config) error
}

Configurer is an interface for any component which will provide the config.

type FileSink

type FileSink struct {
	Directory string `json:"dir" yaml:"dir" env:"DIR"`
}

FileSink represents a sink to the local file system

type Func

type Func func() *Config

Func represents a config function

func Load

func Load(ctx context.Context, d time.Duration, configurers ...Configurer) Func

Load iterates through all the providers and fills the config object. Order of providers is important as the the last provider can override the previous one It sets watch on the config for hot reload of the config

type GCSSink

type GCSSink struct {
	Bucket string `json:"bucket" yaml:"bucket" env:"BUCKET"` // The name of the bucket
	Prefix string `json:"prefix" yaml:"prefix" env:"PREFIX"` // The prefix to add
}

GCSSink represents a sink to Google Cloud Storage

type GRPC

type GRPC struct {
	Port int32 `json:"port" yaml:"port" env:"PORT"` // The port for the gRPC listener (default: 8080)
}

GRPC represents the configuration for gRPC ingress

type K8s

type K8s struct {
	ProbePort int32 `json:"probePort" yaml:"probePort" env:"PROBEPORT"` // The port which is used for liveness and readiness probes (default: 8080)
}

type Presto

type Presto struct {
	Port   int32  `json:"port" yaml:"port" env:"PORT"`
	Schema string `json:"schema" yaml:"schema" env:"SCHEMA"`
}

Presto represents the Presto configuration

type PubSubSink

type PubSubSink struct {
	Project string `json:"project" yaml:"project" env:"PROJECT"`
	Topic   string `json:"topic" yaml:"topic" env:"TOPIC"`
	Filter  string `json:"filter" yaml:"filter" env:"FILTER"`
	Encoder string `json:"encoder" yaml:"encoder" env:"ENCODER"`
}

PubSubSink represents a stream to Google Pub/Sub

type Readers

type Readers struct {
	Presto *Presto `json:"presto" yaml:"presto" env:"PRESTO"`
}

Readers are ways to read the data

type S3SQS

type S3SQS struct {
	Region            string `json:"region" yaml:"region" env:"REGION"`
	Queue             string `json:"queue" yaml:"queue" env:"QUEUE"`
	WaitTimeout       int64  `json:"waitTimeout,omitempty" yaml:"waitTimeout" env:"WAITTIMEOUT"`                   // in seconds
	VisibilityTimeout int64  `json:"visibilityTimeout,omitempty" yaml:"visibilityTimeout" env:"VISIBILITYTIMEOUT"` // in seconds
	Retries           int    `json:"retries" yaml:"retries" env:"RETRIES"`
}

S3SQS represents the aws S3 SQS configuration

type S3Sink

type S3Sink struct {
	Region      string `json:"region" yaml:"region" env:"REGION"`                // The region of AWS bucket
	Bucket      string `json:"bucket" yaml:"bucket" env:"BUCKET"`                // The name of AWS bucket
	Prefix      string `json:"prefix" yaml:"prefix" env:"PREFIX"`                // The prefix to add
	Endpoint    string `json:"endpoint" yaml:"endpoint" env:"ENDPOINT"`          // The custom endpoint to use
	SSE         string `json:"sse" yaml:"sse" env:"SSE"`                         // The server side encryption to use
	AccessKey   string `json:"accessKey" yaml:"accessKey" env:"ACCESSKEY"`       // The optional static access key
	SecretKey   string `json:"secretKey" yaml:"secretKey" env:"SECRETKEY"`       // The optional static secret key
	Concurrency int    `json:"concurrency" yaml:"concurrency" env:"CONCURRENCY"` // The S3 upload concurrency
}

S3Sink represents a sink for AWS S3 and compatible stores.

type Sinks

type Sinks struct {
	S3       *S3Sink       `json:"s3" yaml:"s3"`              // The S3 writer configuration
	Azure    *AzureSink    `json:"azure" yaml:"azure"`        // The Azure writer configuration
	BigQuery *BigQuerySink `json:"bigquery" yaml:"bigquery" ` // The Big Query writer configuration
	GCS      *GCSSink      `json:"gcs" yaml:"gcs" `           // The Google Cloud Storage writer configuration
	File     *FileSink     `json:"file" yaml:"file" `         // The local file system writer configuration
	Talaria  *TalariaSink  `json:"talaria" yaml:"talaria" `   // The Talaria writer configuration
	PubSub   *PubSubSink   `json:"pubsub" yaml:"pubsub" `     // The Google Pub/Sub writer configuration
}

Sinks represents a configuration for writer sinks

type StatsD

type StatsD struct {
	Host string `json:"host" yaml:"host" env:"HOST"`
	Port int64  `json:"port" port:"port" env:"PORT"`
}

StatsD represents the configuration for statsD client

type Storage

type Storage struct {
	Badger
	Directory string `json:"dir" yaml:"dir" env:"DIR"`
}

Storage is the location to write the data

type Streams

type Streams []Sinks

Streams are lists of sinks to be streamed to

type Table

type Table struct {
	TTL     int64       `json:"ttl,omitempty" yaml:"ttl" env:"TTL"`          // The ttl (in seconds) for the storage, defaults to 1 hour.
	HashBy  string      `json:"hashBy,omitempty" yaml:"hashBy" env:"HASHBY"` // The column to use as key (metric), defaults to 'event'.
	SortBy  string      `json:"sortBy,omitempty" yaml:"sortBy" env:"SORTBY"` // The column to use as time, defaults to 'tsi'.
	Schema  string      `json:"schema" yaml:"schema" env:"SCHEMA"`           // The schema of the table
	Compact *Compaction `json:"compact" yaml:"compact" env:"COMPACT"`        // The compaction configuration for the table
	Streams Streams     `json:"streams" yaml:"streams" env:"STREAMS"`        // The streams to stream data to for data in this table
}

Table is the config for the timeseries table

type Tables

type Tables map[string]Table

Tables is a list of table configs

type TalariaSink

type TalariaSink struct {
	Endpoint              string         `json:"endpoint" yaml:"endpoint" env:"ENDPOINT"`                    // The second Talaria endpoint
	CircuitTimeout        *time.Duration `json:"timeout" yaml:"timeout" env:"TIMEOUT"`                       // The timeout (in seconds) for requests to the second Talaria
	MaxConcurrent         *int           `json:"concurrency" yaml:"concurrency" env:"CONCURRENCY"`           // The number of concurrent requests permissible
	ErrorPercentThreshold *int           `json:"errorThreshold" yaml:"errorThreshold" env:"ERROR_THRESHOLD"` // The percentage of failed requests tolerated
}

TalariaSink represents a sink to an instance of Talaria

type Writers

type Writers struct {
	GRPC  *GRPC  `json:"grpc,omitempty" yaml:"grpc" env:"GRPC"`    // The GRPC ingress
	S3SQS *S3SQS `json:"s3sqs,omitempty" yaml:"s3sqs" env:"S3SQS"` // The S3SQS ingress
}

Writers are sources to write data