output

package
v0.1.0-alpha Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 19, 2021 License: MIT Imports: 32 Imported by: 0

Documentation

Overview

Package output provides output components

Index

Constants

This section is empty.

Variables

View Source 
var All = []baker.OutputDesc{
	DynamoDBDesc,
	FileWriterDesc,
	NopDesc,
	OpLogDesc,
	StatsDesc,
	WebSocketDesc,
}

All is the list of all baker outputs.

View Source 
var BakerTransport = http.DefaultTransport.(*http.Transport).Clone()
View Source 
var DynamoDBDesc = baker.OutputDesc{
	Name:   "DynamoDB",
	New:    NewDynamoDB,
	Config: &DynamoDBConfig{},
	Raw:    false,
	Help: "This output writes the filtered log lines to DynamoDB. It must be\n" +
		"configured specifying the region, the table name, and the columns\n" +
		"to write.\nColumns are specified using the syntax \"t:name\" where \"t\"\n" +
		"is the type of the data, and \"name\" is the name of column. Supported\n" +
		"types are: \"n\" - integers; \"s\" - strings.\n" +
		"The first column (and field) must be the primary key.\n",
}
View Source 
var DynamoGlobals dynamoGlobals
View Source 
var FileWriterDesc = baker.OutputDesc{
	Name:   "FileWriter",
	New:    NewFileWriter,
	Config: &FileWriterConfig{},
	Raw:    true,
	Help:   helpMsg,
}
View Source 
var NopDesc = baker.OutputDesc{
	Name:   "Nop",
	New:    NewNop,
	Config: &NopConfig{},
	Help:   "No-operation output. This output simply drops all lines and does not write them anywhere.",
}
View Source 
var OpLogDesc = baker.OutputDesc{
	Name:   "OpLog",
	New:    NewOpLog,
	Config: &OpLogConfig{},
	Raw:    false,
	Help:   "This output writes the filtered log lines into the current baker log, purely for development purpose.\n",
}
View Source 
var SQLiteDesc = baker.OutputDesc{
	Name:   "SQLite",
	New:    NewSQLite(false),
	Config: &SQLiteConfig{},
	Help:   "Writes a chosen set of fields as table columns into a local SQLite database file",
	Raw:    false,
}

SQLiteDesc declares the standard (non-raw) SQLite output.

View Source 
var SQLiteRawDesc = baker.OutputDesc{
	Name:   "SQLiteRaw",
	New:    NewSQLite(true),
	Config: &SQLiteRawWriterConfig{},
	Help:   "Writes a chosen set of fields, plus the raw record, as table columns into a local SQLite database file",
	Raw:    true,
}

SQLiteRawDesc declares the raw SQLite output.

View Source 
var StatsDesc = baker.OutputDesc{
	Name:   "Stats",
	New:    NewStats,
	Config: &StatsConfig{},
	Raw:    true,
	Help: "Compute various distributions of the records it " +
		"receives and dumps that to CSV. It computes the " +
		"distribution of record by size and the distribution " +
		"of the values of certain fields\n",
}
View Source 
var WebSocketDesc = baker.OutputDesc{
	Name:   "WebSocket",
	New:    NewWebSocket,
	Config: &WebSocketConfig{},
	Raw:    false,
	Help:   "This output writes the filtered log lines into any connected WebSocket client.\n",
}

Functions

func NewDynamoDB 

func NewDynamoDB(cfg baker.OutputParams) (baker.Output, error)

NewDynamoDB create a new DynamoDB output.

TableName is the name of the DynamoDB table to be written. Columns is a slice listing the columns that will be written; the first item in the slice *MUST* be the primary key of the table.

func NewFileWriter 

func NewFileWriter(cfg baker.OutputParams) (baker.Output, error)

func NewNop 

func NewNop(cfg baker.OutputParams) (baker.Output, error)

func NewOpLog 

func NewOpLog(cfg baker.OutputParams) (baker.Output, error)

func NewSQLite 

func NewSQLite(isRaw bool) func(baker.OutputParams) (baker.Output, error)

func NewStats 

func NewStats(cfg baker.OutputParams) (baker.Output, error)

NewStats returns a new Stats Baker output.

func NewWebSocket 

func NewWebSocket(cfg baker.OutputParams) (baker.Output, error)

Types

type DynamoDB 

type DynamoDB struct {
	TableName string
	Fields    []baker.FieldIndex
	Columns   []string
	Cfg       *DynamoDBConfig
	// contains filtered or unexported fields
}

DynamoDB is a class to do optimized batched writes to a single DynamoDB table with a fixed schema (same number of columns for all records).

func (*DynamoDB) CanShard 

func (b *DynamoDB) CanShard() bool

func (*DynamoDB) Flush 

func (b *DynamoDB) Flush()

func (*DynamoDB) NumProcessedRecords 

func (b *DynamoDB) NumProcessedRecords() int64

func (*DynamoDB) Run 

func (b *DynamoDB) Run(input <-chan baker.OutputRecord, _ chan<- string) error

func (*DynamoDB) Stats 

func (b *DynamoDB) Stats() baker.OutputStats

type DynamoDBConfig 

type DynamoDBConfig struct {
	Regions         []string      `help:"DynamoDB regions to connect to" default:"us-west-2"`
	Table           string        `help:"Name of the table to modify" required:"true"`
	Columns         []string      `help:"Table columns that correspond to each of the fields being written"`
	FlushInterval   time.Duration `help:"Interval at which flush the data to DynamoDB even if we have not reached 25 records" default:"1s"`
	MaxWritesPerSec int64         `help:"Maximum number of writes per second that DynamoDB can accept (0 for unlimited)" default:"0"`
	MaxBackoff      time.Duration `help:"Maximum retry/backoff time in case of errors before giving up" default:"2m"`
	// contains filtered or unexported fields
}

type FileWriter 

type FileWriter struct {
	Cfg *FileWriterConfig

	Fields []baker.FieldIndex
	// contains filtered or unexported fields
}

func (*FileWriter) CanShard 

func (w *FileWriter) CanShard() bool

func (*FileWriter) Run 

func (w *FileWriter) Run(input <-chan baker.OutputRecord, upch chan<- string) error

func (*FileWriter) Stats 

func (w *FileWriter) Stats() baker.OutputStats

type FileWriterConfig 

type FileWriterConfig struct {
	PathString           string        `` /* 182-byte string literal not displayed */
	RotateInterval       time.Duration `help:"Time after which data will be rotated. If -1, it will not rotate until the end." default:"60s"`
	ZstdCompressionLevel int           `help:"zstd compression level, ranging from 1 (best speed) to 19 (best compression)." default:"3"`
	ZstdWindowLog        int           `` /* 183-byte string literal not displayed */
}

type Nop 

type Nop struct {
	// contains filtered or unexported fields
}

func (*Nop) CanShard 

func (b *Nop) CanShard() bool

func (*Nop) Run 

func (nop *Nop) Run(input <-chan baker.OutputRecord, upch chan<- string) error

func (*Nop) Stats 

func (nop *Nop) Stats() baker.OutputStats

type NopConfig 

type NopConfig struct{}

type OpLog 

type OpLog struct {
	Cfg *OpLogConfig

	Fields []baker.FieldIndex
	// contains filtered or unexported fields
}

func (*OpLog) CanShard 

func (b *OpLog) CanShard() bool

func (*OpLog) Run 

func (w *OpLog) Run(input <-chan baker.OutputRecord, _ chan<- string) error

func (*OpLog) Stats 

func (w *OpLog) Stats() baker.OutputStats

type OpLogConfig 

type OpLogConfig struct{}

type SQLite 

type SQLite struct {
	// contains filtered or unexported fields
}

func (*SQLite) CanShard 

func (c *SQLite) CanShard() bool

func (*SQLite) Run 

func (c *SQLite) Run(input <-chan baker.OutputRecord, upch chan<- string) error

func (*SQLite) Stats 

func (c *SQLite) Stats() baker.OutputStats

type SQLiteConfig 

type SQLiteConfig struct {
	PathString string   `` /* 167-byte string literal not displayed */
	TableName  string   `help:"Table name to which to write the records to." required:"true"`
	PreRun     []string `help:"List of SQL statements to run at startup (before table creation)."`
	PostRun    []string `help:"List of SQL statements to run at exit. (good place to create indexes if needed)."`
	Clear      bool     `` /* 233-byte string literal not displayed */
	Vacuum     bool     `` /* 259-byte string literal not displayed */
	Wal        bool     `` /* 167-byte string literal not displayed */
	PageSize   int64    `help:"The page size to use for SQLite. By default, we use whatever SQLite decides to use as default."`
}

SQLiteConfig holds the configuration parameters for the standard SQLite baker output.

type SQLiteRawWriterConfig 

type SQLiteRawWriterConfig struct {
	PathString     string   `` /* 167-byte string literal not displayed */
	TableName      string   `help:"Table name to which to write the records to." required:"true"`
	PreRun         []string `help:"List of SQL statements to run at startup (before table creation)."`
	PostRun        []string `help:"List of SQL statements to run at exit. (good place to create indexes if needed)."`
	Clear          bool     `` /* 233-byte string literal not displayed */
	Vacuum         bool     `` /* 259-byte string literal not displayed */
	Wal            bool     `` /* 167-byte string literal not displayed */
	PageSize       int64    `help:"The page size to use for SQLite. By default, we use whatever SQLite decides to use as default."`
	RecordBlobName string   `help:"Name of the column in which the whole raw record should be put." required:"true"`
}

SQLiteRawWriterConfig holds the configuration parameters for the raw SQLite baker output.

type Stats 

type Stats struct {
	// contains filtered or unexported fields
}

The Stats output gathers statistics about the sizes of log lines it sees.

func (*Stats) CanShard 

func (s *Stats) CanShard() bool

CanShard implements baker.Output

func (*Stats) Run 

func (s *Stats) Run(input <-chan baker.OutputRecord, _ chan<- string) error

Run implements baker.Output

func (*Stats) Stats 

func (s *Stats) Stats() baker.OutputStats

Stats implements baker.Output

type StatsConfig 

type StatsConfig struct {
	CountEmptyFields bool   `help:"Whether fields with empty values are counted or not" default:"false"`
	CSVPath          string `help:"Path of the CSV file to create" default:"stats.csv"`
	TimestampField   string `help:"Name of a field containing a POSIX timestamp (in seconds) used to build the times stats" required:"true"`
}

type WebSocket 

type WebSocket struct {
	Cfg *WebSocketConfig

	Fields []baker.FieldIndex
	// contains filtered or unexported fields
}

func (*WebSocket) CanShard 

func (b *WebSocket) CanShard() bool

func (*WebSocket) Run 

func (w *WebSocket) Run(input <-chan baker.OutputRecord, _ chan<- string) error

func (*WebSocket) Stats 

func (w *WebSocket) Stats() baker.OutputStats

type WebSocketConfig 

type WebSocketConfig struct{}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.