harvester

Documentation

Index

• type CloseChan
• type Config
  • func GetConfig() *Config
  • func (c *Config) GetDescription() []*Definition
  • func (c *Config) Load(ini string)
  • func (c *Config) LoadFile(filename string)
• type Container
  • func GetContainer() *Container
  • func (c *Container) GetFormat(key string) intf.Format
  • func (c *Container) GetInput(key string) intf.Input
  • func (c *Container) GetOutput(key string) intf.Output
  • func (c *Container) GetPostProcessor(key string) intf.PostProcessor
  • func (c *Container) GetReader(key string) *Reader
  • func (c *Container) GetWriter(key string) *Writer
  • func (c *Container) GetWriterGroup() *WriterGroup
• type Definition
• type FieldDefinition
• type Harvester
  • func NewHarvester() *Harvester
  • func (h *Harvester) Boot()
  • func (h *Harvester) Configure(filename string)
  • func (h *Harvester) PrintCounters(elapsedSeconds int)
  • func (h *Harvester) Run()
• type OutputsFactory
• type Reader
  • func NewReader() *Reader
  • func (r *Reader) GoRead()
  • func (r *Reader) SetChannels(recordsChan RecordsChan, closeChan CloseChan)
  • func (r *Reader) SetInputs(inputs []intf.Input)
  • func (r *Reader) SetProcessors(processors []intf.PostProcessor)
  • func (r *Reader) Teardown()
• type ReaderConfig
• type RecordsChan
• type Writer
  • func NewWriter() *Writer
  • func (w *Writer) Boot()
  • func (w *Writer) GetChannels() (RecordsChan, CloseChan)
  • func (w *Writer) GetCounters() (int32, int32, int32, int32)
  • func (w *Writer) IsAlive() bool
  • func (w *Writer) ResetCounters()
  • func (w *Writer) SetOutputsFactory(factory OutputsFactory)
  • func (w *Writer) SetReaders(readers []*Reader)
  • func (w *Writer) SetThreads(threads int)
  • func (w *Writer) Setup()
  • func (w *Writer) Teardown()
• type WriterConfig
• type WriterGroup
  • func NewWriterGroup() *WriterGroup
  • func (wg *WriterGroup) Boot()
  • func (wg *WriterGroup) GetCounters() (int32, int32, int32, int32)
  • func (wg *WriterGroup) IsAlive() bool
  • func (wg *WriterGroup) ResetCounters()
  • func (wg *WriterGroup) SetWriters(writers []intf.Writer)
  • func (wg *WriterGroup) Setup()
  • func (wg *WriterGroup) Teardown()

Types

type CloseChan

type CloseChan chan bool

type Config

type Config struct {
	Logger               *logger.LoggerConfig
	Writer               map[string]*WriterConfig
	Reader               map[string]*ReaderConfig
	Format_JSON          map[string]*format.JSONConfig
	Format_CSV           map[string]*format.CSVConfig
	Format_RegExp        map[string]*format.RegExpConfig
	Format_Apache2       map[string]*format.Apache2Config
	Format_Nginx         map[string]*format.NginxConfig
	Input_File           map[string]*input.FileConfig
	Input_Tail           map[string]*input.TailConfig
	Input_S3             map[string]*input.S3Config
	Input_Mongo          map[string]*input.MongoConfig
	Output_HTTP          map[string]*output.HTTPConfig
	Output_Elasticsearch map[string]*output.ElasticsearchConfig
	Output_Mongo         map[string]*output.MongoConfig
	Output_Dummy         map[string]*output.DummyConfig
	Processor_Anonymize  map[string]*processor.AnonymizeConfig
	Processor_Metrics    map[string]*processor.MetricsConfig
	Processor_Mutate     map[string]*processor.MutateConfig
}

func GetConfig

func GetConfig() *Config

func (*Config) GetDescription

func (c *Config) GetDescription() []*Definition

func (*Config) Load

func (c *Config) Load(ini string)

func (*Config) LoadFile

func (c *Config) LoadFile(filename string)

type Container

type Container struct {
}

func GetContainer

func GetContainer() *Container

func (*Container) GetFormat

func (c *Container) GetFormat(key string) intf.Format

func (*Container) GetInput

func (c *Container) GetInput(key string) intf.Input

func (*Container) GetOutput

func (c *Container) GetOutput(key string) intf.Output

func (*Container) GetPostProcessor

func (c *Container) GetPostProcessor(key string) intf.PostProcessor

func (*Container) GetReader

func (c *Container) GetReader(key string) *Reader

func (*Container) GetWriter

func (c *Container) GetWriter(key string) *Writer

func (*Container) GetWriterGroup

func (c *Container) GetWriterGroup() *WriterGroup

type Definition

type Definition struct {
	Name          string
	AllowMultiple bool
	Fields        []*FieldDefinition
}

type FieldDefinition

type FieldDefinition struct {
	Name        string
	Type        string
	Description string
	Default     string
}

type Harvester

type Harvester struct {
	// contains filtered or unexported fields
}

func NewHarvester

func NewHarvester() *Harvester

func (*Harvester) Boot

func (h *Harvester) Boot()

func (*Harvester) Configure

func (h *Harvester) Configure(filename string)

func (*Harvester) PrintCounters

func (h *Harvester) PrintCounters(elapsedSeconds int)

func (*Harvester) Run

func (h *Harvester) Run()

type OutputsFactory

type OutputsFactory func() []intf.Output

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

func NewReader

func NewReader() *Reader

func (*Reader) GoRead

func (r *Reader) GoRead()

func (*Reader) SetChannels

func (r *Reader) SetChannels(recordsChan RecordsChan, closeChan CloseChan)

func (*Reader) SetInputs

func (r *Reader) SetInputs(inputs []intf.Input)

func (*Reader) SetProcessors

func (r *Reader) SetProcessors(processors []intf.PostProcessor)

func (*Reader) Teardown

func (r *Reader) Teardown()

type ReaderConfig

type ReaderConfig struct {
	Input     []string
	Processor []string
}

type RecordsChan

type RecordsChan chan intf.Record

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

func NewWriter

func NewWriter() *Writer

func (*Writer) Boot

func (w *Writer) Boot()

func (*Writer) GetChannels

func (w *Writer) GetChannels() (RecordsChan, CloseChan)

func (*Writer) GetCounters

func (w *Writer) GetCounters() (int32, int32, int32, int32)

func (*Writer) IsAlive

func (w *Writer) IsAlive() bool

func (*Writer) ResetCounters

func (w *Writer) ResetCounters()

func (*Writer) SetOutputsFactory

func (w *Writer) SetOutputsFactory(factory OutputsFactory)

func (*Writer) SetReaders

func (w *Writer) SetReaders(readers []*Reader)

func (*Writer) SetThreads

func (w *Writer) SetThreads(threads int)

func (*Writer) Setup

func (w *Writer) Setup()

func (*Writer) Teardown

func (w *Writer) Teardown()

type WriterConfig

type WriterConfig struct {
	Output  []string
	Reader  []string
	Threads int
}

type WriterGroup

type WriterGroup struct {
	// contains filtered or unexported fields
}

func NewWriterGroup

func NewWriterGroup() *WriterGroup

func (*WriterGroup) Boot

func (wg *WriterGroup) Boot()

func (*WriterGroup) GetCounters

func (wg *WriterGroup) GetCounters() (int32, int32, int32, int32)

func (*WriterGroup) IsAlive

func (wg *WriterGroup) IsAlive() bool

func (*WriterGroup) ResetCounters

func (wg *WriterGroup) ResetCounters()

func (*WriterGroup) SetWriters

func (wg *WriterGroup) SetWriters(writers []intf.Writer)

func (*WriterGroup) Setup

func (wg *WriterGroup) Setup()

func (*WriterGroup) Teardown

func (wg *WriterGroup) Teardown()