crawler

package
v0.0.0-...-320d922 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 10, 2018 License: Apache-2.0 Imports: 10 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Crawler

type Crawler struct {
	// Registrar object to persist the state
	Registrar *Registrar
	// contains filtered or unexported fields
}

func (*Crawler) Start

func (c *Crawler) Start(prospectorConfigs []config.ProspectorConfig, eventChan chan *input.FileEvent) error

func (*Crawler) Stop

func (c *Crawler) Stop()

type Prospector

type Prospector struct {
	ProspectorConfig cfg.ProspectorConfig
	// contains filtered or unexported fields
}

func NewProspector

func NewProspector(prospectorConfig cfg.ProspectorConfig, registrar *Registrar, channel chan *input.FileEvent) (*Prospector, error)

func (*Prospector) AddHarvester

func (p *Prospector) AddHarvester(file string, stat *harvester.FileStat) (*harvester.Harvester, error)

func (*Prospector) Init

func (p *Prospector) Init() error

Init sets up default config for prospector

func (*Prospector) Run

func (p *Prospector) Run(wg *sync.WaitGroup)

Starts scanning through all the file paths and fetch the related files. Start a harvester for each file

func (*Prospector) Stop

func (p *Prospector) Stop()

type ProspectorLog

type ProspectorLog struct {
	Prospector *Prospector
	// contains filtered or unexported fields
}

func NewProspectorLog

func NewProspectorLog(p *Prospector) (*ProspectorLog, error)

func (ProspectorLog) Init

func (p ProspectorLog) Init()

func (ProspectorLog) Run

func (p ProspectorLog) Run()

type ProspectorStdin

type ProspectorStdin struct {
	Prospector *Prospector
	// contains filtered or unexported fields
}

func NewProspectorStdin

func NewProspectorStdin(p *Prospector) (*ProspectorStdin, error)

func (ProspectorStdin) Init

func (p ProspectorStdin) Init()

func (ProspectorStdin) Run

func (prospector ProspectorStdin) Run()

type Prospectorer

type Prospectorer interface {
	Run()
	Init()
}

type Registrar

type Registrar struct {

	// Map with all file paths inside and the corresponding state
	State map[string]*FileState
	// Channel used by the prospector and crawler to send FileStates to be persisted
	Persist chan *input.FileState

	Channel chan []*FileEvent
	// contains filtered or unexported fields
}

func NewRegistrar

func NewRegistrar(registryFile string) (*Registrar, error)

func (*Registrar) GetFileState

func (r *Registrar) GetFileState(path string) (*FileState, bool)

func (*Registrar) Init

func (r *Registrar) Init() error

func (*Registrar) LoadState

func (r *Registrar) LoadState()

loadState fetches the previous reading state from the configure RegistryFile file The default file is .filebeat file which is stored in the same path as the binary is running

func (*Registrar) Run

func (r *Registrar) Run()

func (*Registrar) Stop

func (r *Registrar) Stop()

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL