crawler

Documentation

Index

• Constants
• type Crawler
  • func (c *Crawler) Start(prospectorConfigs []*common.Config, eventChan chan *input.FileEvent) error
  • func (c *Crawler) Stop()
• type Prospector
  • func NewProspector(cfg *common.Config, states input.States, spoolerChan chan *input.FileEvent) (*Prospector, error)
  • func (p *Prospector) Init() error
  • func (p *Prospector) Run()
  • func (p *Prospector) Stop(wg *sync.WaitGroup)
• type ProspectorLog
  • func NewProspectorLog(p *Prospector) (*ProspectorLog, error)
  • func (p *ProspectorLog) Init()
  • func (p *ProspectorLog) Run()
• type ProspectorStdin
  • func NewProspectorStdin(p *Prospector) (*ProspectorStdin, error)
  • func (p *ProspectorStdin) Init()
  • func (p *ProspectorStdin) Run()
• type Prospectorer
• type Registrar
  • func NewRegistrar(registryFile string) (*Registrar, error)
  • func (r *Registrar) Init() error
  • func (r *Registrar) LoadState() error
  • func (r *Registrar) Run()
  • func (r *Registrar) Start()
  • func (r *Registrar) Stop()

Constants

const (
	DefaultIgnoreOlder   time.Duration = 0
	DefaultScanFrequency time.Duration = 10 * time.Second
)

Types

type Crawler

type Crawler struct {
	// Registrar object to persist the state
	Registrar *Registrar
	// contains filtered or unexported fields
}

func (*Crawler) Start

func (c *Crawler) Start(prospectorConfigs []*common.Config, eventChan chan *input.FileEvent) error

func (*Crawler) Stop

func (c *Crawler) Stop()

type Prospector

type Prospector struct {
	// contains filtered or unexported fields
}

func NewProspector

func NewProspector(cfg *common.Config, states input.States, spoolerChan chan *input.FileEvent) (*Prospector, error)

func (*Prospector) Init

func (p *Prospector) Init() error

Init sets up default config for prospector

func (*Prospector) Run

func (p *Prospector) Run()

Starts scanning through all the file paths and fetch the related files. Start a harvester for each file

func (*Prospector) Stop

func (p *Prospector) Stop(wg *sync.WaitGroup)

type ProspectorLog

type ProspectorLog struct {
	Prospector *Prospector
	// contains filtered or unexported fields
}

func NewProspectorLog

func NewProspectorLog(p *Prospector) (*ProspectorLog, error)

func (*ProspectorLog) Init

func (p *ProspectorLog) Init()

func (*ProspectorLog) Run

func (p *ProspectorLog) Run()

type ProspectorStdin

type ProspectorStdin struct {
	Prospector *Prospector
	// contains filtered or unexported fields
}

func NewProspectorStdin

func NewProspectorStdin(p *Prospector) (*ProspectorStdin, error)

func (*ProspectorStdin) Init

func (p *ProspectorStdin) Init()

func (*ProspectorStdin) Run

func (p *ProspectorStdin) Run()

type Prospectorer

type Prospectorer interface {
	Init()
	Run()
}

type Registrar

type Registrar struct {
	Channel chan []*FileEvent
	// contains filtered or unexported fields
}

func NewRegistrar

func NewRegistrar(registryFile string) (*Registrar, error)

func (*Registrar) Init

func (r *Registrar) Init() error

Init sets up the Registrar and make sure the registry file is setup correctly

func (*Registrar) LoadState

func (r *Registrar) LoadState() error

loadState fetches the previous reading state from the configure RegistryFile file The default file is `registry` in the data path.

func (*Registrar) Run

func (r *Registrar) Run()

func (*Registrar) Start

func (r *Registrar) Start()

func (*Registrar) Stop

func (r *Registrar) Stop()

Stop stops the registry. It waits until Run function finished.