crawler

Documentation

Index

• func Start(crawler *Crawler, cfg *Config, logger log.Logger)
• type Config
• type Crawler
  • func NewCrawler(cfg *Config, state *state.State, rpc *common.RPCClient, writer *kafka.Writer, ...) *Crawler
  • func (c *Crawler) RunLoop()

Functions

func Start

func Start(crawler *Crawler, cfg *Config, logger log.Logger)

Types

type Config

type Config struct {
	Interval    string `json:"interval"`
	MaxRoutines int    `json:"routines"`
	CacheLimit  int    `json:"cache"`
	Start       uint64 `json:"start"`
}

type Crawler

type Crawler struct {
	// contains filtered or unexported fields
}

func NewCrawler

func NewCrawler(cfg *Config, state *state.State, rpc *common.RPCClient, writer *kafka.Writer, logger log.Logger) *Crawler

func (*Crawler) RunLoop

func (c *Crawler) RunLoop()