internal

Documentation

Index

• func ConfigFetcher(proxyFunc proxy.Func, timeout int, logger *zap.Logger) spider.Fetcher
• func ConfigLogger(cfg config.Config) (*zap.Logger, error)
• func ConfigProxyFunc(cfg config.Config, logger *zap.Logger) (proxy.Func, int, error)
• func ConfigStorager(cfg config.Config, logger *zap.Logger) (spider.Storage, error)
• func ConfigTasks(cfg config.Config, f spider.Fetcher, storager spider.Storage, ...) ([]*spider.Task, error)
• func ConfigWorkerEngine(sconfig *ServerConfig, seeds []*spider.Task, f spider.Fetcher, ...) (*engine.Crawler, error)
• func LoadConfig(cfgFile string) (config.Config, error)
• func ParseTaskConfig(logger *zap.Logger, f spider.Fetcher, s spider.Storage, ...) []*spider.Task
• func RunGRPCServer(hdlr *master.Master, logger *zap.Logger, cfg ServerConfig, ...)
• func RunGRPCServerForMaster(hdlr *master.Master, logger *zap.Logger, cfg ServerConfig, ...)
• func RunHTTPServer(logger *zap.Logger, cfg ServerConfig)
• type Greeter
  • func (g *Greeter) Hello(ctx context.Context, req *greeter.Request, resp *greeter.Response) error
• type ServerConfig
  • func ConfigMasterServer(cfg config.Config, logger *zap.Logger) (*ServerConfig, error)
  • func ConfigWorkerServer(cfg config.Config, logger *zap.Logger) (*ServerConfig, error)

Functions

func ConfigFetcher

func ConfigFetcher(proxyFunc proxy.Func, timeout int, logger *zap.Logger) spider.Fetcher

func ConfigLogger

func ConfigLogger(cfg config.Config) (*zap.Logger, error)

func ConfigProxyFunc

func ConfigProxyFunc(cfg config.Config, logger *zap.Logger) (proxy.Func, int, error)

func ConfigStorager

func ConfigStorager(cfg config.Config, logger *zap.Logger) (spider.Storage, error)

func ConfigTasks

func ConfigTasks(cfg config.Config, f spider.Fetcher, storager spider.Storage, logger *zap.Logger) ([]*spider.Task, error)

func ConfigWorkerEngine

func ConfigWorkerEngine(sconfig *ServerConfig, seeds []*spider.Task, f spider.Fetcher, storager spider.Storage, logger *zap.Logger) (*engine.Crawler, error)

func LoadConfig

func LoadConfig(cfgFile string) (config.Config, error)

func ParseTaskConfig

func ParseTaskConfig(logger *zap.Logger, f spider.Fetcher, s spider.Storage, cfgs []spider.TaskConfig) []*spider.Task

func RunGRPCServer

func RunGRPCServer(hdlr *master.Master, logger *zap.Logger, cfg ServerConfig, reg registry.Registry)

func RunGRPCServerForMaster

func RunGRPCServerForMaster(hdlr *master.Master, logger *zap.Logger, cfg ServerConfig, reg registry.Registry)

func RunHTTPServer

func RunHTTPServer(logger *zap.Logger, cfg ServerConfig)

Types

type Greeter

type Greeter struct{}

func (*Greeter) Hello

func (g *Greeter) Hello(ctx context.Context, req *greeter.Request, resp *greeter.Response) error

type ServerConfig

type ServerConfig struct {
	GRPCListenAddr   string
	HTTPListenAddr   string
	ID               string
	RegistryAddr     string
	RegisterTTL      int
	RegisterInterval int
	Name             string
	ClientTimeOut    int
}

func ConfigMasterServer

func ConfigMasterServer(cfg config.Config, logger *zap.Logger) (*ServerConfig, error)

func ConfigWorkerServer

func ConfigWorkerServer(cfg config.Config, logger *zap.Logger) (*ServerConfig, error)