crawler

Documentation

Index

• type Crawler
  • func New(id int) Crawler
• type CrawlerPool
  • func NewCrawlerPool() CrawlerPool
• type SpiderQueue
  • func NewSpiderQueue() SpiderQueue

Types

type Crawler

type Crawler interface {
	Init(*spider.Spider) Crawler //初始化采集引擎
	Run()                        //运行任务
	Stop()                       //主动终止
	CanStop() bool               //能否终止
	GetId() int                  //获取引擎ID
}

采集引擎

func New

func New(id int) Crawler

type CrawlerPool

type CrawlerPool interface {
	Reset(spiderNum int) int
	Use() Crawler
	Free(Crawler)
	Stop()
}

采集引擎池

func NewCrawlerPool

func NewCrawlerPool() CrawlerPool

type SpiderQueue

type SpiderQueue interface {
	Reset() //重置清空队列
	Add(*Spider)
	AddAll([]*Spider)
	AddKeyins(string) //为队列成员遍历添加Keyin属性，但前提必须是队列成员未被添加过keyin
	GetByIndex(int) *Spider
	GetByName(string) *Spider
	GetAll() []*Spider
	Len() int // 返回队列长度
}

采集引擎中规则队列

func NewSpiderQueue

func NewSpiderQueue() SpiderQueue