support

Documentation

Index

• type DefaultCluster
  • func NewDefaultCluster(settings *util.Settings) *DefaultCluster
  • func (this *DefaultCluster) AddNode(nodeInfo *node.NodeInfo)
  • func (this *DefaultCluster) AddRequest(request *http.Request)
  • func (this *DefaultCluster) AddToCrawlingQuene(request *http.Request)
  • func (this *DefaultCluster) CanWeStopSpider(spiderName string) bool
  • func (this *DefaultCluster) CrawlStatus() *crawler.CrawlerStatus
  • func (this *DefaultCluster) Crawled(scrapyResult *crawler.ScrapeResult)
  • func (this *DefaultCluster) DeleteDeadNode(nodeName string)
  • func (this *DefaultCluster) ElectMaster() *node.NodeInfo
  • func (this *DefaultCluster) GetAllNode() []*node.NodeInfo
  • func (this *DefaultCluster) GetClusterInfo() *ClusterInfo
  • func (this *DefaultCluster) GetMasterName() string
  • func (this *DefaultCluster) GetMasterNode() *node.NodeInfo
  • func (this *DefaultCluster) GetRequestStatus() *RequestStatus
  • func (this *DefaultCluster) HasNode(nodeName string) bool
  • func (this *DefaultCluster) Init(node *node.NodeInfo)
  • func (this *DefaultCluster) IsMasterNode() bool
  • func (this *DefaultCluster) IsReady() bool
  • func (this *DefaultCluster) IsSpiderRunning(spiderName string) bool
  • func (this *DefaultCluster) IsStop() bool
  • func (this *DefaultCluster) Join()
  • func (this *DefaultCluster) MakeMasterNode(nodeName string)
  • func (this *DefaultCluster) PopRequest() *http.Request
  • func (this *DefaultCluster) Ready()
  • func (this *DefaultCluster) StartSpider(spiderName string)
  • func (this *DefaultCluster) StopSpider(spiderName string)

Types

type DefaultCluster

type DefaultCluster struct {
	RequestStatus *RequestStatus
	// contains filtered or unexported fields
}

func NewDefaultCluster

func NewDefaultCluster(settings *util.Settings) *DefaultCluster

func (*DefaultCluster) AddNode

func (this *DefaultCluster) AddNode(nodeInfo *node.NodeInfo)

add a node to cluster node list

func (*DefaultCluster) AddRequest

func (this *DefaultCluster) AddRequest(request *http.Request)

add a request to quene

func (*DefaultCluster) AddToCrawlingQuene

func (this *DefaultCluster) AddToCrawlingQuene(request *http.Request)

record distribute request job

func (*DefaultCluster) CanWeStopSpider

func (this *DefaultCluster) CanWeStopSpider(spiderName string) bool

func (*DefaultCluster) CrawlStatus

func (this *DefaultCluster) CrawlStatus() *crawler.CrawlerStatus

get crawl status

func (*DefaultCluster) Crawled

func (this *DefaultCluster) Crawled(scrapyResult *crawler.ScrapeResult)

a request job is done delete it from crawling quene add crawled num

func (*DefaultCluster) DeleteDeadNode

func (this *DefaultCluster) DeleteDeadNode(nodeName string)

func (*DefaultCluster) ElectMaster

func (this *DefaultCluster) ElectMaster() *node.NodeInfo

choose a new master node

func (*DefaultCluster) GetAllNode

func (this *DefaultCluster) GetAllNode() []*node.NodeInfo

func (*DefaultCluster) GetClusterInfo

func (this *DefaultCluster) GetClusterInfo() *ClusterInfo

func (*DefaultCluster) GetMasterName

func (this *DefaultCluster) GetMasterName() string

get master node name

func (*DefaultCluster) GetMasterNode

func (this *DefaultCluster) GetMasterNode() *node.NodeInfo

get master node

func (*DefaultCluster) GetRequestStatus

func (this *DefaultCluster) GetRequestStatus() *RequestStatus

func (*DefaultCluster) HasNode

func (this *DefaultCluster) HasNode(nodeName string) bool

func (*DefaultCluster) Init

func (this *DefaultCluster) Init(node *node.NodeInfo)

func (*DefaultCluster) IsMasterNode

func (this *DefaultCluster) IsMasterNode() bool

is local node master node

func (*DefaultCluster) IsReady

func (this *DefaultCluster) IsReady() bool

is cluster ready for crawl

func (*DefaultCluster) IsSpiderRunning

func (this *DefaultCluster) IsSpiderRunning(spiderName string) bool

is the spider running

func (*DefaultCluster) IsStop

func (this *DefaultCluster) IsStop() bool

is all loop stop

func (*DefaultCluster) Join

func (this *DefaultCluster) Join()

func (*DefaultCluster) MakeMasterNode

func (this *DefaultCluster) MakeMasterNode(nodeName string)

make master node by node name

func (*DefaultCluster) PopRequest

func (this *DefaultCluster) PopRequest() *http.Request

pop a request from waiting quene add to crawling quenu

func (*DefaultCluster) Ready

func (this *DefaultCluster) Ready()

func (*DefaultCluster) StartSpider

func (this *DefaultCluster) StartSpider(spiderName string)

when start a spider ,cluster should record it

func (*DefaultCluster) StopSpider

func (this *DefaultCluster) StopSpider(spiderName string)