watcher

Documentation

Index

• Constants
• type Distributer
  • func NewDistributer(node node.Node, cluster cluster.Cluster, rpcClient action.RpcClientAnts) *Distributer
  • func (this *Distributer) Distribute(request *http.Request)
  • func (this *Distributer) IsPause() bool
  • func (this *Distributer) IsStop() bool
  • func (this *Distributer) Pause()
  • func (this *Distributer) Run()
  • func (this *Distributer) Start()
  • func (this *Distributer) Stop()
  • func (this *Distributer) Unpause()
• type Reporter
  • func NewReporter(node node.Node, cluster cluster.Cluster, rpcClient action.RpcClientAnts, ...) *Reporter
  • func (this *Reporter) IsPause() bool
  • func (this *Reporter) IsStop() bool
  • func (this *Reporter) JudgeAndStopNode(result *crawler.ScrapeResult)
  • func (this *Reporter) Pause()
  • func (this *Reporter) Run()
  • func (this *Reporter) Start()
  • func (this *Reporter) Stop()
  • func (this *Reporter) Unpause()

Constants

const (
	DISTRIBUTE_RUNING = iota
	DISTRIBUTE_PAUSE
	DISTRIBUTE_STOP
	DISTRIBUTE_STOPED
)

what a Distributer do * status,running|parse|stop * distribute a request,by some strategy * *

const (
	REPORT_STATUS_RUNNING = iota
	REPORT_STATUS_PAUSE
	REPORT_STATUS_STOP
	REPORT_STATUS_STOPED
)

Types

type Distributer

type Distributer struct {
	Status    int
	Cluster   cluster.Cluster
	Node      node.Node
	LastIndex int
	RpcClient action.RpcClientAnts
}

func NewDistributer

func NewDistributer(node node.Node, cluster cluster.Cluster, rpcClient action.RpcClientAnts) *Distributer

func (*Distributer) Distribute

func (this *Distributer) Distribute(request *http.Request)

if cookiejar > 0 means it require cookie context ,so we should send it to where it come from else distribute it by order

func (*Distributer) IsPause

func (this *Distributer) IsPause() bool

func (*Distributer) IsStop

func (this *Distributer) IsStop() bool

func (*Distributer) Pause

func (this *Distributer) Pause()

func (*Distributer) Run

func (this *Distributer) Run()

dead loop cluster pop request

func (*Distributer) Start

func (this *Distributer) Start()

func (*Distributer) Stop

func (this *Distributer) Stop()

func (*Distributer) Unpause

func (this *Distributer) Unpause()

type Reporter

type Reporter struct {
	Status      int
	ResultQuene *crawler.ResultQuene
	Node        node.Node
	// contains filtered or unexported fields
}

func NewReporter

func NewReporter(node node.Node, cluster cluster.Cluster, rpcClient action.RpcClientAnts, resultQuene *crawler.ResultQuene, distributer action.Watcher) *Reporter

func (*Reporter) IsPause

func (this *Reporter) IsPause() bool

func (*Reporter) IsStop

func (this *Reporter) IsStop() bool

if it is stoped

func (*Reporter) JudgeAndStopNode

func (this *Reporter) JudgeAndStopNode(result *crawler.ScrapeResult)

stop action is start by local report action so put it here

func (*Reporter) Pause

func (this *Reporter) Pause()

func (*Reporter) Run

func (this *Reporter) Run()

pop result quene if scraped new request set node name local node name send it to master

func (*Reporter) Start

func (this *Reporter) Start()

func (*Reporter) Stop

func (this *Reporter) Stop()

set it stop,if deap loop see this sign,make it stoped

func (*Reporter) Unpause

func (this *Reporter) Unpause()