action

Documentation

Index

• type CloseSpiderRequest
• type CloseSpiderResponse
• type DistributeReqponse
• type DistributeRequest
• type HttpServer
• type HttpServerAnts
• type HttpServerCluster
• type HttpServerCrawler
• type LeftMeInRequest
• type LeftMeInResponse
• type MulticastClient
• type MulticastServer
• type ReportRequest
• type ReportResponse
• type RpcBase
• type RpcClient
• type RpcClientAnts
• type RpcClientCluster
• type RpcClientCrawl
• type RpcServer
• type RpcServerAnts
• type RpcServerCluster
• type RpcServerCrawl
• type StopRequest
• type StopResponse
• type Watcher

Types

type CloseSpiderRequest

type CloseSpiderRequest struct {
	RpcBase
	SpiderName string
}

type CloseSpiderResponse

type CloseSpiderResponse struct {
	RpcBase
	SpiderName string
}

type DistributeReqponse

type DistributeReqponse struct {
	RpcBase
}

type DistributeRequest

type DistributeRequest struct {
	RpcBase
	Request *http.Request
}

type HttpServer

type HttpServer interface {
	ServeHTTP(w http.ResponseWriter, r *http.Request)
	Welcome(w http.ResponseWriter, r *http.Request)
}

http server

type HttpServerAnts

type HttpServerAnts interface {
	HttpServer
	HttpServerCluster
	HttpServerCrawler
}

ants http server

type HttpServerCluster

type HttpServerCluster interface {
	Cluster(w http.ResponseWriter, r *http.Request)
}

http server for cluster

type HttpServerCrawler

type HttpServerCrawler interface {
	Spiders(w http.ResponseWriter, r *http.Request)
	Crawl(w http.ResponseWriter, r *http.Request)
	CrawlCluster(w http.ResponseWriter, r *http.Request)
	CrawlNode(w http.ResponseWriter, r *http.Request)
}

http server for crawler

type LeftMeInRequest

type LeftMeInRequest struct {
	RpcBase
}

type LeftMeInResponse

type LeftMeInResponse struct {
	RpcBase
}

if result is true, it is master ;if false,close the client and connect to nodeinfo

type MulticastClient

type MulticastClient interface {
	// contains filtered or unexported methods
}

type MulticastServer

type MulticastServer interface {
	// contains filtered or unexported methods
}

type ReportRequest

type ReportRequest struct {
	RpcBase
	ScrapeResult *crawler.ScrapeResult
}

report to master the result of crawl request

type ReportResponse

type ReportResponse struct {
	RpcBase
}

type RpcBase

type RpcBase struct {
	NodeInfo *node.NodeInfo
	Result   bool
}

type RpcClient

type RpcClient interface {
	Dial(ip string, port int) (*rpc.Client, error)
	Detect() // if client connection is down
	Start()  // start dead loop for detect
}

* RpcClient * *

type RpcClientAnts

type RpcClientAnts interface {
	RpcClient
	RpcClientCrawl
	RpcClientCluster
}

* RpcClientAnts *

type RpcClientCluster

type RpcClientCluster interface {
	LetMeIn(ip string, port int) error
	Connect(ip string, port int) error
	StartSpider(nodeName, spiderName string) error
}

* RpcClientCluster *

type RpcClientCrawl

type RpcClientCrawl interface {
	StopNode(nodeName string) error
	Distribute(nodeName string, request *http.Request) error
	ReportResult(nodeName string, result *crawler.ScrapeResult) error
	CloseSpider(nodeName, spiderName string) error
}

* RpcClientCrawl *

type RpcServer

type RpcServer interface {
	IsAlive(request *RpcBase, response *RpcBase) error
}

* RpcServer *

type RpcServerAnts

type RpcServerAnts interface {
	RpcServer
	RpcServerCrawl
	RpcServerCluster
}

* RpcServerAnts *

type RpcServerCluster

type RpcServerCluster interface {
	LetMeIn(request *LeftMeInRequest, response *LeftMeInResponse) error
	Connect(request *RpcBase, response *RpcBase) error
	StopNode(request *StopRequest, response *StopResponse) error
	StartSpider(request *DistributeRequest, response *DistributeReqponse) error
}

* RpcServerCluster *

type RpcServerCrawl

type RpcServerCrawl interface {
	AcceptRequest(request *DistributeRequest, response *DistributeReqponse) error
	AcceptResult(request *ReportRequest, response *ReportResponse) error
	CloseSpider(request *CloseSpiderRequest, response *CloseSpiderResponse) error
}

* RpcServerCrawl *

type StopRequest

type StopRequest struct {
	RpcBase
}

stop the node when all request is finished

type StopResponse

type StopResponse struct {
	RpcBase
}

type Watcher

type Watcher interface {
	Stop()
	IsStop() bool
	IsPause() bool
	Pause()
	Unpause()
	Start()
	Run()
}