engine

package

v0.0.0-...-632f8f1 Latest Latest Go to latest Published: Apr 3, 2019 License: MIT Imports: 1 Imported by: 0

Details

This section is empty.

This section is empty.

func CreateWorkers(out chan ParseResult, s Scheduler, f Fetcher)

CreateWorkers

type ConcurrentEngine struct {
	Scheduler   Scheduler
	Fetcher     Fetcher
	WorkerCount int
	Writer      WriteWorker
}

ConcurrentEngine 开启并发爬虫采集器

func (c *ConcurrentEngine) Run(seeds ...Request)

Run

type EtcdSave interface {
	EtcdWrite(host, name string, endpoint []string) chan pb.Profile
}

type Fetcher interface {
	Fetch(string) ([]byte, error)
	Work(Request) (ParseResult, error)
	GetProxy() (string, error)
}

Fetcher

type GSaver interface {
	GrpcWrite(host string) chan pb.Profile
}

GrpcSaver

type ParseResult struct {
	Requests []Request
	Items    []pb.Profile
}

ParseResult

type Request struct {
	Url        string
	ParserFunc func([]byte) ParseResult
}

Request

type RpcSaver interface {
	RpcWrite(interface{}, *string) error
}

RpcSaver

type Saver interface {
	Write(...string) chan interface{}
}

Writer

type Scheduler interface {
	Submit(Request)
	ConfigWorkerChan(chan Request)
	WorkChanFree(chan Request)
	Begin()
}

Scheduler

type WriteWorker struct {
	Payload  chan pb.Profile
	Storage  Saver
	RpcSaver RpcSaver
	GSaver   GSaver
	EtcdSave EtcdSave
}

WriteWorker