engine

package

v0.0.0-...-9dec8c7 Latest Latest Go to latest Published: Sep 7, 2020 License: MIT Imports: 2 Imported by: 0

Details

This section is empty.

This section is empty.

This section is empty.

type ConcurrentEngine struct {
	Scheduler   Scheduler
	WorkerCount int
	ItemChan    chan Item
}

func (e *ConcurrentEngine) Run(seeds ...Request)

type FuncParser struct {
	// contains filtered or unexported fields
}

func NewFuncParser(p ParserFunc, name string) *FuncParser

func (f *FuncParser) Parse(contents []byte, url string) ParseResult

func (f *FuncParser) Serialize() (name string, args interface{})

type Item struct {
	Url     string
	Type    string
	Id      string
	Payload interface{}
}

type NilParser struct{}

func (NilParser) Parse(_ []byte, _ string) ParseResult

func (NilParser) Serialize() (name string, args interface{})

type ParseResult struct {
	Requests []Request
	Items    []Item
}

type Parser interface {
	Parse(contents []byte, url string) ParseResult
	Serialize() (name string, args interface{})
}

type ParserFunc func(contents []byte, url string) ParseResult

type ReadyNotifier interface {
	WorkerReady(chan Request)
}

type Request struct {
	Url    string
	Parser Parser
}

type Scheduler interface {
	ReadyNotifier
	Submit(Request)
	WorkerChan() chan Request
	Run()
}

type SimpleEngine struct{}

func (e SimpleEngine) Run(seeds ...Request)

可以送很多种子