engine

Documentation

Index

• type ChanProxy
• type Concurrent
  • func (e *Concurrent) Run(seeds ...Request)
• type ParserResult
• type Request
• type Scheduler

Types

type ChanProxy

type ChanProxy struct {
	ProxyChan chan string
}

type Concurrent

type Concurrent struct {
	Scheduler   Scheduler
	Workercount int
}

func (*Concurrent) Run

func (e *Concurrent) Run(seeds ...Request)

type ParserResult

type ParserResult struct {
	Requests []Request
	Items    []interface{}
}

type Request

type Request struct {
	Url        string
	ParserFunc func(node *html.Node) ParserResult
}

type Scheduler

type Scheduler interface {
	Submit(Request)
	ConfigurWorkerChan(chan Request)
}