engine

Documentation

Index

• type ConcurrentEngine
  • func (e *ConcurrentEngine) Run(seeds ...Request)
• type IConcurrent
• type IQueued
• type Item
• type ParseResult
  • func NilParser([]byte) ParseResult
• type QueuedEngine
  • func (e *QueuedEngine) Run(seeds ...Request)
• type Request
• type SimpleEngine
  • func (engine SimpleEngine) Run(seeds ...Request)

Types

type ConcurrentEngine

type ConcurrentEngine struct {
	Scheduler   IConcurrent
	WorkerCount int
	ItemChan    chan Item
}

func (*ConcurrentEngine) Run

func (e *ConcurrentEngine) Run(seeds ...Request)

type IConcurrent

type IConcurrent interface {
	// contains filtered or unexported methods
}

type IQueued

type IQueued interface {
	WorkerReady(chan Request)
	// contains filtered or unexported methods
}

type Item

type Item struct {
	Id      string      `db:"id"`
	Url     string      `db:"url"`
	Type    string      `db:"type"`
	Payload interface{} `db:"payload"`
}

type ParseResult

type ParseResult struct {
	Requests []Request
	Items    []Item
}

func NilParser

func NilParser([]byte) ParseResult

type QueuedEngine

type QueuedEngine struct {
	Scheduler   IQueued
	WorkerCount int
}

func (*QueuedEngine) Run

func (e *QueuedEngine) Run(seeds ...Request)

type Request

type Request struct {
	Url        string
	ParserFunc func([]byte) ParseResult
}

type SimpleEngine

type SimpleEngine struct {
}

func (SimpleEngine) Run

func (engine SimpleEngine) Run(seeds ...Request)