pipeline

Documentation

Index

• type Pipeline
  • func NewPipeline(exchangename string, storer *storage.BetterInMemoryStorage) *Pipeline
  • func (pipe *Pipeline) Deal(tasktype uint, rawb []byte, task *models.CrawlTask) (interface{}, error)
  • func (pipe *Pipeline) Error(r *colly.Response, taskq *scheduler.TaskQueue)
  • func (pipe *Pipeline) Hook(c *colly.Collector, taskq *scheduler.TaskQueue)
  • func (pipe *Pipeline) Response(r *colly.Response, taskq *scheduler.TaskQueue)
  • func (pipe *Pipeline) Send(tid string, typeid uint, data interface{}, priority uint8, ...)

Types

type Pipeline

type Pipeline struct {
	// contains filtered or unexported fields
}

func NewPipeline

func NewPipeline(exchangename string, storer *storage.BetterInMemoryStorage) *Pipeline

func (*Pipeline) Deal

func (pipe *Pipeline) Deal(tasktype uint, rawb []byte, task *models.CrawlTask) (interface{}, error)

func (*Pipeline) Error

func (pipe *Pipeline) Error(r *colly.Response, taskq *scheduler.TaskQueue)

func (*Pipeline) Hook

func (pipe *Pipeline) Hook(c *colly.Collector, taskq *scheduler.TaskQueue)

func (*Pipeline) Response

func (pipe *Pipeline) Response(r *colly.Response, taskq *scheduler.TaskQueue)

func (*Pipeline) Send

func (pipe *Pipeline) Send(tid string, typeid uint, data interface{}, priority uint8, taskq *scheduler.TaskQueue)