scheduler

Documentation

Index

• func ConstructRequest(task *models.CrawlTask) (string, bool)
• type Scheduler
  • func NewScheduler(cookie *http.Cookie, storer *storage.BetterInMemoryStorage) *Scheduler
  • func (sch *Scheduler) Schedule(c *colly.Collector, taskq *TaskQueue) error
• type TaskQueue
  • func NewTaskQueue(mq models.MessageQueue, channame string) *TaskQueue
  • func (taskq *TaskQueue) Ack(ackid uint64) error
  • func (taskq *TaskQueue) GetTask() (*models.CrawlTask, uint64, uint8, error)
  • func (taskq *TaskQueue) Publish(name string, message models.MQMessage) error
  • func (taskq *TaskQueue) Reject(ackid uint64) error
  • func (taskq *TaskQueue) Resend(task *models.CrawlTask, priority uint8) error

Functions

func ConstructRequest

func ConstructRequest(task *models.CrawlTask) (string, bool)

Types

type Scheduler

type Scheduler struct {
	// contains filtered or unexported fields
}

func NewScheduler

func NewScheduler(cookie *http.Cookie, storer *storage.BetterInMemoryStorage) *Scheduler

func (*Scheduler) Schedule

func (sch *Scheduler) Schedule(c *colly.Collector, taskq *TaskQueue) error

type TaskQueue

type TaskQueue struct {
	// contains filtered or unexported fields
}

func NewTaskQueue

func NewTaskQueue(mq models.MessageQueue, channame string) *TaskQueue

func (*TaskQueue) Ack

func (taskq *TaskQueue) Ack(ackid uint64) error

func (*TaskQueue) GetTask

func (taskq *TaskQueue) GetTask() (*models.CrawlTask, uint64, uint8, error)

func (*TaskQueue) Publish

func (taskq *TaskQueue) Publish(name string, message models.MQMessage) error

func (*TaskQueue) Reject

func (taskq *TaskQueue) Reject(ackid uint64) error

func (*TaskQueue) Resend

func (taskq *TaskQueue) Resend(task *models.CrawlTask, priority uint8) error