scheduler

Documentation

Index

• Variables
• func HandleArchive(job Job)
• func HandleArchived(job Job)
• func HandleDownloadImage(job Job)
• func HandleDownloadMeta(job Job)
• func HandleDownloadSources(job Job)
• func HandleSave(job Job)
• func HandleSaveHtml(job Job)
• type Archive
• type Job
• type Result
• type Scheduler
  • func New(config *m.Config, db *db.DB, n int) *Scheduler
  • func (s *Scheduler) GetTask(work string) (func(Job), error)
  • func (s *Scheduler) Job(work, data string) Job
  • func (s *Scheduler) Next(w string, job Job)
  • func (s *Scheduler) RegisterTasks(t map[string]func(Job))
  • func (s *Scheduler) Schedule(job Job)
  • func (s *Scheduler) Start()
• type Worker
  • func NewWorker(ready chan chan Job, done sync.WaitGroup, s *Scheduler) *Worker
  • func (w *Worker) Start()
  • func (w *Worker) Stop()

Variables

var (
	// ErrorInavlidMaxWorkers will be thrown if the amount of max workers is < -1
	ErrorInavlidMaxWorkers = errors.New("Inavlid max workers amount")

	ErrorNoTaskFound = errors.New("Did not find task with this name")
)

var (
	ErrorHTTPStatusCode = errors.New("HTTP status code greater than 203")
)

Functions

func HandleArchive

func HandleArchive(job Job)

HandleArchive handles the archiving of entities

func HandleArchived

func HandleArchived(job Job)

func HandleDownloadImage

func HandleDownloadImage(job Job)

HandleDownloadImage handles the download of images

func HandleDownloadMeta

func HandleDownloadMeta(job Job)

HandleDownloadMeta handles the download of meta data

func HandleDownloadSources

func HandleDownloadSources(job Job)

HandleDownloadSources handles the download of CSS and JS source files from archived entities

func HandleSave

func HandleSave(job Job)

HandleSave handles the saving of entities to the database

func HandleSaveHtml

func HandleSaveHtml(job Job)

Types

type Archive

type Archive struct {
	Body []byte
}

type Job

type Job struct {
	Work      string
	Data      string
	Entity    m.Entity
	Result    Result
	Archive   Archive
	Scheduler *Scheduler
}

Job represents the work each worker consumes

type Result

type Result struct {
	Title       string
	Description string
	Image       string
	Favicon     string
}

type Scheduler

type Scheduler struct {
	Config     *m.Config
	DB         *db.DB
	Tasks      map[string]func(Job)
	Queue      chan Job
	Ready      chan chan Job
	Workers    []*Worker
	MaxWorkers int
	HTTPClient http.Client
	// contains filtered or unexported fields
}

Scheduler is the top level struct

func New

func New(config *m.Config, db *db.DB, n int) *Scheduler

New returns a new scheduler instance with n workers, provide n = -1 to use runtime.NumCPU - 1 workers.

func (*Scheduler) GetTask

func (s *Scheduler) GetTask(work string) (func(Job), error)

func (*Scheduler) Job

func (s *Scheduler) Job(work, data string) Job

func (*Scheduler) Next

func (s *Scheduler) Next(w string, job Job)

func (*Scheduler) RegisterTasks

func (s *Scheduler) RegisterTasks(t map[string]func(Job))

func (*Scheduler) Schedule

func (s *Scheduler) Schedule(job Job)

Schedule schedules a new job in the queue to be cunsumed by workers

func (*Scheduler) Start

func (s *Scheduler) Start()

type Worker

type Worker struct {
	Queue     chan Job
	Ready     chan chan Job
	Scheduler *Scheduler
	// contains filtered or unexported fields
}

func NewWorker

func NewWorker(ready chan chan Job, done sync.WaitGroup, s *Scheduler) *Worker

func (*Worker) Start

func (w *Worker) Start()

func (*Worker) Stop

func (w *Worker) Stop()