backend

Documentation

Index

• Variables
• func ScrapydDownloadFeed(jobID string) (*os.File, error)
• func ScrapydIsFinished(jobID string) (bool, error)
• func ScrapydSchedule(spiderName string) (string, error)
• func Start(db *pg.DB)
• func Stop()
• type JobContext
  • func (c *JobContext) Log(job *work.Job, next work.NextMiddlewareFunc) error
  • func (c *JobContext) ParseFeed(job *work.Job) error
  • func (c *JobContext) ParseItem(job *work.Job) error
  • func (c *JobContext) ScheduleDeal(job *work.Job) error
  • func (c *JobContext) ScheduleFull(job *work.Job) error
  • func (c *JobContext) UpdateExchangeRates(job *work.Job) error
  • func (c *JobContext) WaitForScraper(job *work.Job) error

Variables

var ErrInvalidScrapydJob = errors.New("Invalid Scrapyd job id")

ErrInvalidScrapydJob occurs when a scrapyd job doesb't exist.

Functions

func ScrapydDownloadFeed

func ScrapydDownloadFeed(jobID string) (*os.File, error)

func ScrapydIsFinished

func ScrapydIsFinished(jobID string) (bool, error)

ScrapydIsFinished checks if a job has finished. Returns whether the job has finished the ErrInvalidScrapydJob error if the job doesn't exist.

func ScrapydSchedule

func ScrapydSchedule(spiderName string) (string, error)

ScrapydSchedule schedules a scrapyd job with the given spider name. Returns a scrapyd job id and any error occured.

func Start

func Start(db *pg.DB)

func Stop

func Stop()

Types

type JobContext

type JobContext struct {
	DB *pg.DB
}

func (*JobContext) Log

func (c *JobContext) Log(job *work.Job, next work.NextMiddlewareFunc) error

func (*JobContext) ParseFeed

func (c *JobContext) ParseFeed(job *work.Job) error

ParseFeed parses downloads and parses the item feed and creates a 'parse_item' job for each item.

func (*JobContext) ParseItem

func (c *JobContext) ParseItem(job *work.Job) error

ParseItem parses the item data, creating the required database rows.

func (*JobContext) ScheduleDeal

func (c *JobContext) ScheduleDeal(job *work.Job) error

ScheduleDeal schedules jobs for active sites which have deal_scraper enabled.

func (*JobContext) ScheduleFull

func (c *JobContext) ScheduleFull(job *work.Job) error

ScheduleFull schedules jobs for active sites which have full_scraper enabled.

func (*JobContext) UpdateExchangeRates

func (c *JobContext) UpdateExchangeRates(job *work.Job) error

func (*JobContext) WaitForScraper

func (c *JobContext) WaitForScraper(job *work.Job) error

WaitForScraper waits until scrapyd reports that the job is finished and schedules a 'parse_feed' job, otherwise reschedules a 'wait_for_scraper job in 5 seconds.