scraper

Documentation

Index

• Variables
• type Product
• type ProductPageCallbackFunc
• type Scraper
  • func NewScraper(cacheDir string, threads int, callback ProductPageCallbackFunc) Scraper
  • func (s Scraper) EnableLimits()
  • func (s Scraper) Start() error
• type StackQueueStorage
  • func (s *StackQueueStorage) AddRequest(r []byte) error
  • func (s *StackQueueStorage) GetRequest() ([]byte, error)
  • func (s *StackQueueStorage) Init() error
  • func (s *StackQueueStorage) QueueSize() (int, error)

Variables

var ErrRedirectToErrorPage = errors.New("redirected to error page")

Types

type Product

type Product struct {
	URL        string
	Name       string
	ProdID     string
	CatID      string
	Price      float64
	Savings    float64
	Percentage float64
}

type ProductPageCallbackFunc

type ProductPageCallbackFunc func(p Product)

type Scraper

type Scraper struct {
	// contains filtered or unexported fields
}

func NewScraper

func NewScraper(cacheDir string, threads int, callback ProductPageCallbackFunc) Scraper

cacheDir can be empty to disable caching.

func (Scraper) EnableLimits

func (s Scraper) EnableLimits()

func (Scraper) Start

func (s Scraper) Start() error

type StackQueueStorage

type StackQueueStorage struct {
	// contains filtered or unexported fields
}

StackQueueStorage is a very simple FILO stack storage backend for the colly queue.

func (*StackQueueStorage) AddRequest

func (s *StackQueueStorage) AddRequest(r []byte) error

func (*StackQueueStorage) GetRequest

func (s *StackQueueStorage) GetRequest() ([]byte, error)

func (*StackQueueStorage) Init

func (s *StackQueueStorage) Init() error

func (*StackQueueStorage) QueueSize

func (s *StackQueueStorage) QueueSize() (int, error)