scrape

Documentation

Index

• Variables
• type Config
• type FxNewScraperParams
• type NewScraperParams
• type ScrapeResult
• type Scraper
  • func FxNewScraper(p FxNewScraperParams) *Scraper
  • func NewScraper(p NewScraperParams) *Scraper
  • func (s *Scraper) Scrape(ch chan *ScrapeResult)
• type Target

Variables

var Module = fx.Module("scrape",
	fx.Provide(
		FxNewScraper,
	),
)

Types

type Config

type Config struct {
	Targets    []Target
	UserAgents []string
}

type FxNewScraperParams

type FxNewScraperParams struct {
	fx.In
	Log       *zap.Logger
	Config    Config
	Extractor extractors.Extractor
	CurlHTTP  *curlhttp.CurlHttp
}

type NewScraperParams

type NewScraperParams struct {
	Log        *zap.Logger
	Config     Config
	Extractors extractors.Extractor
	CurlHTTP   *curlhttp.CurlHttp
}

type ScrapeResult

type ScrapeResult struct {
	Name   string
	Labels map[string]string
	Price  float64
}

type Scraper

type Scraper struct {
	Log        *zap.Logger
	Config     Config
	Extractors extractors.Extractor
	CurlHTTP   *curlhttp.CurlHttp
}

func FxNewScraper

func FxNewScraper(p FxNewScraperParams) *Scraper

func NewScraper

func NewScraper(p NewScraperParams) *Scraper

func (*Scraper) Scrape

func (s *Scraper) Scrape(ch chan *ScrapeResult)

type Target

type Target struct {
	Cookies     map[string]string
	Headers     map[string][]string
	Page        string
	HTTPVersion string `mapstructure: "httpversion"`
	Compression bool

	Script string
	Labels map[string]string
	Name   string
}