webscraper

Documentation

Index

• Constants
• type Element
• type Scraper
  • func NewScraper() *Scraper
  • func (o *Scraper) FetchAndScrape(ctx context.Context, url, queryPath string, handler func(e Element)) error
  • func (o *Scraper) Scrape(queryPath string, handler func(e Element)) error
  • func (o *Scraper) WithLogger(l log.Logger) *Scraper
  • func (o *Scraper) WithPreloadedDoc(ctx context.Context, url string) (*Scraper, error)
  • func (o *Scraper) WithPreloadedDocFromBytes(b []byte) (*Scraper, error)

Constants

const LoggerTag = "SCRAPER"

Types

type Element

type Element struct {
	Text  string
	Index int
}

type Scraper

type Scraper struct {
	// contains filtered or unexported fields
}

func NewScraper

func NewScraper() *Scraper

func (*Scraper) FetchAndScrape

func (o *Scraper) FetchAndScrape(ctx context.Context, url, queryPath string, handler func(e Element)) error

func (*Scraper) Scrape

func (o *Scraper) Scrape(queryPath string, handler func(e Element)) error

func (*Scraper) WithLogger

func (o *Scraper) WithLogger(l log.Logger) *Scraper

To be useful WithLogger should come ahead of With...() functions

func (*Scraper) WithPreloadedDoc

func (o *Scraper) WithPreloadedDoc(ctx context.Context, url string) (*Scraper, error)

func (*Scraper) WithPreloadedDocFromBytes

func (o *Scraper) WithPreloadedDocFromBytes(b []byte) (*Scraper, error)