controller

Documentation

Index

• func GoQuery(doc *html.Node, selector string) (string, error)
• func HTMLQuery(doc *html.Node, selector string) (string, error)
• type Analyzer
  • func AnalyzerBySelector(selector string) Analyzer
• type C
• type Controller
  • func New(p *fetcher.FetcherPool) *Controller
  • func (c *Controller) AvailableMethods() []fetcher.FetchMethod
  • func (c *Controller) Scrape(ctx context.Context, cfg ScrapeConfig) (ScrapeResults, error)
• type ScrapeConfig
• type ScrapeResults

Functions

func GoQuery

func GoQuery(doc *html.Node, selector string) (string, error)

func HTMLQuery

func HTMLQuery(doc *html.Node, selector string) (string, error)

Types

type Analyzer

type Analyzer func(*html.Node, string) (string, error)

func AnalyzerBySelector

func AnalyzerBySelector(selector string) Analyzer

type C

type C interface {
	Scrape(ctx context.Context, cfg ScrapeConfig) (ScrapeResults, error)
	AvailableMethods() []fetcher.FetchMethod
}

type Controller

type Controller struct {
	// contains filtered or unexported fields
}

func New

func New(p *fetcher.FetcherPool) *Controller

func (*Controller) AvailableMethods

func (c *Controller) AvailableMethods() []fetcher.FetchMethod

func (*Controller) Scrape

func (c *Controller) Scrape(ctx context.Context, cfg ScrapeConfig) (ScrapeResults, error)

type ScrapeConfig

type ScrapeConfig struct {
	Method             string
	URL                string
	DataSelectors      map[string]string
	Multiple           bool
	ItemParentSelector string

	NextPageSelector string
	Limit            int

	VisitItemDetailPage bool
	ItemLinkSelector    string
}

type ScrapeResults

type ScrapeResults struct {
	Num    int
	Data   []map[string]string
	Errors []error
}