scrape

Documentation

Index

• func GetCloudFlareRoundTripper() http.RoundTripper
• func ScrapeDetails(options ScrapeDetailsOptions) map[string]string
• func ScrapeList(schema *config.PageSchema, url string) config.ParsedLinks
• func ScrapeListFlat(schema *config.PageSchema, url string) []string
• func SetVerbose(v bool)
• func Verbose() bool
• type DetailsCollector
• type MemoryDetailsCollector
  • func NewMemoryDetailsCollector() *MemoryDetailsCollector
  • func (c *MemoryDetailsCollector) CollectField(url string, field string, data interface{}) error
• type ScrapeDetailsOptions

Functions

func GetCloudFlareRoundTripper

func GetCloudFlareRoundTripper() http.RoundTripper

func ScrapeDetails

func ScrapeDetails(options ScrapeDetailsOptions) map[string]string

func ScrapeList

func ScrapeList(schema *config.PageSchema, url string) config.ParsedLinks

func ScrapeListFlat

func ScrapeListFlat(schema *config.PageSchema, url string) []string

func SetVerbose

func SetVerbose(v bool)

func Verbose

func Verbose() bool

Types

type DetailsCollector

type DetailsCollector interface {
	// Collect(url string, data T) error
	CollectField(url string, field string, data interface{}) error
}

type MemoryDetailsCollector

type MemoryDetailsCollector struct {
	Items map[string]map[string]interface{}
	// contains filtered or unexported fields
}

func NewMemoryDetailsCollector

func NewMemoryDetailsCollector() *MemoryDetailsCollector

func (*MemoryDetailsCollector) CollectField

func (c *MemoryDetailsCollector) CollectField(url string, field string, data interface{}) error

type ScrapeDetailsOptions

type ScrapeDetailsOptions struct {
	Async            bool
	BatchSize        int
	URLs             []string
	Schema           *config.PageSchema
	DetailsCollector DetailsCollector
}