common

Documentation

Index

• Variables
• func BuildHttpClient(dialer *fastdialer.Dialer, options *types.Options, ...) (*retryablehttp.Client, *fastdialer.Dialer, error)
• type CrawlSession
• type DoRequestFunc
• type RedirectCallback
• type Shared
  • func NewShared(options *types.CrawlerOptions) (*Shared, error)
  • func (s *Shared) Do(crawlSession *CrawlSession, doRequest DoRequestFunc) error
  • func (s *Shared) Enqueue(queue *queue.Queue, navigationRequests ...*navigation.Request)
  • func (s *Shared) NewCrawlSessionWithURL(URL string) (*CrawlSession, error)
  • func (s *Shared) Output(navigationRequest *navigation.Request, navigationResponse *navigation.Response, ...)
  • func (s *Shared) ValidateScope(URL string, root string) bool

Variables

var ErrOutOfScope = errors.New("out of scope")

Functions

func BuildHttpClient

func BuildHttpClient(dialer *fastdialer.Dialer, options *types.Options, redirectCallback RedirectCallback) (*retryablehttp.Client, *fastdialer.Dialer, error)

BuildClient builds a http client based on a profile

Types

type CrawlSession

type CrawlSession struct {
	Ctx        context.Context
	CancelFunc context.CancelFunc
	URL        *url.URL
	Hostname   string
	Queue      *queue.Queue
	HttpClient *retryablehttp.Client
	Browser    *rod.Browser
}

type DoRequestFunc

type DoRequestFunc func(crawlSession *CrawlSession, req *navigation.Request) (*navigation.Response, error)

type RedirectCallback

type RedirectCallback func(resp *http.Response, depth int)

type Shared

type Shared struct {
	Headers    map[string]string
	KnownFiles *files.KnownFiles
	Options    *types.CrawlerOptions
}

func NewShared

func NewShared(options *types.CrawlerOptions) (*Shared, error)

func (*Shared) Do

func (s *Shared) Do(crawlSession *CrawlSession, doRequest DoRequestFunc) error

func (*Shared) Enqueue

func (s *Shared) Enqueue(queue *queue.Queue, navigationRequests ...*navigation.Request)

func (*Shared) NewCrawlSessionWithURL

func (s *Shared) NewCrawlSessionWithURL(URL string) (*CrawlSession, error)

func (*Shared) Output

func (s *Shared) Output(navigationRequest *navigation.Request, navigationResponse *navigation.Response, err error)

func (*Shared) ValidateScope

func (s *Shared) ValidateScope(URL string, root string) bool