crawl

Documentation

Index

• func DismissPagePopups(page *rod.Page) error
• type ClickedElement
• type CrawlItem
• type CrawlOptions
• type CrawledPageResut
• type Crawler
  • func NewCrawler(startURLs []string, maxPagesToCrawl int, maxDepth int, poolSize int, ...) *Crawler
  • func (c *Crawler) CreateScopeFromProvidedUrls()
  • func (c *Crawler) Run() []*db.History
• type SubmittedForm

Functions

func DismissPagePopups

func DismissPagePopups(page *rod.Page) error

DismissPagePopups tries to find and click on elements that are likely to be dismissible pop-ups.

Types

type ClickedElement

type ClickedElement struct {
	// contains filtered or unexported fields
}

type CrawlItem

type CrawlItem struct {
	// contains filtered or unexported fields
}

type CrawlOptions

type CrawlOptions struct {
	ExtraHeaders    map[string][]string
	MaxDepth        int
	MaxPagesToCrawl int
}

type CrawledPageResut

type CrawledPageResut struct {
	URL            string
	DiscoveredURLs []string
	IsError        bool
}

type Crawler

type Crawler struct {
	Options CrawlOptions
	// contains filtered or unexported fields
}

func NewCrawler

func NewCrawler(startURLs []string, maxPagesToCrawl int, maxDepth int, poolSize int, excludePatterns []string, workspaceID, taskID uint, extraHeaders map[string][]string) *Crawler

func (*Crawler) CreateScopeFromProvidedUrls

func (c *Crawler) CreateScopeFromProvidedUrls()

CreateScopeFromProvidedUrls creates scope items given the received urls

func (*Crawler) Run

func (c *Crawler) Run() []*db.History

type SubmittedForm

type SubmittedForm struct {
	// contains filtered or unexported fields
}