Versions in this module Expand all Collapse all v1 v1.1.1 Jun 19, 2023 Changes in this version + var GeneralRegex = `((?:https?)://[\w\-]+(?:\.[\w\-]+)+[\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])` + type Cache struct + Lock sync.Mutex + Visited map[string]bool + func (c *Cache) AddVisited(url string) + func (c *Cache) Flush() + func (c *Cache) IsVisited(url string) bool + type Crawler struct + Cache Cache + Client *http.Client + ExcludedStatus []int + ExportFile string + IncludedUrls []string + Level int + LiveMode bool + RegexMap map[string]string + RootURL string + func NewCrawler(url string, level int, liveMode bool, exportFile string, ...) *Crawler + func (c *Crawler) AddMatches(page webtree.Page) + func (c *Crawler) Crawl() + func (c *Crawler) CrawlNodeBlock(w *webtree.Node) + func (c *Crawler) CrawlNodeLive(w *webtree.Node) + func (c *Crawler) Export(tree webtree.Node, format string, filename string) error + func (c *Crawler) ExportJSON(root webtree.Node, filename string) error + func (c *Crawler) ExportTXT(root webtree.Node, filename string) error + func (c *Crawler) ExportXML(tree webtree.Node, filename string) error + func (c *Crawler) ExtractLinks(page *webtree.Page) (links []string) + func (c *Crawler) Fetch(page *webtree.Page) + func (c *Crawler) IsSkipablePage(page webtree.Page) bool + func (c *Crawler) SaveResults(root webtree.Node) Other modules containing this package github.com/Malwarize/webpalm/v2
Go to main page