scraper

Documentation

Index

• func InitializeCollector(domain string) *colly.Collector
• func SetupHandlers(c *colly.Collector, sc *spellcheck.Trie, wg *sync.WaitGroup, ...)
• func StartScraping(c *colly.Collector, sc *spellcheck.Trie, url string) (map[string]bool, map[string][]string, map[string]bool, map[string]bool, ...)

Functions

func InitializeCollector

func InitializeCollector(domain string) *colly.Collector

func SetupHandlers

func SetupHandlers(c *colly.Collector, sc *spellcheck.Trie, wg *sync.WaitGroup, fontSet map[string]bool, altTexts map[string][]string, misspelledWords map[string]bool, brokenLinks map[string]bool, visitedUrls map[string]bool, nonNativeUrls map[string]bool, containsMediaQueries *bool)

func StartScraping

func StartScraping(c *colly.Collector, sc *spellcheck.Trie, url string) (map[string]bool, map[string][]string, map[string]bool, map[string]bool, map[string]bool, map[string]bool, bool)

StartScraping begins the scraping process on the specified URL