crawl

Documentation

Index

• type LinkExtractor
  • func NewLinkExtractor() LinkExtractor
• type Scraper
  • func NewScraper() *Scraper
  • func (s *Scraper) ScrapeUrl(baseUrl string) ([]string, error)
• type WebPageExtractor
  • func NewWebPageExtractor() *WebPageExtractor
  • func (w *WebPageExtractor) ExtractWebPage(webPageUrl string, webPageResponse chan *http.Response, ...)
• type WebPageParser
  • func NewWebPageParser(linkExtractor LinkExtractor) *WebPageParser
  • func (w *WebPageParser) Parse(response *http.Response, urlChannel chan string, wg *sync.WaitGroup, ...)

Types

type LinkExtractor

type LinkExtractor struct {
}

func NewLinkExtractor

func NewLinkExtractor() LinkExtractor

type Scraper

type Scraper struct {
	// contains filtered or unexported fields
}

func NewScraper

func NewScraper() *Scraper

func (*Scraper) ScrapeUrl

func (s *Scraper) ScrapeUrl(baseUrl string) ([]string, error)

type WebPageExtractor

type WebPageExtractor struct {
}

func NewWebPageExtractor

func NewWebPageExtractor() *WebPageExtractor

func (*WebPageExtractor) ExtractWebPage

func (w *WebPageExtractor) ExtractWebPage(webPageUrl string, webPageResponse chan *http.Response, errorChannel chan error)

type WebPageParser

type WebPageParser struct {
	// contains filtered or unexported fields
}

func NewWebPageParser

func NewWebPageParser(linkExtractor LinkExtractor) *WebPageParser

func (*WebPageParser) Parse

func (w *WebPageParser) Parse(response *http.Response, urlChannel chan string, wg *sync.WaitGroup,
	urlCache *sync.Map, baseUrl string)