crawler

Documentation

Index

• type Crawler
  • func New(URL *url.URL, configuration *configuration.Configuration) (crawler Crawler, err error)
  • func (crawler *Crawler) Crawl() (results chan string, err error)
  • func (crawler *Crawler) FindLinks(source string) (links []string, err error)
  • func (crawler *Crawler) ParseRobots()
  • func (crawler *Crawler) ParseSitemap()

Types

type Crawler

type Crawler struct {
	URL                   *url.URL
	Configuration         *configuration.Configuration
	PageCollector         *colly.Collector
	LinkFindCollector     *colly.Collector
	URLsToLinkFindRegex   *regexp.Regexp
	URLsNotToRequestRegex *regexp.Regexp
}

func New

func New(URL *url.URL, configuration *configuration.Configuration) (crawler Crawler, err error)

func (*Crawler) Crawl

func (crawler *Crawler) Crawl() (results chan string, err error)

func (*Crawler) FindLinks

func (crawler *Crawler) FindLinks(source string) (links []string, err error)

func (*Crawler) ParseRobots

func (crawler *Crawler) ParseRobots()

func (*Crawler) ParseSitemap

func (crawler *Crawler) ParseSitemap()