crawl

Documentation

Index

• type Crawler
  • func NewConcurrentCrawler(maxGoroutines int, followExternalLinks bool) Crawler
• type Response

Types

type Crawler

type Crawler interface {
	Crawl(urls []string, onResp func(response Response), onQuit func())
}

func NewConcurrentCrawler

func NewConcurrentCrawler(maxGoroutines int, followExternalLinks bool) Crawler

type Response

type Response struct {
	StartURL    *url.URL
	URL         *url.URL
	HTML        string
	Headers     http.Header
	Cookies     []*http.Cookie
	ScriptLinks []string
	Scripts     []string
	Styles      []string
	// contains filtered or unexported fields
}