crawler

Documentation

Index

• Constants
• func Crawl(seedPage url.URL, topicKeywords []string, targetCount int, ...) []url.URL
• func GetPage(pageURL url.URL) (string, error)
• func GetRobotsData(host string) (*robotstxt.RobotsData, error)
• func ShouldCrawl(url url.URL) (bool, error)

Constants

const (
	WORKER_COUNT          = 40
	WORKER_SLEEP_TIME_SEC = 4 // seconds
)

const (
	USER_AGENT             = "Googlebot"
	ROBOTS_REQUEST_TIMEOUT = 2 // seconds
	ROBOTS_PATH            = "/robots.txt"
)

Functions

func Crawl

func Crawl(seedPage url.URL, topicKeywords []string, targetCount int, timeLimit time.Duration) []url.URL

func GetPage

func GetPage(pageURL url.URL) (string, error)

func GetRobotsData

func GetRobotsData(host string) (*robotstxt.RobotsData, error)

func ShouldCrawl

func ShouldCrawl(url url.URL) (bool, error)