contracts

Documentation

Index

• type Fetcher
• type Processor

Types

type Fetcher

type Fetcher interface {
	// Returns the content and the title of the crawled resource and
	// a slice of URLs to children resources found on that resource
	Fetch(url string) (content, title string, urls []string, err error)
	// Determine if we should crawl a link present on the baseURL
	ShouldCrawl(baseURL, href string) bool
}

type Processor

type Processor interface {
	// Process a fetched resource with its content, title, its children URLs
	// and an optional error
	Process(url, content, title string, urls []string, err error)
}