crawler

Documentation

Index

• type Config
• type Crawler
  • func NewCrawler(cfg Config) *Crawler
  • func (c *Crawler) Crawl(ctx context.Context, linkIt graph.LinkIterator) (int, error)
• type Graph
• type Indexer
• type PrivateNetworkDetector
• type URLGetter

Types

type Config

type Config struct {
	PrivateNetworkDetector PrivateNetworkDetector
	URLGetter              URLGetter
	Graph                  Graph
	Indexer                Indexer
	FetchWorkers           int
}

type Crawler

type Crawler struct {
	// contains filtered or unexported fields
}

func NewCrawler

func NewCrawler(cfg Config) *Crawler

func (*Crawler) Crawl

func (c *Crawler) Crawl(ctx context.Context, linkIt graph.LinkIterator) (int, error)

type Graph

type Graph interface {
	UpsertLink(link *graph.Link) error
	UpsertEdge(edge *graph.Edge) error
	RemoveStaleEdges(fromID uuid.UUID, updateBefore time.Time) error
}

type Indexer

type Indexer interface {
	Index(doc *index.Document) error
}

Indexer is implemented by objects that can index the contents of web-pages retrieved by the crawler pipeline.

type PrivateNetworkDetector

type PrivateNetworkDetector interface {
	IsPrivate(host string) (bool, error)
}

PrivateNetworkDetector is implemented by objects that can detect whether a host resolves to a private network address

type URLGetter

type URLGetter interface {
	Get(url string) (*http.Response, error)
}