internal

Documentation

Index

• func Extract(body io.Reader) ([]string, error)
• type Crawler
  • func NewCrawler(store storage.Storage, workQueue queue.Queue, resultsQueue queue.Queue, ...) (Crawler, error)
  • func (c Crawler) Crawl()
• type Worker
  • func NewWorker(activeCounter *int64, workQueue queue.ReadQueue, resultsQueue queue.WriteQueue) *Worker
  • func (w *Worker) Start()

Functions

func Extract

func Extract(body io.Reader) ([]string, error)

Types

type Crawler

type Crawler struct {
	// contains filtered or unexported fields
}

func NewCrawler

func NewCrawler(store storage.Storage, workQueue queue.Queue, resultsQueue queue.Queue, startingURL string, numWorkers int64) (Crawler, error)

func (Crawler) Crawl

func (c Crawler) Crawl()

type Worker

type Worker struct {
	// contains filtered or unexported fields
}

func NewWorker

func NewWorker(activeCounter *int64, workQueue queue.ReadQueue, resultsQueue queue.WriteQueue) *Worker

func (*Worker) Start

func (w *Worker) Start()