scraper

Documentation

Index

• type ErrorsFetcher
• type Processor
  • func NewProcessor(numWorkers int) Processor
  • func (p Processor) Enqueue(r Request)
  • func (p Processor) Run()
• type Request
• type Scraper
  • func NewScraper(resolver servicediscovery.Resolver, r *repository.ErrorsRepository, ...) Scraper
  • func (scraper Scraper) Scrape()

Types

type ErrorsFetcher

type ErrorsFetcher func(string) (responsePayload, error)

type Processor

type Processor struct {
	// contains filtered or unexported fields
}

func NewProcessor

func NewProcessor(numWorkers int) Processor

func (Processor) Enqueue

func (p Processor) Enqueue(r Request)

func (Processor) Run

func (p Processor) Run()

type Request

type Request struct {
	Target        string
	ResultChannel chan<- responsePayload
	WaitGroup     *sync.WaitGroup
}

type Scraper

type Scraper struct {
	Resolver      servicediscovery.Resolver
	Repository    *repository.ErrorsRepository
	ServiceConfig config.Service
	// contains filtered or unexported fields
}

func NewScraper

func NewScraper(resolver servicediscovery.Resolver, r *repository.ErrorsRepository,
	serviceConfig config.Service, processor Processor) Scraper

NewScraper create a new scraper for a given service name

func (Scraper) Scrape

func (scraper Scraper) Scrape()

Scrape runs go routines scrapping the list of targets of this service, processes the errors and stores them into the repository.