scraper

Documentation

Overview

Package scraper represents a main application logic

Index

• type Runner
  • func NewAsyncRunner(cfg *config.AppConfig, cats []models.Category, cache cache.Cache) Runner
• type Service
  • func NewService(cfg *config.AppConfig) *Service
  • func (s *Service) GetCategories() []models.Category
  • func (s *Service) InvalidateCache(sel models.RunSelector)
  • func (s *Service) Scrape(ctx context.Context, selector models.RunSelector) []models.RunResult

Types

type Runner

type Runner interface {
	// Run the runner to get pages content
	// based on the selector
	Run(ctx context.Context, selector models.RunSelector) []models.RunResult
}

Runner interface

func NewAsyncRunner

func NewAsyncRunner(
	cfg *config.AppConfig,
	cats []models.Category,
	cache cache.Cache,
) Runner

NewAsyncRunner instance of the new asynchronous runner

type Service

type Service struct {
	Cfg config.AppConfig
	// contains filtered or unexported fields
}

Service main service representation

func NewService

func NewService(cfg *config.AppConfig) *Service

NewService create a new instance of the service

func (*Service) GetCategories

func (s *Service) GetCategories() []models.Category

GetCategories get all categories

func (*Service) InvalidateCache

func (s *Service) InvalidateCache(sel models.RunSelector)

InvalidateCache for the provided selector

func (*Service) Scrape

func (s *Service) Scrape(ctx context.Context, selector models.RunSelector) []models.RunResult

Scrape the pages based on selector