services

Documentation

Index

• Constants
• func CanRetry(statusCode int, attempts uint, err error) bool
• func NewHTTPClient() *http.Client
• type Bytes
• type FetchHelper
  • func (hlp FetchHelper) Try(fetcher *Fetcher, url string, attempts uint) (*pkg.UrlScraped, uint, error)
• type Fetcher
  • func (f Fetcher) Fetch(url string) (*pkg.UrlScraped, int, error)
• type FetcherIntf
• type Publisher
• type RedisPublisher
  • func NewRedisPublisher() *RedisPublisher
  • func (p *RedisPublisher) Publish(queue string, payload []byte) error
• type RedisSubscriber
  • func NewRedisSubscriber(queue string) *RedisSubscriber
  • func (s *RedisSubscriber) Close() error
  • func (s *RedisSubscriber) Consume(out chan string)
• type Scraper
  • func (s Scraper) Scrape(buf []byte)
• type Subscriber

Constants

const MaxAttempts = 3

const MaxFetchBytes = 4096

Functions

func CanRetry

func CanRetry(statusCode int, attempts uint, err error) bool

func NewHTTPClient

func NewHTTPClient() *http.Client

Types

type Bytes

type Bytes []byte

type FetchHelper

type FetchHelper struct {
}

func (FetchHelper) Try

func (hlp FetchHelper) Try(fetcher *Fetcher, url string, attempts uint) (*pkg.UrlScraped, uint, error)

type Fetcher

type Fetcher struct {
	Client *http.Client
}

func (Fetcher) Fetch

func (f Fetcher) Fetch(url string) (*pkg.UrlScraped, int, error)

type FetcherIntf

type FetcherIntf interface {
	Fetch(url string) (Bytes, error)
}

type Publisher

type Publisher interface {
	Publish(queue string, payload []byte) error
}

type RedisPublisher

type RedisPublisher struct {
	// contains filtered or unexported fields
}

func NewRedisPublisher

func NewRedisPublisher() *RedisPublisher

func (*RedisPublisher) Publish

func (p *RedisPublisher) Publish(queue string, payload []byte) error

type RedisSubscriber

type RedisSubscriber struct {
	PubSub *redis.PubSub
	// contains filtered or unexported fields
}

func NewRedisSubscriber

func NewRedisSubscriber(queue string) *RedisSubscriber

func (*RedisSubscriber) Close

func (s *RedisSubscriber) Close() error

func (*RedisSubscriber) Consume

func (s *RedisSubscriber) Consume(out chan string)

Consume messages.

type Scraper

type Scraper struct {
}

func (Scraper) Scrape

func (s Scraper) Scrape(buf []byte)

type Subscriber

type Subscriber interface {
	Subscribe(queue string, payload []byte) error
}