tweetfinder

Documentation

Index

• Variables
• type ConfigPool
  • func GetConfigPool() ConfigPool
• type Finder
  • func NewFinder(scraper *twitterscraper.Scraper, delayManager delayManager, logger log.Logger) Finder
  • func NewMetricMiddleware(login string, next Finder) Finder
  • func NewPool(finders []Finder, logger log.Logger) Finder
  • func NewPoolFabric(ctx context.Context, config ConfigPool, pkgKey string, repo fdb.DB, ...) (Finder, error)
• type Manager
  • func NewDelayManager(setter func(seconds int64), minimalDelay int64, logger log.Logger) Manager
  • func NewDelayManagerV2(setter func(seconds int64), windowLimiters []WindowLimiter, minimalDelay int64, ...) Manager
• type WindowLimiter

Variables

var (
	ErrNoTops   = errors.New("no top tweets")
	ErrNotFound = errors.New("not found")
)

Types

type ConfigPool

type ConfigPool struct {
	XCreds          map[string]string `envconfig:"X_CREDS" required:"true"`
	XConfirmation   []string          `envconfig:"X_CONFIRMATION"`
	CookiesFilename string            `envconfig:"COOKIES_FILENAME" default:"cookies.json"`
	Proxies         []string          `envconfig:"PROXIES"`
}

func GetConfigPool

func GetConfigPool() ConfigPool

type Finder

type Finder interface {
	FindAll(ctx context.Context, start, end *time.Time, search string) ([]common.TweetSnapshot, error)
	Find(ctx context.Context, id string) (*common.TweetSnapshot, error)
	CurrentDelay() int64
}

func NewFinder

func NewFinder(scraper *twitterscraper.Scraper, delayManager delayManager, logger log.Logger) Finder

func NewMetricMiddleware

func NewMetricMiddleware(login string, next Finder) Finder

func NewPool

func NewPool(finders []Finder, logger log.Logger) Finder

func NewPoolFabric

func NewPoolFabric(ctx context.Context, config ConfigPool, pkgKey string, repo fdb.DB, logger log.Logger) (Finder, error)

type Manager

type Manager interface {
	TooManyRequests(ctx context.Context)
	AfterRequest()
	ProcessedQuery()
	SetSetterFn(func(seconds int64))
	CurrentDelay() int64
	Start(ctx context.Context) error
}

func NewDelayManager

func NewDelayManager(setter func(seconds int64), minimalDelay int64, logger log.Logger) Manager

func NewDelayManagerV2

func NewDelayManagerV2(
	setter func(seconds int64),
	windowLimiters []WindowLimiter,
	minimalDelay int64,
	log log.Logger,
) Manager

type WindowLimiter

type WindowLimiter interface {
	Inc()
	TrySetThreshold(ctx context.Context, startTime time.Time) error
	Duration() time.Duration
	TooFast(ctx context.Context) (uint64, error)
	SetResetLimiter(resetLimiter windowlimiter.ResetLimiter)
	Threshold(ctx context.Context) uint64
	Start(ctx context.Context, delay int64) error
}