scraper

Documentation

Index

• type SubjectUserScraper
  • func NewSubjectUserScraper(coldStartIntervalInDays, uidChanSize int) *SubjectUserScraper
  • func (scraper *SubjectUserScraper) CloseUidChan()
  • func (scraper *SubjectUserScraper) CollectUids() []string
  • func (scraper *SubjectUserScraper) Crawl(sid string)
• type UserAgentGenerator
  • func NewUserAgentGenerator() *UserAgentGenerator
  • func (agentGen *UserAgentGenerator) RandomUserAgent() string

Types

type SubjectUserScraper

type SubjectUserScraper struct {
	// contains filtered or unexported fields
}

func NewSubjectUserScraper

func NewSubjectUserScraper(coldStartIntervalInDays, uidChanSize int) *SubjectUserScraper

func (*SubjectUserScraper) CloseUidChan

func (scraper *SubjectUserScraper) CloseUidChan()

func (*SubjectUserScraper) CollectUids

func (scraper *SubjectUserScraper) CollectUids() []string

func (*SubjectUserScraper) Crawl

func (scraper *SubjectUserScraper) Crawl(sid string)

type UserAgentGenerator

type UserAgentGenerator struct {
	// contains filtered or unexported fields
}

func NewUserAgentGenerator

func NewUserAgentGenerator() *UserAgentGenerator

func (*UserAgentGenerator) RandomUserAgent

func (agentGen *UserAgentGenerator) RandomUserAgent() string