scrapemate

Nov 2, 2024

Changes in this version

+ type Proxy struct

+ Password string

+ URL string

+ Username string

+ func NewProxy(u string) (Proxy, error)

Oct 20, 2024

Changes in this version

type HTTPFetcher

+ Close func() error

type ScrapeMate

+ func (s *ScrapeMate) Close() error

Oct 20, 2024

Changes in this version

+ type ProxyRotator interface

+ GetCredentials func() (string, string)

+ Next func() string

+ RoundTrip func(req *http.Request) (*http.Response, error)

Nov 24, 2023

Changes in this version

type IJob

+ ProcessOnFetchError func() bool

type Job

+ func (j *Job) ProcessOnFetchError() bool

Sep 2, 2023

Changes in this version

type IJob

+ GetParentID func() string

type Job

+ ParentID string

+ func (j *Job) GetParentID() string

Sep 1, 2023

Changes in this version

+ var ErrInactivityTimeout = errors.New("inactivity timeout")

+ func WithExitBecauseOfInactivity(duration time.Duration) func(*ScrapeMate) error

May 14, 2023

Changes in this version

+ func ContextWithLogger(ctx context.Context, logger logging.Logger) context.Context

+ func WithInitJob(job IJob) func(*ScrapeMate) error

May 13, 2023

Changes in this version

+ const PriorityHigh

+ const PriorityLow

+ const PriorityMedium

+ var ErrorNoHTMLFetcher = errors.New("no http fetcher set")

+ var ErrorNoHTMLParser = errors.New("no html parser set")

+ func WithHTMLParser(parser HTMLParser) func(*ScrapeMate) error

+ func WithHTTPFetcher(client HTTPFetcher) func(*ScrapeMate) error

+ type HTMLParser interface

+ Parse func(ctx context.Context, body []byte) (any, error)

+ type HTTPFetcher interface

+ Fetch func(ctx context.Context, job IJob) Response

type IJob

+ GetURLParams func() map[string]string

type Job

+ URLParams map[string]string

+ func (j *Job) GetURLParams() map[string]string

May 7, 2023

Changes in this version

+ func GetLoggerFromContext(ctx context.Context) logging.Logger

+ func WithSession() func(*ScrapeMate) error

type IJob

+ GetFullURL func() string

type Job

+ func (j *Job) GetFullURL() string

Apr 15, 2023

Changes in this version

+ var ErrorNotCsvCapable = errors.New("not csv capable")

+ type CsvCapable interface

+ CsvHeaders func() []string

+ CsvRow func() []string

type IJob

+ UseInResults func() bool

type Job

+ func (j *Job) UseInResults() bool

+ type ResultWriter interface

+ Run func(ctx context.Context, in <-chan Result) error

+ type ScrapeMate struct

+ func (s *ScrapeMate) Concurrency() int

+ func (s *ScrapeMate) DoJob(ctx context.Context, job IJob) (result any, next []IJob, err error)

+ func (s *ScrapeMate) Done() <-chan struct{}

+ func (s *ScrapeMate) Err() error

+ func (s *ScrapeMate) Failed() <-chan IJob

+ func (s *ScrapeMate) Results() <-chan Result

+ func (s *ScrapeMate) Start() error

Apr 10, 2023

Changes in this version

+ var ErrorNoCacher = errors.New("no cacher set")

+ func WithCache(cache Cacher) func(*scrapeMate) error

+ type Cacher interface

+ Close func() error

+ Get func(ctx context.Context, key string) (Response, error)

+ Set func(ctx context.Context, key string, value Response) error

type IJob

+ GetCacheKey func() string

type Job

+ func (j *Job) GetCacheKey() string

Mar 26, 2023

Changes in this version

+ const DefaultMaxRetryDelay

+ const DefaultUserAgent

+ const DiscardJob

+ const RefreshIP

+ const RetryJob

+ const StopScraping

+ var ErrorConcurrency = errors.New("concurrency must be greater than 0")

+ var ErrorExitSignal = errors.New("exit signal received")

+ var ErrorNoContext = errors.New("no context set")

+ var ErrorNoHtmlParser = errors.New("no html parser set")

+ var ErrorNoHttpFetcher = errors.New("no http fetcher set")

+ var ErrorNoJobProvider = errors.New("no job provider set")

+ var ErrorNoLogger = errors.New("no logger set")

+ func New(options ...func(*scrapeMate) error) (*scrapeMate, error)

+ func WithConcurrency(concurrency int) func(*scrapeMate) error

+ func WithContext(ctx context.Context, cancelFn context.CancelCauseFunc) func(*scrapeMate) error

+ func WithFailed() func(*scrapeMate) error

+ func WithHtmlParser(parser HtmlParser) func(*scrapeMate) error

+ func WithHttpFetcher(client HttpFetcher) func(*scrapeMate) error

+ func WithJobProvider(provider JobProvider) func(*scrapeMate) error

+ func WithLogger(log logging.Logger) func(*scrapeMate) error

+ type HtmlParser interface

+ Parse func(ctx context.Context, body []byte) (any, error)

+ type HttpFetcher interface

+ Fetch func(ctx context.Context, job IJob) Response

+ type IJob interface

+ BrowserActions func(browser playwright.Browser) Response

+ DoCheckResponse func(resp Response) bool

+ DoScreenshot func() bool

+ GetBody func() []byte

+ GetHeaders func() map[string]string

+ GetID func() string

+ GetMaxRetries func() int

+ GetMaxRetryDelay func() time.Duration

+ GetMethod func() string

+ GetPriority func() int

+ GetRetryPolicy func() RetryPolicy

+ GetTimeout func() time.Duration

+ GetURL func() string

+ GetUrlParams func() map[string]string

+ Process func(ctx context.Context, resp Response) (any, []IJob, error)

+ type Job struct

+ Body []byte

+ CheckResponse func(resp Response) bool

+ Headers map[string]string

+ ID string

+ MaxRetries int

+ MaxRetryDelay time.Duration

+ Method string

+ Priority int

+ Response Response

+ RetryPolicy RetryPolicy

+ TakeScreenshot bool

+ Timeout time.Duration

+ URL string

+ UrlParams map[string]string

+ func (j *Job) BrowserActions(browser playwright.Browser) Response

+ func (j *Job) DoCheckResponse(resp Response) bool

+ func (j *Job) DoScreenshot() bool

+ func (j *Job) GetBody() []byte

+ func (j *Job) GetHeaders() map[string]string

+ func (j *Job) GetID() string

+ func (j *Job) GetMaxRetries() int

+ func (j *Job) GetMaxRetryDelay() time.Duration

+ func (j *Job) GetMethod() string

+ func (j *Job) GetPriority() int

+ func (j *Job) GetRetryPolicy() RetryPolicy

+ func (j *Job) GetTimeout() time.Duration

+ func (j *Job) GetURL() string

+ func (j *Job) GetUrlParams() map[string]string

+ func (j *Job) Process(ctx context.Context, resp Response) (any, []IJob, error)

+ func (j *Job) String() string

+ type JobProvider interface

+ Jobs func(ctx context.Context) (<-chan IJob, <-chan error)

+ Push func(ctx context.Context, job IJob) error

+ type Response struct

+ Body []byte

+ Document any

+ Duration time.Duration

+ Error error

+ Headers http.Header

+ Meta map[string]any

+ Screenshot []byte

+ StatusCode int

+ URL string

+ type Result struct

+ Data any

+ Job IJob

+ type RetryPolicy int