scrapemateapp

Documentation

Index

• Constants
• func DisableImages() func(*jsOptions)
• func Headfull() func(*jsOptions)
• func WithBrowserReuseLimit(limit int) func(*Config) error
• func WithCache(cacheType, cachePath string) func(*Config) error
• func WithConcurrency(concurrency int) func(*Config) error
• func WithExitOnInactivity(duration time.Duration) func(*Config) error
• func WithInitJob(job scrapemate.IJob) func(*Config) error
• func WithJS(opts ...func(*jsOptions)) func(*Config) error
• func WithPageReuseLimit(limit int) func(*Config) error
• func WithProvider(provider scrapemate.JobProvider) func(*Config) error
• func WithProxies(proxies []string) func(*Config) error
• func WithStealth(browser string) func(*Config) error
• func WithUA(ua string) func(*jsOptions)
• type Config
  • func NewConfig(writers []scrapemate.ResultWriter, options ...func(*Config) error) (*Config, error)
• type ScrapemateApp
  • func NewScrapeMateApp(cfg *Config) (*ScrapemateApp, error)
  • func (app *ScrapemateApp) Close() error
  • func (app *ScrapemateApp) Start(ctx context.Context, seedJobs ...scrapemate.IJob) error

Constants

const (
	DefaultConcurrency = 1
	DefaultProvider    = "memory"
)

Functions

func DisableImages

func DisableImages() func(*jsOptions)

DisableImages is a helper function to disable images in the browser.

func Headfull

func Headfull() func(*jsOptions)

Headfull is a helper function to create a headfull browser. Use it as a parameter to WithJS.

func WithBrowserReuseLimit

func WithBrowserReuseLimit(limit int) func(*Config) error

WithBrowserReuseLimit sets the browser reuse limit of the app.

func WithCache

func WithCache(cacheType, cachePath string) func(*Config) error

WithCache sets the cache type and path of the app.

func WithConcurrency

func WithConcurrency(concurrency int) func(*Config) error

WithConcurrency sets the concurrency of the app.

func WithExitOnInactivity

func WithExitOnInactivity(duration time.Duration) func(*Config) error

WithExitOnInactivity sets the duration after which the app will exit if there are no more jobs to run.

func WithInitJob

func WithInitJob(job scrapemate.IJob) func(*Config) error

WithInitJob sets the initial job of the app.

func WithJS

func WithJS(opts ...func(*jsOptions)) func(*Config) error

WithJS sets the app to use JavaScript to render the pages.

func WithPageReuseLimit

func WithPageReuseLimit(limit int) func(*Config) error

WithPageReuseLimit sets the page reuse limit of the app.

func WithProvider

func WithProvider(provider scrapemate.JobProvider) func(*Config) error

WithProvider sets the provider of the app.

func WithProxies

func WithProxies(proxies []string) func(*Config) error

WithProxies sets the proxies of the app.

func WithStealth

func WithStealth(browser string) func(*Config) error

func WithUA

func WithUA(ua string) func(*jsOptions)

WithUA sets the user agent of the browser.

Types

type Config

type Config struct {
	// Concurrency is the number of concurrent scrapers to run.
	// If not set, it defaults to 1.
	Concurrency int `validate:"required,gte=1"`

	// Cache is the cache to use for storing scraped data.
	// If left empty then no caching will be used.
	// Otherwise the CacheType must be one of file or leveldb.
	CacheType string `validate:"omitempty,oneof=file leveldb"`
	// CachePath is the path to the cache file or directory.
	// It is required to be a valid path if CacheType is set.
	CachePath string `validate:"required_with=CacheType"`

	// UseJS is whether to use JavaScript to render the page.
	UseJS bool `validate:"omitempty"`
	// UseStealth is whether to use stealth mode to scrape the page.
	// uses a special http client to scrape the page.
	UseStealth bool `validate:"omitempty"`
	// StealthBrowser is the browser to use for stealth mode.
	StealthBrowser string `validate:"omitempty"`
	// JSOpts are the options for the JavaScript renderer.
	JSOpts jsOptions

	// ProviderType is the type of provider to use.
	// It is required to be a valid type if Provider is set.
	// If not set the memory provider will be used.
	Provider scrapemate.JobProvider

	// Writers are the writers to use for writing the results.
	// At least one writer must be provided.
	Writers []scrapemate.ResultWriter `validate:"required,gt=0"`
	// InitJob is the job to initialize the app with.
	InitJob scrapemate.IJob
	// ExitOnInactivityDuration is whether to exit the app when there are no more jobs to run.
	ExitOnInactivityDuration time.Duration
	// Proxies are the proxies to use for the app.
	Proxies []string
	// BrowserReuseLimit is the limit of browser reuse.
	// Only applicable when using JavaScript renderer.
	// By default it is 0, which means the browser will be reused indefinitely.
	BrowserReuseLimit int
	// PageReuseLimit is the limit of page reuse.
	// Only applicable when using JavaScript renderer.
	// By default it is 0, which means the page will not be reused.
	PageReuseLimit int
}

func NewConfig

func NewConfig(writers []scrapemate.ResultWriter, options ...func(*Config) error) (*Config, error)

NewConfig creates a new config with default values.

type ScrapemateApp

type ScrapemateApp struct {
	// contains filtered or unexported fields
}

func NewScrapeMateApp

func NewScrapeMateApp(cfg *Config) (*ScrapemateApp, error)

NewScrapemateApp creates a new ScrapemateApp.

func (*ScrapemateApp) Close

func (app *ScrapemateApp) Close() error

Close closes the app.

func (*ScrapemateApp) Start

func (app *ScrapemateApp) Start(ctx context.Context, seedJobs ...scrapemate.IJob) error

Start starts the app.