Versions in this module Expand all Collapse all v0 v0.1.0 Aug 2, 2020 Changes in this version + const CRAWLER_FILE + const CRAWLER_IMAGE + const CRAWLER_VB_ATTACHMENTS + const DEFAULT_DL_JOBS + const DEFAULT_REDIRECTS + const PAGER_QUERY + const PAGER_URLCUT + const PAGER_VB4 + func Crawl(cc *CrawlContext) error + type CrawlContext struct + Cookies []*http.Cookie + Crawler CrawlerInterface + Pager PagerInterface + func NewCrawlContext(pager string, crawler string, defaultDir string) (*CrawlContext, error) + func (cc *CrawlContext) SetOptions(args []string) error + type CrawlerInterface interface + Crawl func(*url.URL) error + Finish func() + SetOptions func([]string) error + Setup func() + func NewFileCrawler(cc *CrawlContext) (CrawlerInterface, error) + func NewImageCrawler(cc *CrawlContext) (CrawlerInterface, error) + func NewVBAttachmentCrawler(cc *CrawlContext) (CrawlerInterface, error) + type FileCrawler struct + func (c FileCrawler) Finish() + func (c FileCrawler) SetOptions(args []string) error + func (c FileCrawler) Setup() + func (r *FileCrawler) Crawl(u *url.URL) error + type ImageCrawler struct + func (c ImageCrawler) Finish() + func (c ImageCrawler) Setup() + func (r *ImageCrawler) Crawl(u *url.URL) error + func (r *ImageCrawler) SetOptions(args []string) error + type PagerInterface interface + Next func() (*url.URL, error) + PageNum func() int + SetOptions func([]string) error + SetUrl func(string) error + func NewQueryPager(cc *CrawlContext) PagerInterface + func NewURLCuttingPager(cc *CrawlContext) PagerInterface + func NewVB4Pager(cc *CrawlContext) PagerInterface + type QueryPager struct + func (r *QueryPager) Next() (*url.URL, error) + func (r *QueryPager) PageNum() int + func (r *QueryPager) SetOptions(args []string) error + func (r *QueryPager) SetUrl(addr string) error + type URLCuttingPager struct + func (r *URLCuttingPager) Next() (*url.URL, error) + func (r *URLCuttingPager) PageNum() int + func (r *URLCuttingPager) SetOptions(args []string) error + func (r *URLCuttingPager) SetUrl(addr string) error + type VB4Pager struct + End int + Start int + Thread *url.URL + func (r *VB4Pager) Next() (*url.URL, error) + func (r *VB4Pager) PageNum() int + func (r *VB4Pager) SetOptions(args []string) error + func (r *VB4Pager) SetUrl(addr string) error + type VBAttachmentCrawler struct + func (c VBAttachmentCrawler) Finish() + func (c VBAttachmentCrawler) Setup() + func (r *VBAttachmentCrawler) Crawl(u *url.URL) error + func (r *VBAttachmentCrawler) SetOptions(args []string) error