scrapers

Documentation

Index

• type AsyncSiteScraper
  • func NewAsyncSiteScraper(siteURL string) (*AsyncSiteScraper, error)
  • func (s *AsyncSiteScraper) Scrape() error
• type BaseScraper
  • func (s *BaseScraper) UniqueLinks() []string
• type PageScraper
  • func NewPageScraper(siteURL string, pageURL string) *PageScraper
  • func (s *PageScraper) Scrape() error
• type Parser
  • func NewParser(reader io.Reader) *Parser
  • func (p *Parser) Links() []string
  • func (p *Parser) Parse() error
• type Scraper
• type SyncSiteScraper
  • func NewSyncSiteScraper(siteURL string) (*SyncSiteScraper, error)
  • func (s *SyncSiteScraper) Scrape() error

Types

type AsyncSiteScraper

type AsyncSiteScraper struct {
	BaseScraper
	// contains filtered or unexported fields
}

AsyncSiteScraper scrapes an entire site asynchronously.

func NewAsyncSiteScraper

func NewAsyncSiteScraper(siteURL string) (*AsyncSiteScraper, error)

NewAsyncSiteScraper initializes a new AsyncSiteScraper.

func (*AsyncSiteScraper) Scrape

func (s *AsyncSiteScraper) Scrape() error

Scrape the site for links.

type BaseScraper

type BaseScraper struct {
	// contains filtered or unexported fields
}

BaseScraper is the base scraper for HTML sites/pages.

func (*BaseScraper) UniqueLinks

func (s *BaseScraper) UniqueLinks() []string

UniqueLinks returns all the unique internal links that where found.

type PageScraper

type PageScraper struct {
	BaseScraper
	// contains filtered or unexported fields
}

PageScraper scrapes a website and finds all pages.

func NewPageScraper

func NewPageScraper(siteURL string, pageURL string) *PageScraper

NewPageScraper initializes a PageScraper.

func (*PageScraper) Scrape

func (s *PageScraper) Scrape() error

Scrape the page for links.

type Parser

type Parser struct {
	// contains filtered or unexported fields
}

Parser parses a HTML page and stores the found links.

func NewParser

func NewParser(reader io.Reader) *Parser

NewParser initializes a new Parser.

func (*Parser) Links

func (p *Parser) Links() []string

Links returns the found links.

func (*Parser) Parse

func (p *Parser) Parse() error

Parse a HTML page and store the found links.

type Scraper

type Scraper interface {
	Scrape() error
	UniqueLinks() []string
}

Scraper defines a site/page scraper.

type SyncSiteScraper

type SyncSiteScraper struct {
	BaseScraper
	// contains filtered or unexported fields
}

SyncSiteScraper scrapes an entire site synchronously.

func NewSyncSiteScraper

func NewSyncSiteScraper(siteURL string) (*SyncSiteScraper, error)

NewSyncSiteScraper initializes a new SyncSiteScraper.

func (*SyncSiteScraper) Scrape

func (s *SyncSiteScraper) Scrape() error

Scrape the site for links.