scraper

Documentation

Index

• Variables
• func GetPageFilePath(url *url.URL) string
• type Scraper
  • func New(logger *zap.Logger, startURL string) (*Scraper, error)
  • func (s *Scraper) GetFilePath(url *url.URL, isAPage bool) string
  • func (s *Scraper) RemoveAnchor(path string) string
  • func (s *Scraper) SetExcludes(excludes []string) error
  • func (s *Scraper) SetIncludes(includes []string) error
  • func (s *Scraper) Start() error

Variables

var (
	// PageExtension is the file extension that downloaded pages get
	PageExtension = ".html"
	// PageDirIndex is the file name of the index file for every dir
	PageDirIndex = "index" + PageExtension
)

Functions

func GetPageFilePath

func GetPageFilePath(url *url.URL) string

GetPageFilePath returns a filename for a URL that represents a page.

Types

type Scraper

type Scraper struct {
	// Configuration
	ImageQuality    uint
	MaxDepth        uint
	OutputDirectory string
	Username        string
	Password        string

	URL *url.URL
	// contains filtered or unexported fields
}

Scraper contains all scraping data

func New

func New(logger *zap.Logger, startURL string) (*Scraper, error)

New creates a new Scraper instance

func (*Scraper) GetFilePath

func (s *Scraper) GetFilePath(url *url.URL, isAPage bool) string

GetFilePath returns a file path for a URL to store the URL content in

func (*Scraper) RemoveAnchor

func (s *Scraper) RemoveAnchor(path string) string

RemoveAnchor removes anchors from URLS

func (*Scraper) SetExcludes

func (s *Scraper) SetExcludes(excludes []string) error

SetExcludes sets and checks the exclusions regular expressions.

func (*Scraper) SetIncludes

func (s *Scraper) SetIncludes(includes []string) error

SetIncludes sets and checks the inclusion regular expressions.

func (*Scraper) Start

func (s *Scraper) Start() error

Start starts the scraping