scraper

package

v0.0.3 Latest Latest Go to latest Published: Oct 15, 2017 License: Apache-2.0 Imports: 19 Imported by: 1

Details

This section is empty.

This section is empty.

func NewAsset(url string) *asset

func NewImage(url string) *image

func NewIndex(url string) *index

func NewScript(url string) *script

func NewStylesheet(url string) *stylesheet

type Resource interface {
	Url() string
	Transform(scraper *Scraper, srcUrl *url.URL, reader io.Reader) (io.Reader, error)
}

type Scraper struct {
	SnapshotDir string
	Log         *zap.Logger
	Id          string
	// contains filtered or unexported fields
}

func NewScraper(ctx context.Context, logger *zap.Logger, id, seed string) *Scraper

func (s *Scraper) Cleanup() error

func (s *Scraper) Scrape() error