crawl

Documentation

Index

• Variables
• type Crawl
  • func New() Crawl
• type Element
• type Meta
• type Middleware
  • func LoggingMiddleware(logger log.Logger) Middleware
• type Result

Variables

var (
	// ErrTooManyRedirects too many redirects
	ErrTooManyRedirects = errors.New("too many redirects")
	// ErrMissingLocation missing location
	ErrMissingLocation = errors.New("missing location")
)

Types

type Crawl

type Crawl interface {
	Do(url string) (Result, error)
}

Crawl scrape page interface.

func New

func New() Crawl

New create crawl instance.

type Element

type Element struct {
	TagName string `json:"tag_name"`
	Count   int    `json:"count"`
}

Element statistics on the tags of the loaded page.

type Meta

type Meta struct {
	Status        int    `json:"status"`
	ContentType   string `json:"content_type,omitempty"`
	ContentLength int    `json:"content_length,omitempty"`
}

Meta meta information of the requested page.

type Middleware

type Middleware func(Crawl) Crawl

Middleware describes a service middleware.

func LoggingMiddleware

func LoggingMiddleware(logger log.Logger) Middleware

LoggingMiddleware takes a logger as a dependency and returns a LoggingMiddleware.

type Result

type Result struct {
	URL      string    `json:"url"`
	Meta     Meta      `json:"meta"`
	Elements []Element `json:"elements"`
	Err      error     `json:"-"`
}

Result parsing result.