crawler

Documentation

Index

• type Crawler
  • func NewCrawler(opts *Options) *Crawler
  • func (crawler *Crawler) Run(shutdown chan os.Signal) error
  • func (crawler Crawler) Status() *Status
• type DefaultMatcher
  • func (m DefaultMatcher) Match(u *url.URL) bool
• type DefaultParser
  • func (p *DefaultParser) Links(res *http.Response) map[string]*url.URL
• type Matcher
• type Node
  • func (n *Node) Data() *ResponseData
  • func (n Node) ID() int
  • func (n *Node) Record(d time.Duration, res *http.Response)
• type Nodes
  • func (n Nodes) Len() int
  • func (n Nodes) Less(i, j int) bool
  • func (n Nodes) Swap(i, j int)
• type Options
• type Parser
• type Recorder
  • func NewRecorder(opts *Options) *Recorder
  • func (r *Recorder) Nodes() []*Node
  • func (r *Recorder) RecordLink(parent *Node, url *url.URL) *Node
  • func (r *Recorder) RecordResponse(ctx *fetchbot.Context, res *http.Response, err error) *Node
• type ResponseData
• type Scheduler
  • func NewScheduler(opts *Options) *Scheduler
  • func (s *Scheduler) Depth() int
  • func (s *Scheduler) Run(q *fetchbot.Queue)
• type Status

Types

type Crawler

type Crawler struct {
	// contains filtered or unexported fields
}

func NewCrawler

func NewCrawler(opts *Options) *Crawler

NewCrawler returns a new Crawler structure

func (*Crawler) Run

func (crawler *Crawler) Run(shutdown chan os.Signal) error

func (Crawler) Status

func (crawler Crawler) Status() *Status

type DefaultMatcher

type DefaultMatcher struct {
	Seed *url.URL
}

Match is used to evaluate a Node and determine if it should be crawled.

func (DefaultMatcher) Match

func (m DefaultMatcher) Match(u *url.URL) bool

Match matches a given Node

type DefaultParser

type DefaultParser struct {
	Seed *url.URL
}

func (*DefaultParser) Links

func (p *DefaultParser) Links(res *http.Response) map[string]*url.URL

Links returns an array of unique, normalized url.URL

type Matcher

type Matcher interface {
	Match(*url.URL) bool
}

Matcher is used to evaluate a node and determine if it should be crawled.

type Node

type Node struct {
	URL *url.URL
	Err error
	// contains filtered or unexported fields
}

func (*Node) Data

func (n *Node) Data() *ResponseData

func (Node) ID

func (n Node) ID() int

func (*Node) Record

func (n *Node) Record(d time.Duration, res *http.Response)

Record an HTTP Response

type Nodes

type Nodes []*Node

Nodes implements a sortable array of Nodes

func (Nodes) Len

func (n Nodes) Len() int

func (Nodes) Less

func (n Nodes) Less(i, j int) bool

func (Nodes) Swap

func (n Nodes) Swap(i, j int)

type Options

type Options struct {
	Seed     *url.URL
	Logger   *log.Logger
	MaxDepth int
	Matcher  Matcher
	Parser   Parser
	Debug    bool
}

type Parser

type Parser interface {
	Links(*http.Response) map[string]*url.URL
}

Parser parses an HTTP request returning specific information

type Recorder

type Recorder struct {
	// contains filtered or unexported fields
}

Recorder saves the result of crawling a Node in a graph structure.

func NewRecorder

func NewRecorder(opts *Options) *Recorder

func (*Recorder) Nodes

func (r *Recorder) Nodes() []*Node

func (*Recorder) RecordLink

func (r *Recorder) RecordLink(parent *Node, url *url.URL) *Node

RecordLink creates (or updates) a node and adds an Edge to the Graph

func (*Recorder) RecordResponse

func (r *Recorder) RecordResponse(ctx *fetchbot.Context, res *http.Response, err error) *Node

RecordResponse creates (or updates) a node with the http.Response

type ResponseData

type ResponseData struct {
	Code     int
	Headers  http.Header
	Length   int64
	Duration time.Duration
}

type Scheduler

type Scheduler struct {
	// contains filtered or unexported fields
}

Scheduler schedules a Node for crawling.

func NewScheduler

func NewScheduler(opts *Options) *Scheduler

func (*Scheduler) Depth

func (s *Scheduler) Depth() int

func (*Scheduler) Run

func (s *Scheduler) Run(q *fetchbot.Queue)

type Status

type Status struct {
	Running  bool
	Depth    int
	Options  *Options
	Nodes    []*Node
	Schedule []string
	Logs     []string
}