Why Go
- Case Studies
  
  Common problems companies solve with Go
- Use Cases
  
  Stories about how and why companies use Go
- Security Policy
  
  How Go can help keep you secure by default
Learn
Docs
- Effective Go
  
  Tips for writing clear, performant, and idiomatic Go code
- Go User Manual
  
  A complete introduction to building software with Go
- Standard library
  
  Reference documentation for Go's standard library
- Release Notes
  
  Learn what's new in each Go release
Packages
Community
- Recorded Talks
  
  Videos from prior events
- Meetups
  
  Meet other local Go developers
- Conferences
  
  Learn and network with Go developers from around the world
- Go blog
  
  The Go project's official blog.
- Go project
  
  Get help and stay informed from Go
- Get connected

spider

package module

Go to main page

Versions in this module

v0

v0.3.1

Jan 20, 2016

v0.3.0

Nov 2, 2015

Changes in this version

+ func Add(sched Schedule, spider Spider)

+ func AddFunc(sched Schedule, url string, fn func(*Context) error)

+ func Delete(url string, fn spinFunc) *spiderFunc

+ func Get(url string, fn spinFunc) *spiderFunc

+ func NewHTTPSpider(method, url string, body io.Reader, fn spinFunc) *spiderFunc

+ func Post(url string, body io.Reader, fn spinFunc) *spiderFunc

+ func Put(url string, body io.Reader, fn spinFunc) *spiderFunc

+ func Start()

+ func Stop()

type Context

+ func NewHTTPContext(method, url string, body io.Reader) (*Context, error)

+ type Entries []*Entry

+ func (e Entries) Len() int

+ func (e Entries) Less(i, j int) bool

+ func (e Entries) Swap(i, j int)

+ type Entry struct

+ Ctx *Context

+ Next time.Time

+ Schedule Schedule

+ Spider Spider

+ type InMemory struct

+ func NewScheduler() *InMemory

+ func (in *InMemory) Add(sched Schedule, spider Spider)

+ func (in *InMemory) AddFunc(sched Schedule, url string, fn func(*Context) error)

+ func (in *InMemory) AddWithCtx(sched Schedule, spider Spider, ctx *Context)

+ func (in *InMemory) Start()

+ func (in *InMemory) Stop()

v0.2.0

Nov 1, 2015

Changes in this version

+ type Schedule interface

+ Next func(time.Time) time.Time

v0.1.0

Oct 11, 2015

Changes in this version

+ var ErrNoClient = errors.New("No request has been set")

+ var ErrNoRequest = errors.New("No request has been set")

+ func NewKVStore() *store

+ type BackoffCondition func(*http.Response) error

+ func ErrorIfStatusCodeIsNot(status int) BackoffCondition

+ type BaseSpiderScheduler interface

+ NextSpin func() (time.Duration, bool)

+ NextSpinChan func() (<-chan struct{}, <-chan struct{})

+ type Context struct

+ Children []*Context

+ Client *http.Client

+ Parent *Context

+ func NewContext() *Context

+ func (c *Context) Cookies() []*http.Cookie

+ func (c *Context) DoRequest() (*http.Response, error)

+ func (c *Context) DoRequestWithExponentialBackOff(condition BackoffCondition, b backoff.BackOff) (*http.Response, error)

+ func (c *Context) ExtendWithRequest(ctx Context, r *http.Request) *Context

+ func (c *Context) Get(key string) interface{}

+ func (c *Context) HTMLParser() (*goquery.Document, error)

+ func (c *Context) JSONParser() (*simplejson.Json, error)

+ func (c *Context) NewClient() (*http.Client, error)

+ func (c *Context) NewCookieJar() (*cookiejar.Jar, error)

+ func (c *Context) RAWContent() ([]byte, error)

+ func (c *Context) Request() *http.Request

+ func (c *Context) ResetClient() (*http.Client, error)

+ func (c *Context) ResetCookies() error

+ func (c *Context) Response() *http.Response

+ func (c *Context) Set(key string, value interface{})

+ func (c *Context) SetParent(parent *Context)

+ func (c *Context) SetRequest(req *http.Request)

+ func (c *Context) SetResponse(res *http.Response)

+ type EveryFunc func() time.Duration

+ type Scheduler interface

+ Handle func(Spider) SpiderScheduler

+ Start func() error

+ type Spider interface

+ Setup func(*Context) (*Context, error)

+ Spin func(*Context) error

+ type SpiderScheduler interface

+ After func(time.Duration) SpiderScheduler

+ Delay func() time.Duration

+ Duplicate func(int64) SpiderScheduler

+ Every func(time.Duration) SpiderScheduler

+ EveryFunc func(EveryFunc) SpiderScheduler

+ EveryRandom func(time.Duration, time.Duration, time.Duration) SpiderScheduler

+ From func(time.Time) SpiderScheduler

+ NumGoroutine func() int64

+ To func(time.Time) SpiderScheduler

+ type SpinnerFunc func(ctx *Context) error

+ func (s SpinnerFunc) Spin(ctx *Context) error