Versions in this module Expand all Collapse all v1 v1.2.0 Sep 11, 2019 v1.1.2 Sep 4, 2017 v1.1.1 Sep 4, 2017 v1.1.0 Jul 6, 2017 Changes in this version type Queue + func (q *Queue) Done() <-chan struct{} v1.0.0 Mar 24, 2017 Changes in this version + const DefaultCrawlDelay + const DefaultUserAgent + const DefaultWorkerIdleTTL + var ErrDisallowed = errors.New("fetchbot: disallowed by robots.txt") + var ErrEmptyHost = errors.New("fetchbot: invalid empty host") + var ErrQueueClosed = errors.New("fetchbot: send on a closed queue") + type BasicAuthProvider interface + BasicAuth func() (user string, pwd string) + type Cmd struct + M string + U *url.URL + func (c *Cmd) Method() string + func (c *Cmd) URL() *url.URL + type Command interface + Method func() string + URL func() *url.URL + type Context struct + Cmd Command + Q *Queue + type CookiesProvider interface + Cookies func() []*http.Cookie + type DebugInfo struct + NumHosts int + type Doer interface + Do func(*http.Request) (*http.Response, error) + type Fetcher struct + AutoClose bool + CrawlDelay time.Duration + DisablePoliteness bool + Handler Handler + HttpClient Doer + UserAgent string + WorkerIdleTTL time.Duration + func New(h Handler) *Fetcher + func (f *Fetcher) Debug() <-chan *DebugInfo + func (f *Fetcher) Start() *Queue + type Handler interface + Handle func(*Context, *http.Response, error) + type HandlerCmd struct + func NewHandlerCmd(method, rawURL string, fn func(*Context, *http.Response, error)) (*HandlerCmd, error) + type HandlerFunc func(*Context, *http.Response, error) + func (h HandlerFunc) Handle(ctx *Context, res *http.Response, err error) + type HeaderProvider interface + Header func() http.Header + type Mux struct + DefaultHandler Handler + func NewMux() *Mux + func (mux *Mux) Handle(ctx *Context, res *http.Response, err error) + func (mux *Mux) HandleError(err error, h Handler) + func (mux *Mux) HandleErrors(h Handler) + func (mux *Mux) Response() *ResponseMatcher + type Queue struct + func (q *Queue) Block() + func (q *Queue) Cancel() error + func (q *Queue) Close() error + func (q *Queue) Send(c Command) error + func (q *Queue) SendString(method string, rawurl ...string) (int, error) + func (q *Queue) SendStringGet(rawurl ...string) (int, error) + func (q *Queue) SendStringHead(rawurl ...string) (int, error) + type ReaderProvider interface + Reader func() io.Reader + type ResponseMatcher struct + func (r *ResponseMatcher) ContentType(ct string) *ResponseMatcher + func (r *ResponseMatcher) Custom(predicate func(*http.Response) bool) *ResponseMatcher + func (r *ResponseMatcher) Handler(h Handler) *ResponseMatcher + func (r *ResponseMatcher) Host(host string) *ResponseMatcher + func (r *ResponseMatcher) Method(m string) *ResponseMatcher + func (r *ResponseMatcher) Path(p string) *ResponseMatcher + func (r *ResponseMatcher) Scheme(scheme string) *ResponseMatcher + func (r *ResponseMatcher) Status(code int) *ResponseMatcher + func (r *ResponseMatcher) StatusRange(min, max int) *ResponseMatcher + type ValuesProvider interface + Values func() url.Values
Go to main page