colly

package module

Go to main page

Versions in this module

v1

v1.2.0

Feb 13, 2019

Changes in this version

+ const ProxyURLKey

type Collector

+ CheckHead bool

+ func (c *Collector) Head(URL string) error

type Request

+ ProxyURL string

v1.2.0-beta1

May 15, 2020

v1.1.0

Aug 28, 2018

Changes in this version

type HTMLElement

+ Index int

+ func (h *HTMLElement) ForEachWithBreak(goquerySelector string, callback func(int, *HTMLElement) bool)

type XMLElement

+ func (h *XMLElement) ChildTexts(xpathQuery string) []string

v1.0.0

May 12, 2018

Changes in this version

+ var ErrAlreadyVisited = errors.New("URL already visited")

+ var ErrForbiddenDomain = errors.New("Forbidden domain")

+ var ErrForbiddenURL = errors.New("ForbiddenURL")

+ var ErrMaxDepth = errors.New("Max depth limit reached")

+ var ErrMissingURL = errors.New("Missing URL")

+ var ErrNoCookieJar = errors.New("Cookie jar is not available")

+ var ErrNoPattern = errors.New("No pattern defined in LimitRule")

+ var ErrNoURLFiltersMatch = errors.New("No URLFilters match")

+ var ErrRobotsTxtBlocked = errors.New("URL blocked by robots.txt")

+ func AllowURLRevisit() func(*Collector)

+ func AllowedDomains(domains ...string) func(*Collector)

+ func Async(a bool) func(*Collector)

+ func CacheDir(path string) func(*Collector)

+ func Debugger(d debug.Debugger) func(*Collector)

+ func DetectCharset() func(*Collector)

+ func DisallowedDomains(domains ...string) func(*Collector)

+ func DisallowedURLFilters(filters ...*regexp.Regexp) func(*Collector)

+ func ID(id uint32) func(*Collector)

+ func IgnoreRobotsTxt() func(*Collector)

+ func MaxBodySize(sizeInBytes int) func(*Collector)

+ func MaxDepth(depth int) func(*Collector)

+ func ParseHTTPErrorResponse() func(*Collector)

+ func SanitizeFileName(fileName string) string

+ func URLFilters(filters ...*regexp.Regexp) func(*Collector)

+ func UnmarshalHTML(v interface{}, s *goquery.Selection) error

+ func UserAgent(ua string) func(*Collector)

+ type Collector struct

+ AllowURLRevisit bool

+ AllowedDomains []string

+ Async bool

+ CacheDir string

+ DetectCharset bool

+ DisallowedDomains []string

+ DisallowedURLFilters []*regexp.Regexp

+ ID uint32

+ IgnoreRobotsTxt bool

+ MaxBodySize int

+ MaxDepth int

+ ParseHTTPErrorResponse bool

+ RedirectHandler func(req *http.Request, via []*http.Request) error

+ URLFilters []*regexp.Regexp

+ UserAgent string

+ func NewCollector(options ...func(*Collector)) *Collector

+ func (c *Collector) Appengine(req *http.Request)

+ func (c *Collector) Clone() *Collector

+ func (c *Collector) Cookies(URL string) []*http.Cookie

+ func (c *Collector) DisableCookies()

+ func (c *Collector) Init()

+ func (c *Collector) Limit(rule *LimitRule) error

+ func (c *Collector) Limits(rules []*LimitRule) error

+ func (c *Collector) OnError(f ErrorCallback)

+ func (c *Collector) OnHTML(goquerySelector string, f HTMLCallback)

+ func (c *Collector) OnHTMLDetach(goquerySelector string)

+ func (c *Collector) OnRequest(f RequestCallback)

+ func (c *Collector) OnResponse(f ResponseCallback)

+ func (c *Collector) OnScraped(f ScrapedCallback)

+ func (c *Collector) OnXML(xpathQuery string, f XMLCallback)

+ func (c *Collector) OnXMLDetach(xpathQuery string)

+ func (c *Collector) Post(URL string, requestData map[string]string) error

+ func (c *Collector) PostMultipart(URL string, requestData map[string][]byte) error

+ func (c *Collector) PostRaw(URL string, requestData []byte) error

+ func (c *Collector) Request(method, URL string, requestData io.Reader, ctx *Context, hdr http.Header) error

+ func (c *Collector) SetCookieJar(j *cookiejar.Jar)

+ func (c *Collector) SetCookies(URL string, cookies []*http.Cookie) error

+ func (c *Collector) SetDebugger(d debug.Debugger)

+ func (c *Collector) SetProxy(proxyURL string) error

+ func (c *Collector) SetProxyFunc(p ProxyFunc)

+ func (c *Collector) SetRequestTimeout(timeout time.Duration)

+ func (c *Collector) SetStorage(s storage.Storage) error

+ func (c *Collector) String() string

+ func (c *Collector) UnmarshalRequest(r []byte) (*Request, error)

+ func (c *Collector) Visit(URL string) error

+ func (c *Collector) Wait()

+ func (c *Collector) WithTransport(transport http.RoundTripper)

+ type Context struct

+ func NewContext() *Context

+ func (c *Context) ForEach(fn func(k string, v interface{}) interface{}) []interface{}

+ func (c *Context) Get(key string) string

+ func (c *Context) GetAny(key string) interface{}

+ func (c *Context) MarshalBinary() (_ []byte, _ error)

+ func (c *Context) Put(key string, value interface{})

+ func (c *Context) UnmarshalBinary(_ []byte) error

+ type ErrorCallback func(*Response, error)

+ type HTMLCallback func(*HTMLElement)

+ type HTMLElement struct

+ DOM *goquery.Selection

+ Name string

+ Request *Request

+ Response *Response

+ Text string

+ func NewHTMLElementFromSelectionNode(resp *Response, s *goquery.Selection, n *html.Node) *HTMLElement

+ func (h *HTMLElement) Attr(k string) string

+ func (h *HTMLElement) ChildAttr(goquerySelector, attrName string) string

+ func (h *HTMLElement) ChildAttrs(goquerySelector, attrName string) []string

+ func (h *HTMLElement) ChildText(goquerySelector string) string

+ func (h *HTMLElement) ForEach(goquerySelector string, callback func(int, *HTMLElement))

+ func (h *HTMLElement) Unmarshal(v interface{}) error

+ type LimitRule struct

+ Delay time.Duration

+ DomainGlob string

+ DomainRegexp string

+ Parallelism int

+ RandomDelay time.Duration

+ func (r *LimitRule) Init() error

+ func (r *LimitRule) Match(domain string) bool

+ type ProxyFunc func(*http.Request) (*url.URL, error)

+ type Request struct

+ Body io.Reader

+ Ctx *Context

+ Depth int

+ Headers *http.Header

+ ID uint32

+ Method string

+ ResponseCharacterEncoding string

+ URL *url.URL

+ func (r *Request) Abort()

+ func (r *Request) AbsoluteURL(u string) string

+ func (r *Request) Do() error

+ func (r *Request) Marshal() ([]byte, error)

+ func (r *Request) New(method, URL string, body io.Reader) (*Request, error)

+ func (r *Request) Post(URL string, requestData map[string]string) error

+ func (r *Request) PostMultipart(URL string, requestData map[string][]byte) error

+ func (r *Request) PostRaw(URL string, requestData []byte) error

+ func (r *Request) Retry() error

+ func (r *Request) Visit(URL string) error

+ type RequestCallback func(*Request)

+ type Response struct

+ Body []byte

+ Ctx *Context

+ Headers *http.Header

+ Request *Request

+ StatusCode int

+ func (r *Response) FileName() string

+ func (r *Response) Save(fileName string) error

+ type ResponseCallback func(*Response)

+ type ScrapedCallback func(*Response)

+ type XMLCallback func(*XMLElement)

+ type XMLElement struct

+ DOM interface{}

+ Name string

+ Request *Request

+ Response *Response

+ Text string

+ func NewXMLElementFromHTMLNode(resp *Response, s *html.Node) *XMLElement

+ func NewXMLElementFromXMLNode(resp *Response, s *xmlquery.Node) *XMLElement

+ func (h *XMLElement) Attr(k string) string

+ func (h *XMLElement) ChildAttr(xpathQuery, attrName string) string

+ func (h *XMLElement) ChildAttrs(xpathQuery, attrName string) []string

+ func (h *XMLElement) ChildText(xpathQuery string) string

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL