Why Go
- Case Studies
  
  Common problems companies solve with Go
- Use Cases
  
  Stories about how and why companies use Go
- Security Policy
  
  How Go can help keep you secure by default
Learn
Docs
- Effective Go
  
  Tips for writing clear, performant, and idiomatic Go code
- Go User Manual
  
  A complete introduction to building software with Go
- Standard library
  
  Reference documentation for Go's standard library
- Release Notes
  
  Learn what's new in each Go release
Packages
Community
- Recorded Talks
  
  Videos from prior events
- Meetups
  
  Meet other local Go developers
- Conferences
  
  Learn and network with Go developers from around the world
- Go blog
  
  The Go project's official blog.
- Go project
  
  Get help and stay informed from Go
- Get connected

colly

package module

Go to main page

Versions in this module

v1

v1.2.0

Feb 13, 2019

v1.1.0

Aug 28, 2018

v1.0.0

May 12, 2018

Changes in this version

+ var ErrAlreadyVisited = errors.New("URL already visited")

type Collector

+ RedirectHandler func(req *http.Request, via []*http.Request) error

v0

v0.0.2

Jul 13, 2022

v0.0.1

Jul 8, 2022

Changes in this version

+ const ProxyURLKey

+ var ErrAbortedAfterHeaders = errors.New("Aborted after receiving response headers")

+ var ErrEmptyProxyURL = errors.New("Proxy URL list is empty")

+ var ErrForbiddenDomain = errors.New("Forbidden domain")

+ var ErrForbiddenURL = errors.New("ForbiddenURL")

+ var ErrMaxDepth = errors.New("Max depth limit reached")

+ var ErrMissingURL = errors.New("Missing URL")

+ var ErrNoCookieJar = errors.New("Cookie jar is not available")

+ var ErrNoPattern = errors.New("No pattern defined in LimitRule")

+ var ErrNoURLFiltersMatch = errors.New("No URLFilters match")

+ var ErrQueueFull = errors.New("Queue MaxSize reached")

+ var ErrRobotsTxtBlocked = errors.New("URL blocked by robots.txt")

+ func CacheHandle(f NewCacheHandler) func(*Collector)

+ func SanitizeFileName(fileName string) string

+ func UnmarshalHTML(v interface{}, s *goquery.Selection, structMap map[string]string) error

+ type AlreadyVisitedError struct

+ Destination *url.URL

+ func (e *AlreadyVisitedError) Error() string

+ type CacheHandleFunc func(*http.Request, func() (*Response, error)) (*Response, error)

+ type CacheHandler interface

+ CacheHandle func(*http.Request, func() (*Response, error)) (*Response, error)

+ func NewDefaultCacheHandler(c *Collector) CacheHandler

+ type Collector struct

+ AllowURLRevisit bool

+ AllowedDomains []string

+ Async bool

+ CacheDir string

+ CacheHandle CacheHandleFunc

+ CheckHead bool

+ Context context.Context

+ DetectCharset bool

+ DisallowedDomains []string

+ DisallowedURLFilters []*regexp.Regexp

+ Headers *http.Header

+ ID uint32

+ IgnoreRobotsTxt bool

+ MaxBodySize int

+ MaxDepth int

+ ParseHTTPErrorResponse bool

+ TraceHTTP bool

+ URLFilters []*regexp.Regexp

+ UserAgent string

+ func NewCollector(options ...CollectorOption) *Collector

+ func (c *Collector) Appengine(ctx context.Context)

+ func (c *Collector) Clone() *Collector

+ func (c *Collector) Cookies(URL string) []*http.Cookie

+ func (c *Collector) DisableCookies()

+ func (c *Collector) HasPosted(URL string, requestData map[string]string) (bool, error)

+ func (c *Collector) HasVisited(URL string) (bool, error)

+ func (c *Collector) Head(URL string) error

+ func (c *Collector) Init()

+ func (c *Collector) Limit(rule *LimitRule) error

+ func (c *Collector) Limits(rules []*LimitRule) error

+ func (c *Collector) OnError(f ErrorCallback)

+ func (c *Collector) OnHTML(goquerySelector string, f HTMLCallback)

+ func (c *Collector) OnHTMLDetach(goquerySelector string)

+ func (c *Collector) OnRequest(f RequestCallback)

+ func (c *Collector) OnResponse(f ResponseCallback)

+ func (c *Collector) OnResponseHeaders(f ResponseHeadersCallback)

+ func (c *Collector) OnScraped(f ScrapedCallback)

+ func (c *Collector) OnXML(xpathQuery string, f XMLCallback)

+ func (c *Collector) OnXMLDetach(xpathQuery string)

+ func (c *Collector) Post(URL string, requestData map[string]string) error

+ func (c *Collector) PostMultipart(URL string, requestData map[string][]byte) error

+ func (c *Collector) PostRaw(URL string, requestData []byte) error

+ func (c *Collector) Request(method, URL string, requestData io.Reader, ctx *Context, hdr http.Header) error

+ func (c *Collector) SetClient(client *http.Client)

+ func (c *Collector) SetCookieJar(j http.CookieJar)

+ func (c *Collector) SetCookies(URL string, cookies []*http.Cookie) error

+ func (c *Collector) SetDebugger(d debug.Debugger)

+ func (c *Collector) SetProxy(proxyURL string) error

+ func (c *Collector) SetProxyFunc(p ProxyFunc)

+ func (c *Collector) SetRedirectHandler(f func(req *http.Request, via []*http.Request) error)

+ func (c *Collector) SetRequestTimeout(timeout time.Duration)

+ func (c *Collector) SetStorage(s storage.Storage) error

+ func (c *Collector) String() string

+ func (c *Collector) UnmarshalRequest(r []byte) (*Request, error)

+ func (c *Collector) Visit(URL string) error

+ func (c *Collector) Wait()

+ func (c *Collector) WithTransport(transport http.RoundTripper)

+ type CollectorOption func(*Collector)

+ func AllowURLRevisit() CollectorOption

+ func AllowedDomains(domains ...string) CollectorOption

+ func Async(a ...bool) CollectorOption

+ func CacheDir(path string) CollectorOption

+ func CheckHead() CollectorOption

+ func Debugger(d debug.Debugger) CollectorOption

+ func DetectCharset() CollectorOption

+ func DisallowedDomains(domains ...string) CollectorOption

+ func DisallowedURLFilters(filters ...*regexp.Regexp) CollectorOption

+ func Headers(headers map[string]string) CollectorOption

+ func ID(id uint32) CollectorOption

+ func IgnoreRobotsTxt() CollectorOption

+ func MaxBodySize(sizeInBytes int) CollectorOption

+ func MaxDepth(depth int) CollectorOption

+ func ParseHTTPErrorResponse() CollectorOption

+ func StdlibContext(ctx context.Context) CollectorOption

+ func TraceHTTP() CollectorOption

+ func URLFilters(filters ...*regexp.Regexp) CollectorOption

+ func UserAgent(ua string) CollectorOption

+ type Context struct

+ func NewContext() *Context

+ func (c *Context) ForEach(fn func(k string, v interface{}) interface{}) []interface{}

+ func (c *Context) Get(key string) string

+ func (c *Context) GetAny(key string) interface{}

+ func (c *Context) MarshalBinary() (_ []byte, _ error)

+ func (c *Context) Put(key string, value interface{})

+ func (c *Context) UnmarshalBinary(_ []byte) error

+ type ErrorCallback func(*Response, error)

+ type HTMLCallback func(*HTMLElement)

+ type HTMLElement struct

+ DOM *goquery.Selection

+ Index int

+ Name string

+ Request *Request

+ Response *Response

+ Text string

+ func NewHTMLElementFromSelectionNode(resp *Response, s *goquery.Selection, n *html.Node, idx int) *HTMLElement

+ func (h *HTMLElement) Attr(k string) string

+ func (h *HTMLElement) ChildAttr(goquerySelector, attrName string) string

+ func (h *HTMLElement) ChildAttrs(goquerySelector, attrName string) []string

+ func (h *HTMLElement) ChildText(goquerySelector string) string

+ func (h *HTMLElement) ChildTexts(goquerySelector string) []string

+ func (h *HTMLElement) ForEach(goquerySelector string, callback func(int, *HTMLElement))

+ func (h *HTMLElement) ForEachWithBreak(goquerySelector string, callback func(int, *HTMLElement) bool)

+ func (h *HTMLElement) Unmarshal(v interface{}) error

+ func (h *HTMLElement) UnmarshalWithMap(v interface{}, structMap map[string]string) error

+ type HTTPTrace struct

+ ConnectDuration time.Duration

+ FirstByteDuration time.Duration

+ func (ht *HTTPTrace) WithTrace(req *http.Request) *http.Request

+ type LimitRule struct

+ Delay time.Duration

+ DomainGlob string

+ DomainRegexp string

+ Parallelism int

+ RandomDelay time.Duration

+ func (r *LimitRule) Init() error

+ func (r *LimitRule) Match(domain string) bool

+ type NewCacheHandler func(c *Collector) CacheHandler

+ type ProxyFunc func(*http.Request) (*url.URL, error)

+ type Request struct

+ Body io.Reader

+ Ctx *Context

+ Depth int

+ Headers *http.Header

+ Host string

+ ID uint32

+ Method string

+ ProxyURL string

+ ResponseCharacterEncoding string

+ URL *url.URL

+ func (r *Request) Abort()

+ func (r *Request) AbsoluteURL(u string) string

+ func (r *Request) Do() error

+ func (r *Request) HasVisited(URL string) (bool, error)

+ func (r *Request) Marshal() ([]byte, error)

+ func (r *Request) New(method, URL string, body io.Reader) (*Request, error)

+ func (r *Request) Post(URL string, requestData map[string]string) error

+ func (r *Request) PostMultipart(URL string, requestData map[string][]byte) error

+ func (r *Request) PostRaw(URL string, requestData []byte) error

+ func (r *Request) Retry() error

+ func (r *Request) Visit(URL string) error

+ type RequestCallback func(*Request)

+ type Response struct

+ Body []byte

+ Ctx *Context

+ Headers *http.Header

+ Request *Request

+ StatusCode int

+ Trace *HTTPTrace

+ func (r *Response) FileName() string

+ func (r *Response) Save(fileName string) error

+ type ResponseCallback func(*Response)

+ type ResponseHeadersCallback func(*Response)

+ type ScrapedCallback func(*Response)

+ type XMLCallback func(*XMLElement)

+ type XMLElement struct

+ DOM interface{}

+ Name string

+ Request *Request

+ Response *Response

+ Text string

+ func NewXMLElementFromHTMLNode(resp *Response, s *html.Node) *XMLElement

+ func NewXMLElementFromXMLNode(resp *Response, s *xmlquery.Node) *XMLElement

+ func (h *XMLElement) Attr(k string) string

+ func (h *XMLElement) ChildAttr(xpathQuery, attrName string) string

+ func (h *XMLElement) ChildAttrs(xpathQuery, attrName string) []string

+ func (h *XMLElement) ChildText(xpathQuery string) string

+ func (h *XMLElement) ChildTexts(xpathQuery string) []string