colly

package module

Go to main page

Versions in this module

v1

v1.2.0

Feb 13, 2019

v1.1.0

Aug 28, 2018

v1.0.6

Feb 29, 2024

v1.0.4

Feb 29, 2024

v1.0.1

Feb 29, 2024

Changes in this version

+ const ProxyURLKey

+ var ErrAbortedAfterHeaders = errors.New("Aborted after receiving response headers")

+ var ErrEmptyProxyURL = errors.New("Proxy URL list is empty")

+ var ErrMaxRequests = errors.New("Max Requests limit reached")

+ var ErrQueueFull = errors.New("Queue MaxSize reached")

+ type AlreadyVisitedError struct

+ Destination *url.URL

+ func (e *AlreadyVisitedError) Error() string

type Collector

+ CheckHead bool

+ Context context.Context

+ Headers *http.Header

+ MaxRequests uint32

+ TraceHTTP bool

+ func (c *Collector) HasPosted(URL string, requestData map[string]string) (bool, error)

+ func (c *Collector) HasVisited(URL string) (bool, error)

+ func (c *Collector) Head(URL string) error

+ func (c *Collector) OnResponseHeaders(f ResponseHeadersCallback)

+ func (c *Collector) SetClient(client *http.Client)

+ func (c *Collector) SetRedirectHandler(f func(req *http.Request, via []*http.Request) error)

+ type CollectorOption func(*Collector)

+ func CheckHead() CollectorOption

+ func Headers(headers map[string]string) CollectorOption

+ func MaxRequests(max uint32) CollectorOption

+ func StdlibContext(ctx context.Context) CollectorOption

+ func TraceHTTP() CollectorOption

type HTMLElement

+ Index int

+ func (h *HTMLElement) ChildTexts(goquerySelector string) []string

+ func (h *HTMLElement) ForEachWithBreak(goquerySelector string, callback func(int, *HTMLElement) bool)

+ func (h *HTMLElement) UnmarshalWithMap(v interface{}, structMap map[string]string) error

+ type HTTPTrace struct

+ ConnectDuration time.Duration

+ FirstByteDuration time.Duration

+ func (ht *HTTPTrace) WithTrace(req *http.Request) *http.Request

type Request

+ Host string

+ ProxyURL string

+ func (r *Request) HasVisited(URL string) (bool, error)

type Response

+ Trace *HTTPTrace

+ type ResponseHeadersCallback func(*Response)

type XMLElement

+ func (h *XMLElement) ChildTexts(xpathQuery string) []string

v1.0.0

May 12, 2018

Changes in this version

+ var ErrAlreadyVisited = errors.New("URL already visited")

+ var ErrForbiddenDomain = errors.New("Forbidden domain")

+ var ErrForbiddenURL = errors.New("ForbiddenURL")

+ var ErrMaxDepth = errors.New("Max depth limit reached")

+ var ErrMissingURL = errors.New("Missing URL")

+ var ErrNoCookieJar = errors.New("Cookie jar is not available")

+ var ErrNoPattern = errors.New("No pattern defined in LimitRule")

+ var ErrNoURLFiltersMatch = errors.New("No URLFilters match")

+ var ErrRobotsTxtBlocked = errors.New("URL blocked by robots.txt")

+ func AllowURLRevisit() func(*Collector)

+ func AllowedDomains(domains ...string) func(*Collector)

+ func Async(a bool) func(*Collector)

+ func CacheDir(path string) func(*Collector)

+ func Debugger(d debug.Debugger) func(*Collector)

+ func DetectCharset() func(*Collector)

+ func DisallowedDomains(domains ...string) func(*Collector)

+ func DisallowedURLFilters(filters ...*regexp.Regexp) func(*Collector)

+ func ID(id uint32) func(*Collector)

+ func IgnoreRobotsTxt() func(*Collector)

+ func MaxBodySize(sizeInBytes int) func(*Collector)

+ func MaxDepth(depth int) func(*Collector)

+ func ParseHTTPErrorResponse() func(*Collector)

+ func SanitizeFileName(fileName string) string

+ func URLFilters(filters ...*regexp.Regexp) func(*Collector)

+ func UnmarshalHTML(v interface{}, s *goquery.Selection) error

+ func UserAgent(ua string) func(*Collector)

+ type Collector struct

+ AllowURLRevisit bool

+ AllowedDomains []string

+ Async bool

+ CacheDir string

+ DetectCharset bool

+ DisallowedDomains []string

+ DisallowedURLFilters []*regexp.Regexp

+ ID uint32

+ IgnoreRobotsTxt bool

+ MaxBodySize int

+ MaxDepth int

+ ParseHTTPErrorResponse bool

+ RedirectHandler func(req *http.Request, via []*http.Request) error

+ URLFilters []*regexp.Regexp

+ UserAgent string

+ func NewCollector(options ...func(*Collector)) *Collector

+ func (c *Collector) Appengine(req *http.Request)

+ func (c *Collector) Clone() *Collector

+ func (c *Collector) Cookies(URL string) []*http.Cookie

+ func (c *Collector) DisableCookies()

+ func (c *Collector) Init()

+ func (c *Collector) Limit(rule *LimitRule) error

+ func (c *Collector) Limits(rules []*LimitRule) error

+ func (c *Collector) OnError(f ErrorCallback)

+ func (c *Collector) OnHTML(goquerySelector string, f HTMLCallback)

+ func (c *Collector) OnHTMLDetach(goquerySelector string)

+ func (c *Collector) OnRequest(f RequestCallback)

+ func (c *Collector) OnResponse(f ResponseCallback)

+ func (c *Collector) OnScraped(f ScrapedCallback)

+ func (c *Collector) OnXML(xpathQuery string, f XMLCallback)

+ func (c *Collector) OnXMLDetach(xpathQuery string)

+ func (c *Collector) Post(URL string, requestData map[string]string) error

+ func (c *Collector) PostMultipart(URL string, requestData map[string][]byte) error

+ func (c *Collector) PostRaw(URL string, requestData []byte) error

+ func (c *Collector) Request(method, URL string, requestData io.Reader, ctx *Context, hdr http.Header) error

+ func (c *Collector) SetCookieJar(j *cookiejar.Jar)

+ func (c *Collector) SetCookies(URL string, cookies []*http.Cookie) error

+ func (c *Collector) SetDebugger(d debug.Debugger)

+ func (c *Collector) SetProxy(proxyURL string) error

+ func (c *Collector) SetProxyFunc(p ProxyFunc)

+ func (c *Collector) SetRequestTimeout(timeout time.Duration)

+ func (c *Collector) SetStorage(s storage.Storage) error

+ func (c *Collector) String() string

+ func (c *Collector) UnmarshalRequest(r []byte) (*Request, error)

+ func (c *Collector) Visit(URL string) error

+ func (c *Collector) Wait()

+ func (c *Collector) WithTransport(transport http.RoundTripper)

+ type Context struct

+ func NewContext() *Context

+ func (c *Context) ForEach(fn func(k string, v interface{}) interface{}) []interface{}

+ func (c *Context) Get(key string) string

+ func (c *Context) GetAny(key string) interface{}

+ func (c *Context) MarshalBinary() (_ []byte, _ error)

+ func (c *Context) Put(key string, value interface{})

+ func (c *Context) UnmarshalBinary(_ []byte) error

+ type ErrorCallback func(*Response, error)

+ type HTMLCallback func(*HTMLElement)

+ type HTMLElement struct

+ DOM *goquery.Selection

+ Name string

+ Request *Request

+ Response *Response

+ Text string

+ func NewHTMLElementFromSelectionNode(resp *Response, s *goquery.Selection, n *html.Node) *HTMLElement

+ func (h *HTMLElement) Attr(k string) string

+ func (h *HTMLElement) ChildAttr(goquerySelector, attrName string) string

+ func (h *HTMLElement) ChildAttrs(goquerySelector, attrName string) []string

+ func (h *HTMLElement) ChildText(goquerySelector string) string

+ func (h *HTMLElement) ForEach(goquerySelector string, callback func(int, *HTMLElement))

+ func (h *HTMLElement) Unmarshal(v interface{}) error

+ type LimitRule struct

+ Delay time.Duration

+ DomainGlob string

+ DomainRegexp string

+ Parallelism int

+ RandomDelay time.Duration

+ func (r *LimitRule) Init() error

+ func (r *LimitRule) Match(domain string) bool

+ type ProxyFunc func(*http.Request) (*url.URL, error)

+ type Request struct

+ Body io.Reader

+ Ctx *Context

+ Depth int

+ Headers *http.Header

+ ID uint32

+ Method string

+ ResponseCharacterEncoding string

+ URL *url.URL

+ func (r *Request) Abort()

+ func (r *Request) AbsoluteURL(u string) string

+ func (r *Request) Do() error

+ func (r *Request) Marshal() ([]byte, error)

+ func (r *Request) New(method, URL string, body io.Reader) (*Request, error)

+ func (r *Request) Post(URL string, requestData map[string]string) error

+ func (r *Request) PostMultipart(URL string, requestData map[string][]byte) error

+ func (r *Request) PostRaw(URL string, requestData []byte) error

+ func (r *Request) Retry() error

+ func (r *Request) Visit(URL string) error

+ type RequestCallback func(*Request)

+ type Response struct

+ Body []byte

+ Ctx *Context

+ Headers *http.Header

+ Request *Request

+ StatusCode int

+ func (r *Response) FileName() string

+ func (r *Response) Save(fileName string) error

+ type ResponseCallback func(*Response)

+ type ScrapedCallback func(*Response)

+ type XMLCallback func(*XMLElement)

+ type XMLElement struct

+ DOM interface{}

+ Name string

+ Request *Request

+ Response *Response

+ Text string

+ func NewXMLElementFromHTMLNode(resp *Response, s *html.Node) *XMLElement

+ func NewXMLElementFromXMLNode(resp *Response, s *xmlquery.Node) *XMLElement

+ func (h *XMLElement) Attr(k string) string

+ func (h *XMLElement) ChildAttr(xpathQuery, attrName string) string

+ func (h *XMLElement) ChildAttrs(xpathQuery, attrName string) []string

+ func (h *XMLElement) ChildText(xpathQuery string) string

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL