Why Go
- Case Studies
  
  Common problems companies solve with Go
- Use Cases
  
  Stories about how and why companies use Go
- Security Policy
  
  How Go can help keep you secure by default
Learn
Docs
- Effective Go
  
  Tips for writing clear, performant, and idiomatic Go code
- Go User Manual
  
  A complete introduction to building software with Go
- Standard library
  
  Reference documentation for Go's standard library
- Release Notes
  
  Learn what's new in each Go release
Packages
Community
- Recorded Talks
  
  Videos from prior events
- Meetups
  
  Meet other local Go developers
- Conferences
  
  Learn and network with Go developers from around the world
- Go blog
  
  The Go project's official blog.
- Go project
  
  Get help and stay informed from Go
- Get connected

crawler

package module

Go to main page

Versions in this module

v0

v0.0.3

Jan 4, 2022

Changes in this version

+ func URLJoin(url string, path string) string

type CrawlEngine

+ func (eng *CrawlEngine) WaitTime(seconds time.Duration)

type Crawler

+ func (c *Crawler) AddRequest(req *Request)

+ func (c *Crawler) IsIdle() bool

+ func (c *Crawler) OnRedirect(callback RedirectCallback) *Crawler

+ func (c *Crawler) WaitTime(seconds time.Duration)

+ type History []*HistoryItem

+ func (h History) Append(request *Request, response *Response) History

+ type HistoryItem struct

+ Request *Request

+ Response *Response

type Meta

+ func (m Meta) Has(key string) bool

+ type RedirectCallback func(res *Response, req *Request, ctx *Context) *Request

type Request

+ History History

+ Host string

+ OriginURL string

+ func (req *Request) AddHeader(key string, value string) *Request

+ func (req *Request) AddMeta(key string, value interface{}) *Request

+ func (req *Request) Clone() *Request

+ func (req *Request) WithHost(host string) *Request

type Response

+ History History

+ X509CertChan []*x509.Certificate

+ X509Certificate *x509.Certificate

+ func (res *Response) Redirect(url string) *Request

type Settings

+ AutoParseHtml bool

+ MaxRedirectTimes int

+ SkipTLSVerify bool

+ Transport *http.Transport

v0.0.2

Jan 4, 2022

Changes in this version

+ type Args url.Values

+ type Context struct

+ Crawler *Crawler

+ Depth int32

+ Engine *CrawlEngine

+ LastRequest *Request

+ LastResponse *Response

+ Settings *Settings

+ func (ctx *Context) AddItem(item interface{})

+ func (ctx *Context) AddRequest(req *Request)

+ func (ctx *Context) Emit(item interface{})

+ type Cookies map[string]string

+ type CrawlEngine struct

+ ItemQueue chan *itemWrapper

+ ProcessingItemCount int32

+ RequestMetaMap *sync.Map

+ RequestQueue chan *Request

+ RequestingCount int32

+ Settings *Settings

+ func (eng *CrawlEngine) IsIdle() bool

+ func (eng *CrawlEngine) Start()

+ func (eng *CrawlEngine) StartProcessItems()

+ func (eng *CrawlEngine) StartProcessRequests()

+ func (eng *CrawlEngine) Wait()

+ type Crawler struct

+ Engine *CrawlEngine

+ ItemTypeFuncs map[string]ItemPipelineFunc

+ Name string

+ Pipelines []ItemPipeline

+ RequestErrorCallback RequestErrorCallback

+ ResponseCallback func(res *Response, ctx *Context)

+ Settings *Settings

+ StartRequests func(ctx *Context) []*Request

+ StartUrls []string

+ func NewCrawler(name string) *Crawler

+ func (c *Crawler) AddItemPipeline(p ItemPipeline) *Crawler

+ func (c *Crawler) AddItemPipelineFunc(f ItemPipelineFunc) *Crawler

+ func (c *Crawler) ClearPipelines() *Crawler

+ func (c *Crawler) CrawlURL(url string)

+ func (c *Crawler) OnItem(f ItemPipelineFunc) *Crawler

+ func (c *Crawler) OnItemType(itemExample interface{}, f ItemPipelineFunc) *Crawler

+ func (c *Crawler) OnRequestError(callback RequestErrorCallback) *Crawler

+ func (c *Crawler) OnResponse(callback ResponseCallback) *Crawler

+ func (c *Crawler) OnStart(callback func(ctx *Context)) *Crawler

+ func (c *Crawler) OnStop(callback func(ctx *Context)) *Crawler

+ func (c *Crawler) Start(wait bool) *Crawler

+ func (c *Crawler) Wait()

+ func (c *Crawler) WithDefaultCallback(callback func(res *Response, ctx *Context)) *Crawler

+ func (c *Crawler) WithSettings(s *Settings) *Crawler

+ func (c *Crawler) WithStartRequests(callback func(ctx *Context) []*Request) *Crawler

+ type Headers map[string]interface

+ func (h Headers) Get(name string) string

+ func (h Headers) GetList(name string) []string

+ func (h Headers) Set(name string, value string)

+ type ItemPipeline interface

+ ProcessItem func(item interface{}, ctx *Context) interface{}

+ func DefaultPipeLines() []ItemPipeline

+ func FuncPipeline(callback ItemPipelineFunc) ItemPipeline

+ type ItemPipelineFunc func(item interface{}, ctx *Context) interface

+ type Meta map[string]interface

+ type MongoPipeline struct

+ Collection string

+ Database string

+ MongoDBURI string

+ func (dmp *MongoPipeline) ProcessItem(item interface{}, ctx *Context) interface{}

+ type Request struct

+ Body []byte

+ Callback ResponseCallback

+ Cookies Cookies

+ ErrorCallback RequestErrorCallback

+ Headers Headers

+ Meta Meta

+ Method string

+ ProxyURL string

+ Timeout int

+ URL string

+ func FormRequest(url string, form Args) *Request

+ func GetRequest(url string, args Args) *Request

+ func GetURL(url string) *Request

+ func GetURLs(urls ...string) []*Request

+ func NewRequest(method string, url string, body []byte) *Request

+ func PostRequest(url string, body []byte) *Request

+ func (req *Request) OnError(callback RequestErrorCallback) *Request

+ func (req *Request) OnResponse(callback ResponseCallback) *Request

+ func (req *Request) WithContentType(contentType string) *Request

+ func (req *Request) WithCookies(cookies map[string]string) *Request

+ func (req *Request) WithHeaders(headers map[string]string) *Request

+ func (req *Request) WithMeta(meta Meta) *Request

+ func (req *Request) WithProxy(proxy string) *Request

+ func (req *Request) WithTimeout(timeout int) *Request

+ type RequestErrorCallback func(req *Request, err error, ctx *Context)

+ type Response struct

+ Body []byte

+ Cookies Cookies

+ Headers Headers

+ Meta Meta

+ NativeResponse *http.Response

+ Request *Request

+ Status string

+ StatusCode int

+ URL string

+ func NewResponse(res *http.Response) *Response

+ func (res *Response) Text() string

+ func (res *Response) WithRequest(req *Request) *Response

+ func (res *Response) WithStatus(code int, sataus string) *Response

+ type ResponseCallback func(res *Response, ctx *Context)

+ type Settings struct

+ MaxConcurrentProcessItems int

+ MaxConcurrentRequests int32

+ MaxRetryTimes int

+ RequestDelay int

+ RequestTimeout int

+ func DefaultSettings() *Settings