colly

Documentation

Overview

/ 爬虫系统

Index

• func OnError(f colly.ErrorCallback)
• func OnInit(config map[string]interface{}, option ...Option) (err error)
• func OnResponse(f colly.ResponseCallback)
• func OnScraped(f colly.ScrapedCallback)
• func Post(url string, requestData map[string]string) error
• func PostRaw(url string, requestData []byte) error
• func Visit(url string) error
• func Wait()
• type Colly
  • func (this *Colly) OnError(f colly.ErrorCallback)
  • func (this *Colly) OnResponse(f colly.ResponseCallback)
  • func (this *Colly) OnScraped(f colly.ScrapedCallback)
  • func (this *Colly) Post(url string, requestData map[string]string) error
  • func (this *Colly) PostRaw(url string, requestData []byte) error
  • func (this *Colly) Visit(url string) error
  • func (this *Colly) Wait()
• type ISys
  • func NewSys(option ...Option) (sys ISys, err error)
• type Option
  • func SetDelay(v time.Duration) Option
  • func SetDomainGlob(v string) Option
  • func SetParallelism(v int) Option
  • func SetProxy(v []string) Option
  • func SetUserAgent(v string) Option
• type Options

Functions

func OnError

func OnError(f colly.ErrorCallback)

func OnInit

func OnInit(config map[string]interface{}, option ...Option) (err error)

func OnResponse

func OnResponse(f colly.ResponseCallback)

func OnScraped

func OnScraped(f colly.ScrapedCallback)

func Post

func Post(url string, requestData map[string]string) error

func PostRaw

func PostRaw(url string, requestData []byte) error

func Visit

func Visit(url string) error

func Wait

func Wait()

Types

type Colly

type Colly struct {
	// contains filtered or unexported fields
}

func (*Colly) OnError

func (this *Colly) OnError(f colly.ErrorCallback)

func (*Colly) OnResponse

func (this *Colly) OnResponse(f colly.ResponseCallback)

func (*Colly) OnScraped

func (this *Colly) OnScraped(f colly.ScrapedCallback)

func (*Colly) Post

func (this *Colly) Post(url string, requestData map[string]string) error

func (*Colly) PostRaw

func (this *Colly) PostRaw(url string, requestData []byte) error

func (*Colly) Visit

func (this *Colly) Visit(url string) error

func (*Colly) Wait

func (this *Colly) Wait()

type ISys

type ISys interface {
	OnResponse(f colly.ResponseCallback)
	OnScraped(f colly.ScrapedCallback)
	OnError(f colly.ErrorCallback)
	Post(url string, requestData map[string]string) error
	PostRaw(url string, requestData []byte) error
	Wait()
	Visit(url string) error
}

func NewSys

func NewSys(option ...Option) (sys ISys, err error)

type Option

type Option func(*Options)

func SetDelay

func SetDelay(v time.Duration) Option

func SetDomainGlob

func SetDomainGlob(v string) Option

func SetParallelism

func SetParallelism(v int) Option

func SetProxy

func SetProxy(v []string) Option

func SetUserAgent

func SetUserAgent(v string) Option

type Options

type Options struct {
	DomainGlob  string
	Parallelism int
	Delay       time.Duration
	Proxy       []string
	UserAgent   string
}