spider

Documentation

Index

• func ReadAll(r io.Reader, size int) (buff []byte, err error)
• func UnZipHttpResp(response *http.Response, size int) ([]byte, error)
• func UnZipRetryResp(response *resty.Response, size int) ([]byte, error)
• type ApiSpider
  • func NewApiSpider(size int) *ApiSpider
  • func NewSingleApiSpider(size int, timeout time.Duration, transport *http.Transport) *ApiSpider
  • func (a *ApiSpider) Do(request Request) (Response, error)
• type Buffer
  • func (b *Buffer) Read(p []byte) (n int, err error)
  • func (b *Buffer) ReadFrom(r io.Reader, size int) (n int64, err error)
  • func (b *Buffer) Reset()
• type CloudflareSpider
  • func NewCloudflareSpider(size int) *CloudflareSpider
  • func (f *CloudflareSpider) Do(request Request) (Response, error)
• type CookieCallBack
• type DhtmlSpider
  • func NewDhtmlSpider() *DhtmlSpider
  • func (d *DhtmlSpider) Do(request Request) (Response, error)
• type FileSpider
  • func NewFileSpider(size int) *FileSpider
  • func NewSingleFileSpider(size int) *FileSpider
  • func (f *FileSpider) Do(request Request) (Response, error)
• type ProxyCallBack
• type Request
• type Response
  • func (r *Response) GetBytes() []byte
  • func (r *Response) GetCharset() string
  • func (r *Response) GetContent() string
• type ShtmlSpider
  • func NewShtmlSpider(size int) *ShtmlSpider
  • func NewSingleShtmlSpider(size int, timeout time.Duration, transport *http.Transport) *ShtmlSpider
  • func (s *ShtmlSpider) Do(request Request) (Response, error)
• type SocketProxyCallBack
• type Spider
• type SpiderType

Functions

func ReadAll

func ReadAll(r io.Reader, size int) (buff []byte, err error)

func UnZipHttpResp

func UnZipHttpResp(response *http.Response, size int) ([]byte, error)

func UnZipRetryResp

func UnZipRetryResp(response *resty.Response, size int) ([]byte, error)

Types

type ApiSpider

type ApiSpider struct {
	Jar http.CookieJar
	// contains filtered or unexported fields
}

func NewApiSpider

func NewApiSpider(size int) *ApiSpider

请求数据最大size限制

func NewSingleApiSpider

func NewSingleApiSpider(size int, timeout time.Duration, transport *http.Transport) *ApiSpider

func (*ApiSpider) Do

func (a *ApiSpider) Do(request Request) (Response, error)

type Buffer

type Buffer struct {
	// contains filtered or unexported fields
}

先看一下Buffer的定义，有帮助下面理解

func (*Buffer) Read

func (b *Buffer) Read(p []byte) (n int, err error)

func (*Buffer) ReadFrom

func (b *Buffer) ReadFrom(r io.Reader, size int) (n int64, err error)

func (*Buffer) Reset

func (b *Buffer) Reset()

type CloudflareSpider

type CloudflareSpider struct {
	Jar http.CookieJar
	// contains filtered or unexported fields
}

func NewCloudflareSpider

func NewCloudflareSpider(size int) *CloudflareSpider

请求数据最大size限制

func (*CloudflareSpider) Do

func (f *CloudflareSpider) Do(request Request) (Response, error)

type CookieCallBack

type CookieCallBack func() []*http.Cookie

type DhtmlSpider

type DhtmlSpider struct {
	// contains filtered or unexported fields
}

func NewDhtmlSpider

func NewDhtmlSpider() *DhtmlSpider

func (*DhtmlSpider) Do

func (d *DhtmlSpider) Do(request Request) (Response, error)

type FileSpider

type FileSpider struct {
	Jar http.CookieJar
	// contains filtered or unexported fields
}

func NewFileSpider

func NewFileSpider(size int) *FileSpider

请求数据最大size限制

func NewSingleFileSpider

func NewSingleFileSpider(size int) *FileSpider

func (*FileSpider) Do

func (f *FileSpider) Do(request Request) (Response, error)

type ProxyCallBack

type ProxyCallBack func() string

type Request

type Request struct {
	Url                 string
	Method              string
	Param               string
	Headers             map[string]string
	Timeout             time.Duration
	Retry               int
	ProxyCallBack       ProxyCallBack
	SocketProxyCallBack SocketProxyCallBack
	CookieCallBack      CookieCallBack
}

type Response

type Response struct {
	RedirectUrl string
	Cookies     []*http.Cookie
	Header      http.Header
	SpiderType  SpiderType
	// contains filtered or unexported fields
}

func (*Response) GetBytes

func (r *Response) GetBytes() []byte

func (*Response) GetCharset

func (r *Response) GetCharset() string

func (*Response) GetContent

func (r *Response) GetContent() string

type ShtmlSpider

type ShtmlSpider struct {
	Jar http.CookieJar
	// contains filtered or unexported fields
}

func NewShtmlSpider

func NewShtmlSpider(size int) *ShtmlSpider

请求数据最大size限制

func NewSingleShtmlSpider

func NewSingleShtmlSpider(size int, timeout time.Duration, transport *http.Transport) *ShtmlSpider

func (*ShtmlSpider) Do

func (s *ShtmlSpider) Do(request Request) (Response, error)

type SocketProxyCallBack

type SocketProxyCallBack func() (string, string, string)

type Spider

type Spider interface {
	Do(request Request) (Response, error)
}

type SpiderType

type SpiderType int

const (
	Shtml SpiderType = iota + 1
	Dhtml
	File
	Api
)