spider

Documentation

Index

• func RandomUA() string
• type DoFunc
• type HttpSpider
  • func NewHttpSpider(do DoFunc) *HttpSpider
  • func (b *HttpSpider) Do(req *http.Request) (*http.Response, error)
  • func (b *HttpSpider) Get(url string) (*http.Response, error)
  • func (b *HttpSpider) ReadBody(resp *http.Response) ([]byte, error)

Functions

func RandomUA

func RandomUA() string

RandomUA 生成随机 UA 字符串

Types

type DoFunc

type DoFunc func(req *http.Request) (*http.Response, error)

type HttpSpider

type HttpSpider struct {
	CookieMap map[string]string
	Headers   map[string]string
	HttpDo    DoFunc
}

func NewHttpSpider

func NewHttpSpider(do DoFunc) *HttpSpider

NewHttpSpider 创建一个 http 爬虫，do 为 nil 时默认使用 http.DefaultClient.Do

func (*HttpSpider) Do

func (b *HttpSpider) Do(req *http.Request) (*http.Response, error)

func (*HttpSpider) Get

func (b *HttpSpider) Get(url string) (*http.Response, error)

func (*HttpSpider) ReadBody

func (b *HttpSpider) ReadBody(resp *http.Response) ([]byte, error)