collect

Documentation

Index

• func DetermineEncoding(r *bufio.Reader) encoding.Encoding
• func HandleLinks(body []byte)
• type BaseFetch
  • func (BaseFetch) Get(req *spider.Request) ([]byte, error)
• type BrowserFetch
  • func (b BrowserFetch) Get(request *spider.Request) ([]byte, error)

Functions

func DetermineEncoding

func DetermineEncoding(r *bufio.Reader) encoding.Encoding

DetermineEncoding 检测并返回当前HTML文本的编码格式

func HandleLinks

func HandleLinks(body []byte)

Types

type BaseFetch

type BaseFetch struct{}

func (BaseFetch) Get

func (BaseFetch) Get(req *spider.Request) ([]byte, error)

type BrowserFetch

type BrowserFetch struct {
	Timeout time.Duration
	Proxy   proxy.Func
	Logger  *zap.Logger
}

func (BrowserFetch) Get

func (b BrowserFetch) Get(request *spider.Request) ([]byte, error)

Get 模拟浏览器访问