collect

Documentation

Index

• func DeterminEncoding(r *bufio.Reader) encoding.Encoding
• type BaseFetch
  • func (b *BaseFetch) Get(req *Request) ([]byte, error)
• type BrowserFetch
  • func (b *BrowserFetch) Get(request *Request) ([]byte, error)
• type Fetcher
• type ParseResult
• type Request
  • func (r *Request) Check() error

Functions

func DeterminEncoding

func DeterminEncoding(r *bufio.Reader) encoding.Encoding

Types

type BaseFetch

type BaseFetch struct {
}

func (*BaseFetch) Get

func (b *BaseFetch) Get(req *Request) ([]byte, error)

type BrowserFetch

type BrowserFetch struct {
	Timeout time.Duration
	Proxy   proxy.ProxyFunc
	Logger  *zap.Logger
}

func (*BrowserFetch) Get

func (b *BrowserFetch) Get(request *Request) ([]byte, error)

type Fetcher

type Fetcher interface {
	Get(url *Request) ([]byte, error)
}

type ParseResult

type ParseResult struct {
	Requesrts []*Request
	Items     []interface{}
}

type Request

type Request struct {
	Url       string // 访问的防战
	Cookie    string
	WaitTime  time.Duration
	Depth     int
	MaxDepth  int
	ParseFunc func([]byte, *Request) ParseResult // 解析从网站获取到的网站信息
}

func (*Request) Check

func (r *Request) Check() error