fetcher

Documentation

Index

• func NewClient(proxy *url.URL) (client http.Client)
• type ForgedFetcher
  • func (f *ForgedFetcher) Fetch(weburl string) (html []byte, error error)
  • func (f *ForgedFetcher) GetProxy() (string, error)
  • func (f *ForgedFetcher) Work(request engine.Request) (engine.ParseResult, error)
• type SimpleFetch
  • func (s SimpleFetch) Fetch(url string) (html []byte, error error)
  • func (s SimpleFetch) GetProxy() (string, error)
  • func (s SimpleFetch) Work(request engine.Request) (engine.ParseResult, error)

Functions

func NewClient

func NewClient(proxy *url.URL) (client http.Client)

NewClient

Types

type ForgedFetcher

type ForgedFetcher struct {
}

func (*ForgedFetcher) Fetch

func (f *ForgedFetcher) Fetch(weburl string) (html []byte, error error)

Fetch

func (*ForgedFetcher) GetProxy

func (f *ForgedFetcher) GetProxy() (string, error)

GetProxy

func (*ForgedFetcher) Work

func (f *ForgedFetcher) Work(request engine.Request) (engine.ParseResult, error)

Work

type SimpleFetch

type SimpleFetch struct {
	// contains filtered or unexported fields
}

func (SimpleFetch) Fetch

func (s SimpleFetch) Fetch(url string) (html []byte, error error)

Fetcher

func (SimpleFetch) GetProxy

func (s SimpleFetch) GetProxy() (string, error)

func (SimpleFetch) Work

func (s SimpleFetch) Work(request engine.Request) (engine.ParseResult, error)

Work