fetch

Documentation

Index

• func GQDocument(f Fetcher, urlStr string, opts *FetchOpts) (*goquery.Document, error)
• func MakeURLStringSlug(u string) string
• func TrimURLScheme(u string) string
• type DynamicFetcher
  • func NewDynamicFetcher(ua string, ms int) *DynamicFetcher
  • func (d *DynamicFetcher) Cancel()
  • func (d *DynamicFetcher) Fetch(urlStr string, opts *FetchOpts) (string, error)
• type FetchOpts
• type Fetcher
• type FileFetcher
  • func (s *FileFetcher) Fetch(url string, opts *FetchOpts) (string, error)
• type StaticFetcher
  • func (s *StaticFetcher) Fetch(url string, opts *FetchOpts) (string, error)

Functions

func GQDocument

func GQDocument(f Fetcher, urlStr string, opts *FetchOpts) (*goquery.Document, error)

func MakeURLStringSlug

func MakeURLStringSlug(u string) string

func TrimURLScheme

func TrimURLScheme(u string) string

Types

type DynamicFetcher

type DynamicFetcher struct {
	UserAgent        string
	WaitMilliseconds int
	// contains filtered or unexported fields
}

The DynamicFetcher renders js

func NewDynamicFetcher

func NewDynamicFetcher(ua string, ms int) *DynamicFetcher

func (*DynamicFetcher) Cancel

func (d *DynamicFetcher) Cancel()

func (*DynamicFetcher) Fetch

func (d *DynamicFetcher) Fetch(urlStr string, opts *FetchOpts) (string, error)

type FetchOpts

type FetchOpts struct {
	Interaction []*types.Interaction
}

type Fetcher

type Fetcher interface {
	Fetch(url string, opts *FetchOpts) (string, error)
}

A Fetcher allows to fetch the content of a web page

type FileFetcher

type FileFetcher struct {
}

The FileFetcher fetches static page content

func (*FileFetcher) Fetch

func (s *FileFetcher) Fetch(url string, opts *FetchOpts) (string, error)

type StaticFetcher

type StaticFetcher struct {
	UserAgent string
}

The StaticFetcher fetches static page content

func (*StaticFetcher) Fetch

func (s *StaticFetcher) Fetch(url string, opts *FetchOpts) (string, error)