downloader

package
v0.0.0-...-714e105 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 12, 2024 License: MIT Imports: 23 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func UrlEncode

func UrlEncode(urlStr string) (*url.URL, error)

Types

type DefaultDownloader

type DefaultDownloader struct {
	CookieJar *cookiejar.Jar
}

func (*DefaultDownloader) Download

func (d *DefaultDownloader) Download(request *Request) (*http.Response, error)

type DnsCache

type DnsCache struct {
	// contains filtered or unexported fields
}

DnsCache DNS cache

func (*DnsCache) Del

func (d *DnsCache) Del(addr string)

Del deletes ipPort from DNS cache.

func (*DnsCache) Query

func (d *DnsCache) Query(addr string) (string, bool)

Query queries ipPort from DNS cache.

func (*DnsCache) Reg

func (d *DnsCache) Reg(addr, ipPort string)

Reg registers ipPort to DNS cache.

type Downloader

type Downloader interface {
	Download(request *Request) (*http.Response, error)
}

func New

func New(cookie ...*cookiejar.Jar) Downloader

func NewImroc

func NewImroc(cookie *cookiejar.Jar) Downloader

func NewResty

func NewResty(cookie *cookiejar.Jar) Downloader

type ImrocDownloader

type ImrocDownloader struct {
	CookieJar *cookiejar.Jar
}

func (*ImrocDownloader) Download

func (r *ImrocDownloader) Download(request *Request) (*http.Response, error)

type Param

type Param struct {
	// contains filtered or unexported fields
}

func NewParam

func NewParam(req *Request) (param *Param, err error)

type Request

type Request struct {
	Rule string
	Url  string // url域名

	Header   http.Header
	PostData string // 参数

	// 是否使用cookies,在Spider的EnableCookie设置
	EnableCookie bool

	// dial tcp: i/o timeout
	DialTimeout time.Duration
	// WSARecv tcp: i/o timeout
	ConnTimeout time.Duration
	// the max times of download
	TryTimes int
	// how long pause when retry
	RetryPause time.Duration
	// max redirect times
	// when RedirectTimes equal 0, redirect times is ∞
	// when RedirectTimes less than 0, redirect times is 0
	RedirectTimes int

	Cloudflare bool
	JsEnable   bool
	JsFunc     func(request *Request) (*http.Response, error)
	// contains filtered or unexported fields
}

func (*Request) GetConnTimeout

func (r *Request) GetConnTimeout() time.Duration

WSARecv tcp: i/o timeout

func (*Request) GetDialTimeout

func (r *Request) GetDialTimeout() time.Duration

dial tcp: i/o timeout

func (*Request) GetMethod

func (r *Request) GetMethod() (s string)

GetMethod 获取请求方式

func (*Request) GetProxy

func (r *Request) GetProxy() (s string)

GetProxy 获取代理方式

func (*Request) GetRedirectTimes

func (r *Request) GetRedirectTimes() int

max redirect times

func (*Request) GetRetry

func (r *Request) GetRetry() (b bool, times int)

GetRetry 获取重试

func (*Request) GetRetryPause

func (r *Request) GetRetryPause() time.Duration

the pause time of retry

func (*Request) GetTryTimes

func (r *Request) GetTryTimes() int

the max times of download

func (*Request) SetMethod

func (r *Request) SetMethod(s string) *Request

SetMethod 请求方式

func (*Request) SetProxy

func (r *Request) SetProxy(s string) *Request

SetProxy 代理

func (*Request) SetRetry

func (r *Request) SetRetry(b bool, times int) *Request

SetRetry 设置重试

type RestyDownloader

type RestyDownloader struct {
	CookieJar *cookiejar.Jar
}

func (*RestyDownloader) Download

func (r *RestyDownloader) Download(request *Request) (*http.Response, error)

Directories

Path Synopsis
Package agent generates user agents strings for well known browsers and for custom browsers.
Package agent generates user agents strings for well known browsers and for custom browsers.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL