surfer

Documentation

Index

• Constants
• func AutoToUTF8(resp *http.Response) error
• func BodyBytes(resp *http.Response) ([]byte, error)
• func DestroyJsFiles()
• func Download(req Request) (resp *http.Response, err error)
• func GetWDPath() string
• func IsDirExists(path string) bool
• func IsFileExists(path string) bool
• func UrlEncode(urlStr string) (*url.URL, error)
• func WalkDir(targpath string, suffixes ...string) (dirlist []string)
• type Body
  • func (b *Body) Read(p []byte) (int, error)
• type Cookie
• type DefaultRequest
  • func (self *DefaultRequest) GetConnTimeout() time.Duration
  • func (self *DefaultRequest) GetDialTimeout() time.Duration
  • func (self *DefaultRequest) GetDownloaderID() int
  • func (self *DefaultRequest) GetEnableCookie() bool
  • func (self *DefaultRequest) GetHeader() http.Header
  • func (self *DefaultRequest) GetMethod() string
  • func (self *DefaultRequest) GetPostData() string
  • func (self *DefaultRequest) GetProxy() string
  • func (self *DefaultRequest) GetRedirectTimes() int
  • func (self *DefaultRequest) GetRetryPause() time.Duration
  • func (self *DefaultRequest) GetTryTimes() int
  • func (self *DefaultRequest) GetUrl() string
• type DnsCache
  • func (d *DnsCache) Del(addr string)
  • func (d *DnsCache) Query(addr string) (string, bool)
  • func (d *DnsCache) Reg(addr, ipPort string)
• type Param
  • func NewParam(req Request) (param *Param, err error)
• type Phantom
  • func (self *Phantom) DestroyJsFiles()
  • func (self *Phantom) Download(req Request) (resp *http.Response, err error)
• type Request
• type Response
• type Surf
  • func (self *Surf) Download(req Request) (resp *http.Response, err error)
• type Surfer
  • func New(jar ...*cookiejar.Jar) Surfer
  • func NewPhantom(phantomjsFile, tempJsDir string, jar ...*cookiejar.Jar) Surfer

Constants

const (
	SurfID             = 0
	PhomtomJsID        = 1
	DefaultMethod      = "GET"
	DefaultDialTimeout = 2 * time.Minute
	DefaultConnTimeout = 2 * time.Minute
	DefaultTryTimes    = 3
	DefaultRetryPause  = 2 * time.Second
)

Functions

func AutoToUTF8

func AutoToUTF8(resp *http.Response) error

func BodyBytes

func BodyBytes(resp *http.Response) ([]byte, error)

func DestroyJsFiles

func DestroyJsFiles()

func Download

func Download(req Request) (resp *http.Response, err error)

func GetWDPath

func GetWDPath() string

func IsDirExists

func IsDirExists(path string) bool

func IsFileExists

func IsFileExists(path string) bool

func UrlEncode

func UrlEncode(urlStr string) (*url.URL, error)

func WalkDir

func WalkDir(targpath string, suffixes ...string) (dirlist []string)

Types

type Body

type Body struct {
	io.ReadCloser
	io.Reader
}

func (*Body) Read

func (b *Body) Read(p []byte) (int, error)

type Cookie

type Cookie struct {
	Name   string `json:"name"`
	Value  string `json:"value"`
	Domain string `json:"domain"`
	Path   string `json:"path"`
}

type DefaultRequest

type DefaultRequest struct {
	Url string

	Method string

	Header http.Header

	EnableCookie bool

	PostData string

	DialTimeout time.Duration

	ConnTimeout time.Duration

	TryTimes int

	RetryPause time.Duration

	RedirectTimes int

	Proxy string

	DownloaderID int
	// contains filtered or unexported fields
}

func (*DefaultRequest) GetConnTimeout

func (self *DefaultRequest) GetConnTimeout() time.Duration

func (*DefaultRequest) GetDialTimeout

func (self *DefaultRequest) GetDialTimeout() time.Duration

func (*DefaultRequest) GetDownloaderID

func (self *DefaultRequest) GetDownloaderID() int

func (*DefaultRequest) GetEnableCookie

func (self *DefaultRequest) GetEnableCookie() bool

func (*DefaultRequest) GetHeader

func (self *DefaultRequest) GetHeader() http.Header

func (*DefaultRequest) GetMethod

func (self *DefaultRequest) GetMethod() string

func (*DefaultRequest) GetPostData

func (self *DefaultRequest) GetPostData() string

func (*DefaultRequest) GetProxy

func (self *DefaultRequest) GetProxy() string

func (*DefaultRequest) GetRedirectTimes

func (self *DefaultRequest) GetRedirectTimes() int

func (*DefaultRequest) GetRetryPause

func (self *DefaultRequest) GetRetryPause() time.Duration

func (*DefaultRequest) GetTryTimes

func (self *DefaultRequest) GetTryTimes() int

func (*DefaultRequest) GetUrl

func (self *DefaultRequest) GetUrl() string

type DnsCache

type DnsCache struct {
	// contains filtered or unexported fields
}

func (*DnsCache) Del

func (d *DnsCache) Del(addr string)

func (*DnsCache) Query

func (d *DnsCache) Query(addr string) (string, bool)

func (*DnsCache) Reg

func (d *DnsCache) Reg(addr, ipPort string)

type Param

type Param struct {
	// contains filtered or unexported fields
}

func NewParam

func NewParam(req Request) (param *Param, err error)

type Phantom

type Phantom struct {
	PhantomjsFile string
	TempJsDir     string

	CookieJar *cookiejar.Jar
	// contains filtered or unexported fields
}

func (*Phantom) DestroyJsFiles

func (self *Phantom) DestroyJsFiles()

func (*Phantom) Download

func (self *Phantom) Download(req Request) (resp *http.Response, err error)

type Request

type Request interface {
	GetUrl() string

	GetMethod() string

	GetPostData() string

	GetHeader() http.Header

	GetEnableCookie() bool

	GetDialTimeout() time.Duration

	GetConnTimeout() time.Duration

	GetTryTimes() int

	GetRetryPause() time.Duration

	GetProxy() string

	GetRedirectTimes() int

	GetDownloaderID() int
}

type Response

type Response struct {
	Cookies []string
	Body    string
	Error   string
	Header  []struct {
		Name  string
		Value string
	}
}

type Surf

type Surf struct {
	CookieJar *cookiejar.Jar
}

func (*Surf) Download

func (self *Surf) Download(req Request) (resp *http.Response, err error)

type Surfer

type Surfer interface {
	Download(Request) (resp *http.Response, err error)
}

func New

func New(jar ...*cookiejar.Jar) Surfer

func NewPhantom

func NewPhantom(phantomjsFile, tempJsDir string, jar ...*cookiejar.Jar) Surfer