phantomjs

README

go-phantomjs-fetcher

pyspider phantomjs fetcher clone in golang.

Installation

Install PhantomJS

You can download the phantomjs executable binary here. And add it to your $PATH.

Clone the Source

go get github.com/PuerkitoBio/goquery           # used in example
go get github.com/admpub/go-phantomjs-fetcher

Example

cd $GOPATH/src/github.com/admpub/go-phantomjs-fetcher
go run ./example/mock_baidu_search.go

LICENSE

MIT

Documentation

Index

• Constants
• type Fetcher
  • func NewFetcher(port int, option *Option) (*Fetcher, error)
  • func (this *Fetcher) Get(url string) (*Response, error)
  • func (this *Fetcher) GetWithJS(url, js_script, js_run_at string) (*Response, error)
  • func (this *Fetcher) GetWithOption(url, js_script, js_run_at string, option *Option) (*Response, error)
  • func (this *Fetcher) ShutDownPhantomJSServer()
• type Option
• type Response

Constants

const (
	ErrPhantomJSNotFound = "\"phantomjs\": executable file not found in $PATH"
	ErrFetcherJSNotFound = "cannot find ./phantomjs_fetcher.js or $GOPATH/github.com/src/nladuo/go-phantomjs-fetcher/phantomjs_fetcher.js"
)

const (
	RUN_AT_DOC_START = "document-start"
	RUN_AT_DOC_END   = "document-end"
)

Types

type Fetcher

type Fetcher struct {
	ProxyPort      string
	AllowRedirects bool
	//phantomJSPid   int
	//phantomJSHandlePtr uintptr
	DefaultOption *Option
	// contains filtered or unexported fields
}

func NewFetcher

func NewFetcher(port int, option *Option) (*Fetcher, error)

func (*Fetcher) Get

func (this *Fetcher) Get(url string) (*Response, error)

send httpGet request by phantomjs

func (*Fetcher) GetWithJS

func (this *Fetcher) GetWithJS(url, js_script, js_run_at string) (*Response, error)

send httpGet request by phantomjs with the js_script

func (*Fetcher) GetWithOption

func (this *Fetcher) GetWithOption(url, js_script, js_run_at string, option *Option) (*Response, error)

send httpGet request by phantomjs with the js_script and some option like headers

func (*Fetcher) ShutDownPhantomJSServer

func (this *Fetcher) ShutDownPhantomJSServer()

shutdown the phantomjs server in windows or linux

type Option

type Option struct {
	Headers        map[string]string
	Timeout        int
	UseGzip        bool
	AllowRedirects bool
	Time           float64
	JsScriptResult string
	FetcherJsPath  string
	Method         string
	LoadImages     bool
}

type Response

type Response struct {
	OrigUrl        string            `json:"orig_url"`
	Url            string            `json:"url"`
	Headers        map[string]string `json:"headers"`
	StatusCode     int               `json:"status_code"`
	Content        string            `json:"content"`
	Cookies        []http.Cookie     `json:"cookies"`
	Time           float64           `json:"time"`
	JsScriptResult string
}