http

Documentation

Index

• Constants
• type Client
  • func NewClient() *Client
  • func (this *Client) ClearProxy()
  • func (this *Client) SetProxy(urlString string)
• type HttpServer
  • func NewHttpServer(setting *util.Settings, handler Http.Handler) *HttpServer
  • func (this *HttpServer) Start(wg sync.WaitGroup)
• type Request
  • func NewRequest(method, url, spiderName, parserName string, body io.Reader, cookieJar int) (*Request, error)
  • func (this *Request) SetNodeName(nodeName string)
• type Response
  • func NewResponse(response *Http.Response, request *Request, ...) *Response

Constants

const (
	CONTENT_TYPE      = "Content-Type"
	JSON_CONTENT_TYPE = "application/json"
)

Types

type Client

type Client struct {
	GoClient *Http.Client
}

func NewClient

func NewClient() *Client

func (*Client) ClearProxy

func (this *Client) ClearProxy()

func (*Client) SetProxy

func (this *Client) SetProxy(urlString string)

type HttpServer

type HttpServer struct {
	Http.Server
}

func NewHttpServer

func NewHttpServer(setting *util.Settings, handler Http.Handler) *HttpServer

func (*HttpServer) Start

func (this *HttpServer) Start(wg sync.WaitGroup)

type Request

type Request struct {
	GoRequest  *Http.Request
	CookieJar  int // if the web site store message in cookie which show diffrent result,you shoud use it
	UniqueName string
	SpiderName string
	ParserName string // crawler will send the response to named parseFunction
	NodeName   string
	Depth      int
	Proxy      string // proxy settings,just setting the proxy,downloader will take care of rest thing
	Retry      int    // fail and retry time
}

what a request would do * basic request * global unique key * cookie jar index * spider belong to * parse belong to

func NewRequest

func NewRequest(method, url, spiderName, parserName string, body io.Reader, cookieJar int) (*Request, error)

func (*Request) SetNodeName

func (this *Request) SetNodeName(nodeName string)

type Response

type Response struct {
	GoResponse *Http.Response
	Request    *Request
	SpiderName string
	ParserName string
	NodeName   string
	Body       string
}

func NewResponse

func NewResponse(response *Http.Response, request *Request, spiderName, parserName, nodeName string) *Response