model

Documentation

Index

• Constants
• type Page
• type Request
• type ResultItems
  • func (result *ResultItems) AddTargetRequests(requests []string, referer string) error
  • func (result *ResultItems) AddTargetRequestsPriority(requests []string, priority int32, referer string) error
  • func (result *ResultItems) AddTargetUrls(urls ...string)
  • func (result *ResultItems) AppendTargetRequest(request Request)
  • func (result *ResultItems) Get(key string) interface{}
  • func (result *ResultItems) InValidUrl(url string) bool
  • func (result *ResultItems) Put(key string, val interface{})
  • func (result *ResultItems) SetRequest(request Request)
  • func (result *ResultItems) SetSkip(skip bool)
• type Site
  • func CreateNewSite() *Site
  • func (site *Site) AddCookie(domain string, name string, val string) *Site
  • func (site *Site) AddDefaultCookie(name string, val string) *Site
  • func (site *Site) AddHeader(key string, val string) *Site

Constants

const (
	Priority_Level_1 = 5000
	Priority_Level_2 = 4000
	Priority_Level_3 = 3000
	Priority_Level_4 = 0
)

优先级

Types

type Page

type Page struct {
	UrlStr          string
	DownloadSuccess bool   //是否下载成功
	RawText         string //页面内容
	RawTextBytes    []byte //页面内容字节
	Request         Request
	Headers         map[string][]string
}

Page 页面信息

type Request

type Request struct {
	Method   string
	UrlStr   string
	Body     io.ReadCloser
	Priority int32
}

type ResultItems

type ResultItems struct {
	Skip           bool
	Request        Request
	Items          map[string]interface{}
	TargetRequests []Request //目标请求列表
}

ResultItems 页面处理后的内容

func (*ResultItems) AddTargetRequests

func (result *ResultItems) AddTargetRequests(requests []string, referer string) error

func (*ResultItems) AddTargetRequestsPriority

func (result *ResultItems) AddTargetRequestsPriority(requests []string, priority int32, referer string) error

func (*ResultItems) AddTargetUrls

func (result *ResultItems) AddTargetUrls(urls ...string)

func (*ResultItems) AppendTargetRequest

func (result *ResultItems) AppendTargetRequest(request Request)

func (*ResultItems) Get

func (result *ResultItems) Get(key string) interface{}

func (*ResultItems) InValidUrl

func (result *ResultItems) InValidUrl(url string) bool

func (*ResultItems) Put

func (result *ResultItems) Put(key string, val interface{})

func (*ResultItems) SetRequest

func (result *ResultItems) SetRequest(request Request)

func (*ResultItems) SetSkip

func (result *ResultItems) SetSkip(skip bool)

type Site

type Site struct {
	Domain          string //域名
	UserAgent       string //
	DefaultCookies  map[string]string
	Cookies         map[string]map[string]string
	Charset         string
	SleepTime       time.Duration
	RetryTimes      time.Duration
	RetrySleepTime  time.Duration
	CycleRetryTimes int32
	TimeOut         time.Duration
	Headers         map[string]string
}

Site 站点信息

func CreateNewSite

func CreateNewSite() *Site

CreateNewSite new Site

func (*Site) AddCookie

func (site *Site) AddCookie(domain string, name string, val string) *Site

func (*Site) AddDefaultCookie

func (site *Site) AddDefaultCookie(name string, val string) *Site

func (*Site) AddHeader

func (site *Site) AddHeader(key string, val string) *Site