engine

Documentation

Index

• func ConvertHeadersNoLocation(h map[string][]string) []*fetch.HeaderEntry
• func IsIgnoredByKeywordMatch(req model.Request, IgnoreKeywords []string) bool
• func MergeHeaders(navHeaders map[string]interface{}, headers map[string]interface{}) []*fetch.HeaderEntry
• func RunWithTimeOut(ctx context.Context, timeout time.Duration, tasks chromedp.Tasks) chromedp.ActionFunc
• type Browser
  • func ConnectBrowser(wsUrl string, extraHeaders map[string]interface{}) *Browser
  • func InitBrowser(proxy string, noHeadless bool) *Browser
  • func (bro *Browser) Close()
  • func (bro *Browser) NewTab(timeout time.Duration) (context.Context, context.CancelFunc)
• type FillForm
  • func (f *FillForm) GetMatchInputText(name string) string
• type Tab
  • func NewTab(browser *Browser, navigateReq model.Request, config TabConfig) *Tab
  • func (tab *Tab) AddResultRequest(req model.Request)
  • func (tab *Tab) AddResultUrl(method string, _url string, source string)
  • func (tab *Tab) AfterDOMRun()
  • func (tab *Tab) AfterLoadedRun()
  • func (tab *Tab) DetectCharset()
  • func (tab *Tab) EncodeAllURLWithCharset()
  • func (tab *Tab) Evaluate(expression string)
  • func (tab *Tab) EvaluateWithNode(expression string, node *cdp.Node) error
  • func (tab *Tab) GetContentCharset(v *network.EventResponseReceived)
  • func (tab *Tab) GetExecutor() context.Context
  • func (tab *Tab) GetNodeIDs(sel string) ([]cdp.NodeID, error)
  • func (tab *Tab) GetStatusCode(headerText string) int
  • func (tab *Tab) HandleAuthRequired(req *fetch.EventAuthRequired)
  • func (tab *Tab) HandleBindingCalled(event *runtime.EventBindingCalled)
  • func (tab *Tab) HandleHostBinding(req *model.Request)
  • func (tab *Tab) HandleNavigationReq(req *model.Request, v *fetch.EventRequestPaused)
  • func (tab *Tab) HandleRedirectionResp(v *network.EventResponseReceivedExtraInfo)
  • func (tab *Tab) InterceptRequest(v *fetch.EventRequestPaused)
  • func (tab *Tab) IsNavigatorRequest(networkID string) bool
  • func (tab *Tab) IsTopFrame(FrameID string) bool
  • func (tab *Tab) ParseResponseURL(target string, v *network.EventResponseReceived)
  • func (tab *Tab) RemoveDOMListener()
  • func (tab *Tab) Start()
• type TabConfig

Functions

func ConvertHeadersNoLocation

func ConvertHeadersNoLocation(h map[string][]string) []*fetch.HeaderEntry

func IsIgnoredByKeywordMatch

func IsIgnoredByKeywordMatch(req model.Request, IgnoreKeywords []string) bool

func MergeHeaders

func MergeHeaders(navHeaders map[string]interface{}, headers map[string]interface{}) []*fetch.HeaderEntry

func RunWithTimeOut

func RunWithTimeOut(ctx context.Context, timeout time.Duration, tasks chromedp.Tasks) chromedp.ActionFunc

Types

type Browser

type Browser struct {
	Ctx    *context.Context
	Cancel *context.CancelFunc

	ExtraHeaders map[string]interface{}
	// contains filtered or unexported fields
}

func ConnectBrowser

func ConnectBrowser(wsUrl string, extraHeaders map[string]interface{}) *Browser

func InitBrowser

func InitBrowser(proxy string, noHeadless bool) *Browser

func (*Browser) Close

func (bro *Browser) Close()

func (*Browser) NewTab

func (bro *Browser) NewTab(timeout time.Duration) (context.Context, context.CancelFunc)

type FillForm

type FillForm struct {
	// contains filtered or unexported fields
}

func (*FillForm) GetMatchInputText

func (f *FillForm) GetMatchInputText(name string) string

type Tab

type Tab struct {
	Ctx              context.Context
	Cancel           context.CancelFunc
	NavigateReq      model.Request
	ExtraHeaders     map[string]interface{}
	ResultList       []*model.Request
	TopFrameId       string
	LoaderID         string
	NavNetworkID     string
	PageCharset      string
	PageBindings     map[string]interface{}
	FoundRedirection bool
	DocBodyNodeId    cdp.NodeID

	WG sync.WaitGroup // 当前Tab页的等待同步计数
	// contains filtered or unexported fields
}

func NewTab

func NewTab(browser *Browser, navigateReq model.Request, config TabConfig) *Tab

func (*Tab) AddResultRequest

func (tab *Tab) AddResultRequest(req model.Request)

AddResultRequest 添加请求到结果列表，拦截请求时处理了Host绑定，此处无需处理

func (*Tab) AddResultUrl

func (tab *Tab) AddResultUrl(method string, _url string, source string)

AddResultUrl 添加收集到的URL到结果列表，需要处理Host绑定

func (*Tab) AfterDOMRun

func (tab *Tab) AfterDOMRun()

AfterDOMRun 在DOMContentLoaded完成后执行

func (*Tab) AfterLoadedRun

func (tab *Tab) AfterLoadedRun()

AfterLoadedRun 在页面Loaded之后执行 同时等待 afterDOMRun 之后执行

func (*Tab) DetectCharset

func (tab *Tab) DetectCharset()

DetectCharset 识别页面的编码

func (*Tab) EncodeAllURLWithCharset

func (tab *Tab) EncodeAllURLWithCharset()

func (*Tab) Evaluate

func (tab *Tab) Evaluate(expression string)

Evaluate 执行JS

func (*Tab) EvaluateWithNode

func (tab *Tab) EvaluateWithNode(expression string, node *cdp.Node) error

EvaluateWithNode 根据给的Node执行JS

func (*Tab) GetContentCharset

func (tab *Tab) GetContentCharset(v *network.EventResponseReceived)

func (*Tab) GetExecutor

func (tab *Tab) GetExecutor() context.Context

GetExecutor 获取当前标签页CDP的执行上下文

func (*Tab) GetNodeIDs

func (tab *Tab) GetNodeIDs(sel string) ([]cdp.NodeID, error)

GetNodeIDs 立即根据条件获取Nodes的ID，不等待

func (*Tab) GetStatusCode

func (tab *Tab) GetStatusCode(headerText string) int

func (*Tab) HandleAuthRequired

func (tab *Tab) HandleAuthRequired(req *fetch.EventAuthRequired)

HandleAuthRequired 处理 401 407 认证弹窗

func (*Tab) HandleBindingCalled

func (tab *Tab) HandleBindingCalled(event *runtime.EventBindingCalled)

HandleBindingCalled 处理回调

func (*Tab) HandleHostBinding

func (tab *Tab) HandleHostBinding(req *model.Request)

HandleHostBinding 处理Host绑定

func (*Tab) HandleNavigationReq

func (tab *Tab) HandleNavigationReq(req *model.Request, v *fetch.EventRequestPaused)

HandleNavigationReq 处理导航请求

func (*Tab) HandleRedirectionResp

func (tab *Tab) HandleRedirectionResp(v *network.EventResponseReceivedExtraInfo)

func (*Tab) InterceptRequest

func (tab *Tab) InterceptRequest(v *fetch.EventRequestPaused)

InterceptRequest 处理每一个HTTP请求

func (*Tab) IsNavigatorRequest

func (tab *Tab) IsNavigatorRequest(networkID string) bool

IsNavigatorRequest 判断是否为导航请求

func (*Tab) IsTopFrame

func (tab *Tab) IsTopFrame(FrameID string) bool

func (*Tab) ParseResponseURL

func (tab *Tab) ParseResponseURL(target string, v *network.EventResponseReceived)

ParseResponseURL 解析响应内容中的URL 使用正则匹配

func (*Tab) RemoveDOMListener

func (tab *Tab) RemoveDOMListener()

* 移除DOM节点变化监听

func (*Tab) Start

func (tab *Tab) Start()

type TabConfig

type TabConfig struct {
	TabRunTimeout           time.Duration
	DomContentLoadedTimeout time.Duration
	EventTriggerMode        string        // 事件触发的调用方式： 异步 或 顺序
	EventTriggerInterval    time.Duration // 事件触发的间隔 单位毫秒
	BeforeExitDelay         time.Duration // 退出前的等待时间，等待DOM渲染，等待XHR发出捕获
	EncodeURLWithCharset    bool
	IgnoreKeywords          []string //
	Proxy                   string
	CustomFormValues        map[string]string
	CustomFormKeywordValues map[string]string
}