core

package
v1.3.3-beta10 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 22, 2024 License: AGPL-3.0 Imports: 33 Imported by: 0

Documentation

Index

Constants

View Source 
const GETREADYSTATE = `
()=>{
	return document.readyState;
}
`
View Source 
const GETURL = `
()=>{
	return document.URL;
}
`

Variables

View Source 
var DefaultKeyword = getDefault
View Source 
var StrictKeyword = getStrict

Functions

func CalcSha1 

func CalcSha1(items ...interface{}) string

func CleanupPage 

func CleanupPage(page *rod.Page)

func GetKeywordType 

func GetKeywordType(kw string) string

from name & placehold to know input information type like username, password etc.

Types

type Config 

type Config struct {
	// contains filtered or unexported fields
}

type ConfigOpt 

type ConfigOpt func(c *Config)

func WithBlackDomain 

func WithBlackDomain(matchStr string) ConfigOpt

func WithBrowserProxy 

func WithBrowserProxy(url string, userinfo ...string) ConfigOpt

func WithCaptchaUrl 

func WithCaptchaUrl(captchaUrl string) ConfigOpt

func WithCookie 

func WithCookie(domain, k, v string) ConfigOpt

func WithEvalJs 

func WithEvalJs(url, js string) ConfigOpt

func WithFrom 

func WithFrom(k, v string) ConfigOpt

func WithHeader 

func WithHeader(s string) ConfigOpt

func WithOnRequest 

func WithOnRequest(f func(req *Req)) ConfigOpt

func WithSpiderDepth 

func WithSpiderDepth(depth int) ConfigOpt

func WithStrictUrlDetect 

func WithStrictUrlDetect(status bool) ConfigOpt

func WithTimeout 

func WithTimeout(timeout int) ConfigOpt

func WithUrlCount 

func WithUrlCount(count int) ConfigOpt

func WithWhiteDomain 

func WithWhiteDomain(matchStr string) ConfigOpt

type MakeReq 

type MakeReq struct {
	// contains filtered or unexported fields
}

func (*MakeReq) Request 

func (r *MakeReq) Request() *http.Request

func (*MakeReq) RequestHeader 

func (r *MakeReq) RequestHeader() map[string][]string

func (*MakeReq) Response 

func (r *MakeReq) Response() (*http.Response, error)

func (*MakeReq) ResponseBody 

func (r *MakeReq) ResponseBody() []byte

func (*MakeReq) ResponseHeader 

func (r *MakeReq) ResponseHeader() map[string][]string

func (*MakeReq) Url 

func (r *MakeReq) Url() string

type Manager 

type Manager struct {
	BrowserPool rod.BrowserPool
	Browser     *rod.Browser
	PagePool    rod.PagePool

	StartUrls []string
	// contains filtered or unexported fields
}

func NewManager 

func NewManager(urls string, ch chan RequestIf, opts ...ConfigOpt) (*Manager, error)

func (*Manager) CheckSensitive 

func (m *Manager) CheckSensitive(element *rod.Element) bool

func (*Manager) ElementInputWords 

func (m *Manager) ElementInputWords(element *rod.Element, words string) error

input by keyboard

func (*Manager) GetBrowser 

func (m *Manager) GetBrowser() *rod.Browser

func (*Manager) GetCurrentUrl 

func (m *Manager) GetCurrentUrl(page *rod.Page) (url string, err error)

get current page url by js

func (*Manager) GetPage 

func (m *Manager) GetPage(opts proto.TargetCreateTarget, depth int) (*PageBlock, error)

func (*Manager) GetReadyState 

func (m *Manager) GetReadyState(page *rod.Page) (state string, err error)

get page state by js

func (*Manager) PageBack 

func (m *Manager) PageBack(page *rod.Page) error

func (*Manager) PredictX 

func (m *Manager) PredictX(s string) string

use key words directly to detect whether url is sensitive a complement of random forest used by detect sensitive url

func (*Manager) PutBrowser 

func (m *Manager) PutBrowser(b *rod.Browser)

func (*Manager) PutPage 

func (m *Manager) PutPage(p *rod.Page)

func (*Manager) Release added in v1.3.2 

func (m *Manager) Release(browser *rod.Browser)

func (*Manager) RemoveParamValue 

func (m *Manager) RemoveParamValue(urlStr string) string

func (*Manager) Run 

func (m *Manager) Run() error

func (*Manager) RunContext 

func (m *Manager) RunContext(ctx context.Context) error

func (*Manager) WaitLoad 

func (m *Manager) WaitLoad(page *rod.Page)

type PageBlock 

type PageBlock struct {
	// contains filtered or unexported fields
}

func (PageBlock) GoBack 

func (p PageBlock) GoBack()

func (PageBlock) GoDeeper 

func (p PageBlock) GoDeeper()

type Req 

type Req struct {
	// contains filtered or unexported fields
}

func (*Req) IsHttps 

func (r *Req) IsHttps() bool

func (*Req) Request 

func (r *Req) Request() *http.Request

func (*Req) RequestHeader 

func (r *Req) RequestHeader() map[string][]string

func (*Req) RequestRaw 

func (r *Req) RequestRaw() []byte

func (*Req) Response 

func (r *Req) Response() (*http.Response, error)

func (*Req) ResponseBody 

func (r *Req) ResponseBody() []byte

func (*Req) ResponseHeader 

func (r *Req) ResponseHeader() map[string][]string

func (*Req) Url 

func (r *Req) Url() string

type RequestIf 

type RequestIf interface {
	Url() string
	Request() *http.Request
	ResponseBody() []byte
	Response() (*http.Response, error)
	RequestHeader() map[string][]string
	ResponseHeader() map[string][]string
}

type StringFilterwithCount 

type StringFilterwithCount struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewFilterwithCount 

func NewFilterwithCount() *StringFilterwithCount

func NewStringFilterwithCount 

func NewStringFilterwithCount(config *filter.Config, container *cuckoo.Filter) *StringFilterwithCount

func (*StringFilterwithCount) Count 

func (s *StringFilterwithCount) Count() int64

func (*StringFilterwithCount) Exist 

func (s *StringFilterwithCount) Exist(str string) bool

func (*StringFilterwithCount) Insert 

func (s *StringFilterwithCount) Insert(str string) bool

返回值是 true，插入成功，false 容器满了, 无法继续添加

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.