crawlerx

package
v1.2.3-sp1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 26, 2023 License: AGPL-3.0 Imports: 28 Imported by: 0

Documentation

Overview

Package crawlerx @Author bcy2007 2023/7/13 11:10

Package crawlerx @Author bcy2007 2023/7/12 17:31

Package crawlerx @Author bcy2007 2023/7/12 16:20

Package crawlerx @Author bcy2007 2023/7/12 16:31

Package crawlerx @Author bcy2007 2023/7/13 11:32

Package crawlerx @Author bcy2007 2023/7/12 17:42

Package crawlerx @Author bcy2007 2023/7/12 16:56

Package crawlerx @Author bcy2007 2023/7/17 11:01

Package crawlerx @Author bcy2007 2023/7/14 10:44

Package crawlerx @Author bcy2007 2023/7/14 10:31

Package crawlerx @Author bcy2007 2023/7/14 11:07

Package crawlerx @Author bcy2007 2023/7/12 16:42

Package crawlerx @Author bcy2007 2023/7/12 16:19

Package crawlerx @Author bcy2007 2023/7/13 11:57

Package crawlerx @Author bcy2007 2023/7/12 17:40

Index

Constants

This section is empty.

Variables

View Source 
var CrawlerXExports = map[string]interface{}{
	"StartCrawler": StartCrawler,

	"browserInfo":       WithBrowserInfo,
	"maxUrl":            WithMaxUrl,
	"maxDepth":          WithMaxDepth,
	"concurrent":        WithConcurrent,
	"blacklist":         WithBlackList,
	"whitelist":         WithWhiteList,
	"pageTimeout":       WithPageTimeout,
	"fullTimeout":       WithFullTimeout,
	"extraWaitLoadTime": WithExtraWaitLoadTime,
	"formFill":          WithFormFill,
	"fileInput":         WithFileInput,
	"headers":           WithHeaders,
	"rawHeaders":        WithHeaderInfo,
	"cookies":           WithCookies,
	"rawCookie":         WithCookieInfo,
	"scanRangeLevel":    WithScanRangeLevel,
	"scanRepeatLevel":   WithScanRepeatLevel,
	"ignoreQueryName":   WithIgnoreQueryName,
	"sensitiveWords":    WithSensitiveWords,
	"leakless":          WithLeakless,

	"UnLimitRepeat":      unlimited,
	"LowRepeatLevel":     lowLevel,
	"MediumRepeatLevel":  midLevel,
	"HighRepeatLevel":    highLevel,
	"ExtremeRepeatLevel": extremeLevel,

	"AllDomainScan": mainDomain,
	"SubMenuScan":   subDomain,
}

Functions

func StartCrawler 

func StartCrawler(url string, opts ...ConfigOpt) (chan ReqInfo, error)

func StringArrayContains added in v1.2.3 

func StringArrayContains(array []string, element string) bool

func StringArrayCover added in v1.2.3 

func StringArrayCover(array []string, element string) (bool, string)

func StringPrefixList added in v1.2.3 

func StringPrefixList(origin string, prefixes []string) bool

func StringSuffixList added in v1.2.3 

func StringSuffixList(s string, suffixes []string) bool

func TargetUrlCheck added in v1.2.3 

func TargetUrlCheck(targetUrl string, proxy *url.URL) (string, error)

Types

type BaseConfig added in v1.2.3 

type BaseConfig struct {
	// contains filtered or unexported fields
}

type BrowserConfig added in v1.2.3 

type BrowserConfig struct {
	// contains filtered or unexported fields
}

type BrowserInfo added in v1.2.3 

type BrowserInfo struct {
	ExePath       string `json:"exe_path,omitempty"`
	WsAddress     string `json:"ws_address,omitempty"`
	ProxyAddress  string `json:"proxy_address,omitempty"`
	ProxyUsername string `json:"proxy_username,omitempty"`
	ProxyPassword string `json:"proxy_password,omitempty"`
}

type BrowserManager added in v1.2.3 

type BrowserManager struct {
	// contains filtered or unexported fields
}

func NewBrowserManager added in v1.2.3 

func NewBrowserManager(config *Config) *BrowserManager

func (*BrowserManager) CreateBrowserStarters added in v1.2.3 

func (manager *BrowserManager) CreateBrowserStarters()

func (*BrowserManager) Start added in v1.2.3 

func (manager *BrowserManager) Start()

type BrowserStarter added in v1.2.3 

type BrowserStarter struct {
	// contains filtered or unexported fields
}

func NewBrowserStarter added in v1.2.3 

func NewBrowserStarter(browserConfig *BrowserConfig, baseConfig *BaseConfig) *BrowserStarter

func (*BrowserStarter) GetFormFill added in v1.2.3 

func (starter *BrowserStarter) GetFormFill(element *rod.Element) string

func (*BrowserStarter) GetUploadFile added in v1.2.3 

func (starter *BrowserStarter) GetUploadFile(element *rod.Element) string

func (*BrowserStarter) HttpPostFile added in v1.2.3 

func (starter *BrowserStarter) HttpPostFile(element *rod.Element) error

func (*BrowserStarter) Start added in v1.2.3 

func (starter *BrowserStarter) Start()

type Config added in v1.2.3 

type Config struct {
	// contains filtered or unexported fields
}

func NewConfig added in v1.2.3 

func NewConfig() *Config

type ConfigOpt added in v1.2.3 

type ConfigOpt func(*Config)

func WithBlackList added in v1.2.3 

func WithBlackList(keywords ...string) ConfigOpt

func WithBrowserInfo added in v1.2.3 

func WithBrowserInfo(data string) ConfigOpt

func WithConcurrent added in v1.2.3 

func WithConcurrent(concurrent int) ConfigOpt

func WithContext added in v1.2.3 

func WithContext(ctx context.Context) ConfigOpt

func WithCookieInfo added in v1.2.3 

func WithCookieInfo(cookieInfo string) ConfigOpt

func WithCookies added in v1.2.3 

func WithCookies(cookiesInfo map[string]string) ConfigOpt

func WithExtraWaitLoadTime added in v1.2.3 

func WithExtraWaitLoadTime(extraWaitLoadTime int) ConfigOpt

func WithFileInput added in v1.2.3 

func WithFileInput(fileInput map[string]string) ConfigOpt

func WithFormFill added in v1.2.3 

func WithFormFill(formFills map[string]string) ConfigOpt

func WithFullTimeout added in v1.2.3 

func WithFullTimeout(timeout int) ConfigOpt

func WithHeaderInfo added in v1.2.3 

func WithHeaderInfo(headerInfo string) ConfigOpt

func WithHeaders added in v1.2.3 

func WithHeaders(headersInfo map[string]string) ConfigOpt

func WithIgnoreQueryName added in v1.2.3 

func WithIgnoreQueryName(names ...string) ConfigOpt

func WithLeakless added in v1.2.3 

func WithLeakless(leakless string) ConfigOpt

func WithLocalStorage added in v1.2.4 

func WithLocalStorage(storage map[string]string) ConfigOpt

func WithMaxDepth added in v1.2.3 

func WithMaxDepth(depth int) ConfigOpt

func WithMaxUrl added in v1.2.3 

func WithMaxUrl(maxUrl int) ConfigOpt

func WithPageSizedWaitGroup added in v1.2.3 

func WithPageSizedWaitGroup(pageSizedWaitGroup *utils.SizedWaitGroup) ConfigOpt

func WithPageTimeout added in v1.2.3 

func WithPageTimeout(timeout int) ConfigOpt

func WithPageVisitFilter added in v1.2.3 

func WithPageVisitFilter(pageVisitFilter *tools.StringCountFilter) ConfigOpt

func WithResultChannel added in v1.2.3 

func WithResultChannel(ch chan ReqInfo) ConfigOpt

func WithResultSentFilter added in v1.2.3 

func WithResultSentFilter(resultSentFilter *tools.StringCountFilter) ConfigOpt

func WithScanRangeLevel added in v1.2.3 

func WithScanRangeLevel(scanRange scanRangeLevel) ConfigOpt

func WithScanRepeatLevel added in v1.2.3 

func WithScanRepeatLevel(scanRepeat repeatLevel) ConfigOpt

func WithSensitiveWords added in v1.2.3 

func WithSensitiveWords(words []string) ConfigOpt

func WithStartWaitGroup added in v1.2.4 

func WithStartWaitGroup(waitGroup *utils.SizedWaitGroup) ConfigOpt

func WithTargetUrl added in v1.2.3 

func WithTargetUrl(targetUrl string) ConfigOpt

func WithUChan added in v1.2.3 

func WithUChan(uChan *tools.UChan) ConfigOpt

func WithUrlTree added in v1.2.3 

func WithUrlTree(tree *tools.UrlTree) ConfigOpt

func WithWhiteList added in v1.2.3 

func WithWhiteList(keywords ...string) ConfigOpt

type CrawlerCore added in v1.2.3 

type CrawlerCore struct {
	// contains filtered or unexported fields
}

func NewCrawlerCore added in v1.2.3 

func NewCrawlerCore(targetUrl string, opts ...ConfigOpt) (*CrawlerCore, error)

func (*CrawlerCore) Start added in v1.2.3 

func (core *CrawlerCore) Start()

type HttpRequest added in v1.2.3 

type HttpRequest struct {
	// contains filtered or unexported fields
}

func CreateFileRequest added in v1.2.3 

func CreateFileRequest(url, method string, params, files map[string]string) *HttpRequest

func CreateGetRequest added in v1.2.3 

func CreateGetRequest(url string) *HttpRequest

func CreateRequest added in v1.2.3 

func CreateRequest() *HttpRequest

func (*HttpRequest) Do added in v1.2.3 

func (request *HttpRequest) Do() error

func (*HttpRequest) GetRequest added in v1.2.3 

func (request *HttpRequest) GetRequest() error

func (*HttpRequest) GetUrl added in v1.2.3 

func (request *HttpRequest) GetUrl() string

func (*HttpRequest) MultiPartRequest added in v1.2.3 

func (request *HttpRequest) MultiPartRequest() error

func (*HttpRequest) PostRequest added in v1.2.3 

func (request *HttpRequest) PostRequest() error

func (*HttpRequest) Request added in v1.2.3 

func (request *HttpRequest) Request() error

func (*HttpRequest) Show added in v1.2.3 

func (request *HttpRequest) Show() (string, error)

type OutputBody added in v1.2.3 

type OutputBody struct {
	Size string `json:"size"`
	Md5  string `json:"md5"`
	Data string `json:"data"`
}

type OutputHeader added in v1.2.3 

type OutputHeader struct {
	Name  string `json:"name"`
	Value string `json:"value"`
}

type OutputRequest added in v1.2.3 

type OutputRequest struct {
	Url     string          `json:"url"`
	Method  string          `json:"method"`
	Headers []*OutputHeader `json:"headers"`
	Body    OutputBody      `json:"body"`
}

type OutputResponse added in v1.2.3 

type OutputResponse struct {
	StatusCode int             `json:"status_code"`
	Headers    []*OutputHeader `json:"headers"`
	Body       OutputBody      `json:"body"`
}

type OutputResult added in v1.2.3 

type OutputResult struct {
	Url      string         `json:"url"`
	Request  OutputRequest  `json:"request"`
	Response OutputResponse `json:"response"`
}

func GeneratorOutput added in v1.2.3 

func GeneratorOutput(reqInfo ReqInfo) *OutputResult

type OutputResults added in v1.2.3 

type OutputResults struct {
	// contains filtered or unexported fields
}

type ReqInfo added in v1.2.3 

type ReqInfo interface {
	Type() string

	Url() string
	Method() string

	RequestHeaders() map[string]string
	RequestBody() string

	StatusCode() int
	ResponseHeaders() map[string]string
	ResponseBody() string

	Screenshot() string

	From() string
}

type RequestResult added in v1.2.3 

type RequestResult struct {
	// contains filtered or unexported fields
}

func (*RequestResult) From added in v1.2.3 

func (result *RequestResult) From() string

func (*RequestResult) Method added in v1.2.3 

func (result *RequestResult) Method() string

func (*RequestResult) RequestBody added in v1.2.3 

func (result *RequestResult) RequestBody() string

func (*RequestResult) RequestHeaders added in v1.2.3 

func (result *RequestResult) RequestHeaders() map[string]string

func (*RequestResult) ResponseBody added in v1.2.3 

func (result *RequestResult) ResponseBody() string

func (*RequestResult) ResponseHeaders added in v1.2.3 

func (result *RequestResult) ResponseHeaders() map[string]string

func (*RequestResult) Screenshot added in v1.2.3 

func (result *RequestResult) Screenshot() string

func (*RequestResult) StatusCode added in v1.2.3 

func (result *RequestResult) StatusCode() int

func (*RequestResult) Type added in v1.2.3 

func (result *RequestResult) Type() string

func (*RequestResult) Url added in v1.2.3 

func (result *RequestResult) Url() string

type SimpleResult added in v1.2.3 

type SimpleResult struct {
	// contains filtered or unexported fields
}

func (*SimpleResult) From added in v1.2.3 

func (simpleResult *SimpleResult) From() string

func (*SimpleResult) Method added in v1.2.3 

func (simpleResult *SimpleResult) Method() string

func (*SimpleResult) RequestBody added in v1.2.3 

func (simpleResult *SimpleResult) RequestBody() string

func (*SimpleResult) RequestHeaders added in v1.2.3 

func (simpleResult *SimpleResult) RequestHeaders() map[string]string

func (*SimpleResult) ResponseBody added in v1.2.3 

func (simpleResult *SimpleResult) ResponseBody() string

func (*SimpleResult) ResponseHeaders added in v1.2.3 

func (simpleResult *SimpleResult) ResponseHeaders() map[string]string

func (*SimpleResult) Screenshot added in v1.2.3 

func (simpleResult *SimpleResult) Screenshot() string

func (*SimpleResult) StatusCode added in v1.2.3 

func (*SimpleResult) StatusCode() int

func (*SimpleResult) Type added in v1.2.3 

func (simpleResult *SimpleResult) Type() string

func (*SimpleResult) Url added in v1.2.3 

func (simpleResult *SimpleResult) Url() string

Source Files

View all Source files

Directories

Path Synopsis
Package cmd @Author bcy2007 2023/7/14 11:11
 Package cmd @Author bcy2007 2023/7/14 11:11
Package tools @Author bcy2007 2023/7/12 16:40
 Package tools @Author bcy2007 2023/7/12 16:40
config
Package config https://github.com/unknwon/goconfig
 Package config https://github.com/unknwon/goconfig

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.