Documentation ¶
Overview ¶
Package crawlerx @Author bcy2007 2023/7/13 11:10
Package crawlerx @Author bcy2007 2023/7/12 17:31
Package crawlerx @Author bcy2007 2023/7/12 16:20
Package crawlerx @Author bcy2007 2023/7/12 16:31
Package crawlerx @Author bcy2007 2023/7/13 11:32
Package crawlerx @Author bcy2007 2023/7/12 17:42
Package crawlerx @Author bcy2007 2023/7/12 16:56
Package crawlerx @Author bcy2007 2023/7/17 11:01
Package crawlerx @Author bcy2007 2023/7/14 10:44
Package crawlerx @Author bcy2007 2023/7/14 10:31
Package crawlerx @Author bcy2007 2023/7/14 11:07
Package crawlerx @Author bcy2007 2023/7/12 16:42
Package crawlerx @Author bcy2007 2023/7/12 16:19
Package crawlerx @Author bcy2007 2023/7/13 11:57
Package crawlerx @Author bcy2007 2023/7/12 17:40
Index ¶
- Variables
- func StartCrawler(url string, opts ...ConfigOpt) (chan ReqInfo, error)
- func StringArrayContains(array []string, element string) bool
- func StringArrayCover(array []string, element string) (bool, string)
- func StringPrefixList(origin string, prefixes []string) bool
- func StringSuffixList(s string, suffixes []string) bool
- func TargetUrlCheck(targetUrl string, proxy *url.URL) (string, error)
- type BaseConfig
- type BrowserConfig
- type BrowserInfo
- type BrowserManager
- type BrowserStarter
- type Config
- type ConfigOpt
- func WithBlackList(keywords ...string) ConfigOpt
- func WithBrowserInfo(data string) ConfigOpt
- func WithConcurrent(concurrent int) ConfigOpt
- func WithContext(ctx context.Context) ConfigOpt
- func WithCookieInfo(cookieInfo string) ConfigOpt
- func WithCookies(cookiesInfo map[string]string) ConfigOpt
- func WithExtraWaitLoadTime(extraWaitLoadTime int) ConfigOpt
- func WithFileInput(fileInput map[string]string) ConfigOpt
- func WithFormFill(formFills map[string]string) ConfigOpt
- func WithFullTimeout(timeout int) ConfigOpt
- func WithHeaderInfo(headerInfo string) ConfigOpt
- func WithHeaders(headersInfo map[string]string) ConfigOpt
- func WithIgnoreQueryName(names ...string) ConfigOpt
- func WithInvalidSuffix(suffix []string) ConfigOpt
- func WithLeakless(leakless string) ConfigOpt
- func WithLocalStorage(storage map[string]string) ConfigOpt
- func WithMaxDepth(depth int) ConfigOpt
- func WithMaxUrl(maxUrl int) ConfigOpt
- func WithPageSizedWaitGroup(pageSizedWaitGroup *utils.SizedWaitGroup) ConfigOpt
- func WithPageTimeout(timeout int) ConfigOpt
- func WithPageVisitFilter(pageVisitFilter *tools.StringCountFilter) ConfigOpt
- func WithResultChannel(ch chan ReqInfo) ConfigOpt
- func WithResultSentFilter(resultSentFilter *tools.StringCountFilter) ConfigOpt
- func WithScanRangeLevel(scanRange scanRangeLevel) ConfigOpt
- func WithScanRepeatLevel(scanRepeat repeatLevel) ConfigOpt
- func WithSensitiveWords(words []string) ConfigOpt
- func WithStartWaitGroup(waitGroup *utils.SizedWaitGroup) ConfigOpt
- func WithTargetUrl(targetUrl string) ConfigOpt
- func WithUChan(uChan *tools.UChan) ConfigOpt
- func WithUrlTree(tree *tools.UrlTree) ConfigOpt
- func WithWhiteList(keywords ...string) ConfigOpt
- type CrawlerCore
- type HttpRequest
- func (request *HttpRequest) Do() error
- func (request *HttpRequest) GetRequest() error
- func (request *HttpRequest) GetUrl() string
- func (request *HttpRequest) MultiPartRequest() error
- func (request *HttpRequest) PostRequest() error
- func (request *HttpRequest) Request() error
- func (request *HttpRequest) Show() (string, error)
- type OutputBody
- type OutputHeader
- type OutputRequest
- type OutputResponse
- type OutputResult
- type OutputResults
- type ReqInfo
- type RequestResult
- func (result *RequestResult) From() string
- func (result *RequestResult) Method() string
- func (result *RequestResult) RequestBody() string
- func (result *RequestResult) RequestHeaders() map[string]string
- func (result *RequestResult) ResponseBody() string
- func (result *RequestResult) ResponseHeaders() map[string]string
- func (result *RequestResult) Screenshot() string
- func (result *RequestResult) StatusCode() int
- func (result *RequestResult) Type() string
- func (result *RequestResult) Url() string
- type SimpleResult
- func (simpleResult *SimpleResult) From() string
- func (simpleResult *SimpleResult) Method() string
- func (simpleResult *SimpleResult) RequestBody() string
- func (simpleResult *SimpleResult) RequestHeaders() map[string]string
- func (simpleResult *SimpleResult) ResponseBody() string
- func (simpleResult *SimpleResult) ResponseHeaders() map[string]string
- func (simpleResult *SimpleResult) Screenshot() string
- func (*SimpleResult) StatusCode() int
- func (simpleResult *SimpleResult) Type() string
- func (simpleResult *SimpleResult) Url() string
Constants ¶
This section is empty.
Variables ¶
var CrawlerXExports = map[string]interface{}{ "StartCrawler": StartCrawler, "browserInfo": WithBrowserInfo, "maxUrl": WithMaxUrl, "maxDepth": WithMaxDepth, "concurrent": WithConcurrent, "blacklist": WithBlackList, "whitelist": WithWhiteList, "pageTimeout": WithPageTimeout, "fullTimeout": WithFullTimeout, "extraWaitLoadTime": WithExtraWaitLoadTime, "formFill": WithFormFill, "fileInput": WithFileInput, "headers": WithHeaders, "rawHeaders": WithHeaderInfo, "cookies": WithCookies, "rawCookie": WithCookieInfo, "scanRangeLevel": WithScanRangeLevel, "scanRepeatLevel": WithScanRepeatLevel, "ignoreQueryName": WithIgnoreQueryName, "sensitiveWords": WithSensitiveWords, "leakless": WithLeakless, "localStorage": WithLocalStorage, "invalidSuffix": WithInvalidSuffix, "UnLimitRepeat": unlimited, "LowRepeatLevel": lowLevel, "MediumRepeatLevel": midLevel, "HighRepeatLevel": highLevel, "ExtremeRepeatLevel": extremeLevel, "AllDomainScan": mainDomain, "SubMenuScan": subDomain, }
Functions ¶
func StringArrayContains ¶ added in v1.2.3
func StringArrayCover ¶ added in v1.2.3
func StringPrefixList ¶ added in v1.2.3
func StringSuffixList ¶ added in v1.2.3
Types ¶
type BaseConfig ¶ added in v1.2.3
type BaseConfig struct {
// contains filtered or unexported fields
}
type BrowserConfig ¶ added in v1.2.3
type BrowserConfig struct {
// contains filtered or unexported fields
}
type BrowserInfo ¶ added in v1.2.3
type BrowserManager ¶ added in v1.2.3
type BrowserManager struct {
// contains filtered or unexported fields
}
func NewBrowserManager ¶ added in v1.2.3
func NewBrowserManager(config *Config) *BrowserManager
func (*BrowserManager) CreateBrowserStarters ¶ added in v1.2.3
func (manager *BrowserManager) CreateBrowserStarters()
func (*BrowserManager) Start ¶ added in v1.2.3
func (manager *BrowserManager) Start()
type BrowserStarter ¶ added in v1.2.3
type BrowserStarter struct {
// contains filtered or unexported fields
}
func NewBrowserStarter ¶ added in v1.2.3
func NewBrowserStarter(browserConfig *BrowserConfig, baseConfig *BaseConfig) *BrowserStarter
func (*BrowserStarter) GetFormFill ¶ added in v1.2.3
func (starter *BrowserStarter) GetFormFill(element *rod.Element) string
func (*BrowserStarter) GetUploadFile ¶ added in v1.2.3
func (starter *BrowserStarter) GetUploadFile(element *rod.Element) string
func (*BrowserStarter) HttpPostFile ¶ added in v1.2.3
func (starter *BrowserStarter) HttpPostFile(element *rod.Element) error
func (*BrowserStarter) Start ¶ added in v1.2.3
func (starter *BrowserStarter) Start()
type ConfigOpt ¶ added in v1.2.3
type ConfigOpt func(*Config)
func WithBlackList ¶ added in v1.2.3
func WithBrowserInfo ¶ added in v1.2.3
func WithConcurrent ¶ added in v1.2.3
func WithContext ¶ added in v1.2.3
func WithCookieInfo ¶ added in v1.2.3
func WithCookies ¶ added in v1.2.3
func WithExtraWaitLoadTime ¶ added in v1.2.3
func WithFileInput ¶ added in v1.2.3
func WithFormFill ¶ added in v1.2.3
func WithFullTimeout ¶ added in v1.2.3
func WithHeaderInfo ¶ added in v1.2.3
func WithHeaders ¶ added in v1.2.3
func WithIgnoreQueryName ¶ added in v1.2.3
func WithInvalidSuffix ¶ added in v1.2.4
func WithLeakless ¶ added in v1.2.3
func WithLocalStorage ¶ added in v1.2.4
func WithMaxDepth ¶ added in v1.2.3
func WithMaxUrl ¶ added in v1.2.3
func WithPageSizedWaitGroup ¶ added in v1.2.3
func WithPageSizedWaitGroup(pageSizedWaitGroup *utils.SizedWaitGroup) ConfigOpt
func WithPageTimeout ¶ added in v1.2.3
func WithPageVisitFilter ¶ added in v1.2.3
func WithPageVisitFilter(pageVisitFilter *tools.StringCountFilter) ConfigOpt
func WithResultChannel ¶ added in v1.2.3
func WithResultSentFilter ¶ added in v1.2.3
func WithResultSentFilter(resultSentFilter *tools.StringCountFilter) ConfigOpt
func WithScanRangeLevel ¶ added in v1.2.3
func WithScanRangeLevel(scanRange scanRangeLevel) ConfigOpt
func WithScanRepeatLevel ¶ added in v1.2.3
func WithScanRepeatLevel(scanRepeat repeatLevel) ConfigOpt
func WithSensitiveWords ¶ added in v1.2.3
func WithStartWaitGroup ¶ added in v1.2.4
func WithStartWaitGroup(waitGroup *utils.SizedWaitGroup) ConfigOpt
func WithTargetUrl ¶ added in v1.2.3
func WithUrlTree ¶ added in v1.2.3
func WithWhiteList ¶ added in v1.2.3
type CrawlerCore ¶ added in v1.2.3
type CrawlerCore struct {
// contains filtered or unexported fields
}
func NewCrawlerCore ¶ added in v1.2.3
func NewCrawlerCore(targetUrl string, opts ...ConfigOpt) (*CrawlerCore, error)
func (*CrawlerCore) Start ¶ added in v1.2.3
func (core *CrawlerCore) Start()
type HttpRequest ¶ added in v1.2.3
type HttpRequest struct {
// contains filtered or unexported fields
}
func CreateFileRequest ¶ added in v1.2.3
func CreateFileRequest(url, method string, params, files map[string]string) *HttpRequest
func CreateGetRequest ¶ added in v1.2.3
func CreateGetRequest(url string) *HttpRequest
func CreateRequest ¶ added in v1.2.3
func CreateRequest() *HttpRequest
func (*HttpRequest) Do ¶ added in v1.2.3
func (request *HttpRequest) Do() error
func (*HttpRequest) GetRequest ¶ added in v1.2.3
func (request *HttpRequest) GetRequest() error
func (*HttpRequest) GetUrl ¶ added in v1.2.3
func (request *HttpRequest) GetUrl() string
func (*HttpRequest) MultiPartRequest ¶ added in v1.2.3
func (request *HttpRequest) MultiPartRequest() error
func (*HttpRequest) PostRequest ¶ added in v1.2.3
func (request *HttpRequest) PostRequest() error
func (*HttpRequest) Request ¶ added in v1.2.3
func (request *HttpRequest) Request() error
func (*HttpRequest) Show ¶ added in v1.2.3
func (request *HttpRequest) Show() (string, error)
type OutputBody ¶ added in v1.2.3
type OutputHeader ¶ added in v1.2.3
type OutputRequest ¶ added in v1.2.3
type OutputRequest struct { Url string `json:"url"` Method string `json:"method"` Headers []*OutputHeader `json:"headers"` Body OutputBody `json:"body"` }
type OutputResponse ¶ added in v1.2.3
type OutputResponse struct { StatusCode int `json:"status_code"` Headers []*OutputHeader `json:"headers"` Body OutputBody `json:"body"` }
type OutputResult ¶ added in v1.2.3
type OutputResult struct { Url string `json:"url"` Request OutputRequest `json:"request"` Response OutputResponse `json:"response"` }
func GeneratorOutput ¶ added in v1.2.3
func GeneratorOutput(reqInfo ReqInfo) *OutputResult
type OutputResults ¶ added in v1.2.3
type OutputResults struct {
// contains filtered or unexported fields
}
type RequestResult ¶ added in v1.2.3
type RequestResult struct {
// contains filtered or unexported fields
}
func (*RequestResult) From ¶ added in v1.2.3
func (result *RequestResult) From() string
func (*RequestResult) Method ¶ added in v1.2.3
func (result *RequestResult) Method() string
func (*RequestResult) RequestBody ¶ added in v1.2.3
func (result *RequestResult) RequestBody() string
func (*RequestResult) RequestHeaders ¶ added in v1.2.3
func (result *RequestResult) RequestHeaders() map[string]string
func (*RequestResult) ResponseBody ¶ added in v1.2.3
func (result *RequestResult) ResponseBody() string
func (*RequestResult) ResponseHeaders ¶ added in v1.2.3
func (result *RequestResult) ResponseHeaders() map[string]string
func (*RequestResult) Screenshot ¶ added in v1.2.3
func (result *RequestResult) Screenshot() string
func (*RequestResult) StatusCode ¶ added in v1.2.3
func (result *RequestResult) StatusCode() int
func (*RequestResult) Type ¶ added in v1.2.3
func (result *RequestResult) Type() string
func (*RequestResult) Url ¶ added in v1.2.3
func (result *RequestResult) Url() string
type SimpleResult ¶ added in v1.2.3
type SimpleResult struct {
// contains filtered or unexported fields
}
func (*SimpleResult) From ¶ added in v1.2.3
func (simpleResult *SimpleResult) From() string
func (*SimpleResult) Method ¶ added in v1.2.3
func (simpleResult *SimpleResult) Method() string
func (*SimpleResult) RequestBody ¶ added in v1.2.3
func (simpleResult *SimpleResult) RequestBody() string
func (*SimpleResult) RequestHeaders ¶ added in v1.2.3
func (simpleResult *SimpleResult) RequestHeaders() map[string]string
func (*SimpleResult) ResponseBody ¶ added in v1.2.3
func (simpleResult *SimpleResult) ResponseBody() string
func (*SimpleResult) ResponseHeaders ¶ added in v1.2.3
func (simpleResult *SimpleResult) ResponseHeaders() map[string]string
func (*SimpleResult) Screenshot ¶ added in v1.2.3
func (simpleResult *SimpleResult) Screenshot() string
func (*SimpleResult) StatusCode ¶ added in v1.2.3
func (*SimpleResult) StatusCode() int
func (*SimpleResult) Type ¶ added in v1.2.3
func (simpleResult *SimpleResult) Type() string
func (*SimpleResult) Url ¶ added in v1.2.3
func (simpleResult *SimpleResult) Url() string
Source Files ¶
Directories ¶
Path | Synopsis |
---|---|
Package cmd @Author bcy2007 2023/7/14 11:11
|
Package cmd @Author bcy2007 2023/7/14 11:11 |
Package tools @Author bcy2007 2023/7/12 16:40
|
Package tools @Author bcy2007 2023/7/12 16:40 |
config
Package config https://github.com/unknwon/goconfig
|
Package config https://github.com/unknwon/goconfig |