fast_crawl_engine

Documentation

Index

• func FilterNetWorkRequest(string2 string) bool
• type FastCrawlCookies
• type FastCrawlEngine
  • func NewFastCrawlEngine(params FastCrawlEngineParams) *FastCrawlEngine
  • func (c *FastCrawlEngine) Start()
• type FastCrawlEngineParams
• type FastCrawlResult
  • func NewFastCrawlResult() *FastCrawlResult
  • func (c *FastCrawlResult) Add(result FastCrawlResultData)
  • func (c *FastCrawlResult) All() []*FastCrawlResultData
  • func (c *FastCrawlResult) PrintString() string
  • func (c *FastCrawlResult) Save()
  • func (c *FastCrawlResult) SendTask()
• type FastCrawlResultData

Functions

func FilterNetWorkRequest

func FilterNetWorkRequest(string2 string) bool

Types

type FastCrawlCookies

type FastCrawlCookies struct {
	Value  string
	Domain string
	Path   string
}

type FastCrawlEngine

type FastCrawlEngine struct {
	// contains filtered or unexported fields
}

func NewFastCrawlEngine

func NewFastCrawlEngine(params FastCrawlEngineParams) *FastCrawlEngine

外部实例

func (*FastCrawlEngine) Start

func (c *FastCrawlEngine) Start()

启动扫描

type FastCrawlEngineParams

type FastCrawlEngineParams struct {
	BaseDomain   string            `根域名`
	DomainStr    string            `渲染抓取的域名`
	Cookies      *FastCrawlCookies `cookie`
	Host         string            `服务器地址 可为空`
	MinDeepLevel int               `基础深度`
	MaxDeepLevel int               `最大深度`
}

type FastCrawlResult

type FastCrawlResult struct {
	// contains filtered or unexported fields
}

func NewFastCrawlResult

func NewFastCrawlResult() *FastCrawlResult

func (*FastCrawlResult) Add

func (c *FastCrawlResult) Add(result FastCrawlResultData)

func (*FastCrawlResult) All

func (c *FastCrawlResult) All() []*FastCrawlResultData

func (*FastCrawlResult) PrintString

func (c *FastCrawlResult) PrintString() string

func (*FastCrawlResult) Save

func (c *FastCrawlResult) Save()

func (*FastCrawlResult) SendTask

func (c *FastCrawlResult) SendTask()

type FastCrawlResultData

type FastCrawlResultData struct {
	BaseDomain   string            `基础域名`
	UrlStr       string            `当前URL`
	Title        string            `当前标题`
	Method       string            `请求方式`
	DeepLevel    int               `深度`
	MaxDeepLevel int               `最大深度`
	Cookies      *FastCrawlCookies `cookie`
	Host         string            `服务器地址 可为空`
}