crawl

Documentation

Index

• Constants
• func ExtractCharset(htmlContent string) string
• func ExtractContentTypeCharset(contentType string) (charset string)
• func InsertInto(s string, interval int, sep rune) string
• func NewClient(options ...ClientOption) (*http.Client, error)
• func NewTransport(uri string) (*http.Transport, error)
• func RequestOnce(client *http.Client, uri string) (banner *rule.Banner, redirectURL string, err error)
• func ResponseDecoding(body []byte, label string) string
• type ClientOption
  • func WithProxy(proxy string) ClientOption
  • func WithRedirectPolicy(policy func(req *http.Request, via []*http.Request) error) ClientOption
  • func WithTimeout(timeout time.Duration) ClientOption
• type Crawl
  • func NewCrawl(options *Options, finger *rule.Finger) *Crawl
  • func (c *Crawl) ExecuteWithPlugin(client *http.Client, baseURL string, plugin *rule.Plugin) ([]*rule.Banner, error)
  • func (c *Crawl) Match(uri string) (banner *rule.Banner, m map[string]map[string]string, err error)
• type Options

Constants

const JSExecuteTemplate = `` /* 813-byte string literal not displayed */

Functions

func ExtractCharset

func ExtractCharset(htmlContent string) string

func ExtractContentTypeCharset

func ExtractContentTypeCharset(contentType string) (charset string)

func InsertInto

func InsertInto(s string, interval int, sep rune) string

func NewClient

func NewClient(options ...ClientOption) (*http.Client, error)

func NewTransport

func NewTransport(uri string) (*http.Transport, error)

NewTransport 创建一个带有 SOCKS5 代理的 http.Transport

func RequestOnce

func RequestOnce(client *http.Client, uri string) (banner *rule.Banner, redirectURL string, err error)

func ResponseDecoding

func ResponseDecoding(body []byte, label string) string

Types

type ClientOption

type ClientOption func(*http.Client) error

func WithProxy

func WithProxy(proxy string) ClientOption

WithProxy 配置代理

func WithRedirectPolicy

func WithRedirectPolicy(policy func(req *http.Request, via []*http.Request) error) ClientOption

WithRedirectPolicy 自定义重定向策略

func WithTimeout

func WithTimeout(timeout time.Duration) ClientOption

WithTimeout 配置超时时间

type Crawl

type Crawl struct {
	// contains filtered or unexported fields
}

func NewCrawl

func NewCrawl(options *Options, finger *rule.Finger) *Crawl

func (*Crawl) ExecuteWithPlugin

func (c *Crawl) ExecuteWithPlugin(client *http.Client, baseURL string, plugin *rule.Plugin) ([]*rule.Banner, error)

func (*Crawl) Match

func (c *Crawl) Match(uri string) (banner *rule.Banner, m map[string]map[string]string, err error)

type Options

type Options struct {
	DisableIcon bool
	Proxy       string
	DebugResp   bool
	Timeout     time.Duration
}