crawl

Oct 30, 2024

Changes in this version

+ const JSExecuteTemplate

+ func ExtractCharset(htmlContent string) string

+ func ExtractContentTypeCharset(contentType string) (charset string)

+ func InsertInto(s string, interval int, sep rune) string

+ func NewClient(options ...ClientOption) (*http.Client, error)

+ func NewTransport(uri string) (*http.Transport, error)

+ func RequestOnce(client *http.Client, uri string) (banner *rule.Banner, redirectURL string, err error)

+ func ResponseDecoding(body []byte, label string) string

+ type ClientOption func(*http.Client) error

+ func WithProxy(proxy string) ClientOption

+ func WithRedirectPolicy(policy func(req *http.Request, via []*http.Request) error) ClientOption

+ func WithTimeout(timeout time.Duration) ClientOption

+ type Crawl struct

+ func NewCrawl(options *Options, finger *rule.Finger) *Crawl

+ func (c *Crawl) ExecuteWithPlugin(client *http.Client, baseURL string, plugin *rule.Plugin) ([]*rule.Banner, error)

+ func (c *Crawl) Match(uri string) (banner *rule.Banner, m map[string]map[string]string, err error)

+ type Options struct

+ DebugResp bool

+ DisableIcon bool

+ Proxy string

+ Timeout time.Duration