fisher

README

fisher

Crawl utils.

Documentation

Index

• Constants
• func ExtractJsonListFromFile(srcFile string) ([]map[string]interface{}, error)
• func GetFullLink(ctx context.Context, itemSel string) (hrefUrl string, err error)
• func GetProxyAddress(ctx context.Context, url string) (rtn string, err error)
• func GetProxyAddressArray(ctx context.Context, url string) (rtn []string, err error)
• func GetProxyAddressString(obj ProxyInfo) (result string)
• func GetProxyArrayWithConvertFunc(ctx context.Context, url string, convertFunc ConvertFunc) (rtn []string, err error)
• func GetSocks5Proxy(ctx context.Context, url string) (rtn string, err error)
• func GetSocks5ProxyArray(ctx context.Context, url string) (rtn []string, err error)
• func GetSocks5ProxyUrl(obj ProxyInfo) (result string)
• func GetTbodyDom(content string) (*goquery.Document, error)
• func GetTheadDom(content string) (*goquery.Document, error)
• func NewChromedpContext(ctx context.Context) (context.Context, context.CancelFunc)
• func RecordData(data interface{}) (err error)
• func RunWithCrawler(ctx context.Context, crawler Crawler)
• func SetRecordWriter(writer ioutils.Writer)
• func StartRecoder(ctx context.Context, writer ioutils.Writer)
• func ToCrawl(action CrawlAction)
• type ConvertFunc
• type CrawlAction
  • func GetAction(crawler Crawler) CrawlAction
• type Crawler
• type ProxyInfo
  • func GetProxyInfo(ctx context.Context, url string) (ret []ProxyInfo, err error)
• type ProxyResponse
• type Root

Constants

const COMPLETE_OPT_SIZE = 128

const PARA_OPT_SIZE = 16

Functions

func ExtractJsonListFromFile

func ExtractJsonListFromFile(srcFile string) ([]map[string]interface{}, error)

func GetFullLink

func GetFullLink(ctx context.Context, itemSel string) (hrefUrl string, err error)

使用chromedp.Run方法获取完整链接，目标项之中必须本身包含href属性。

func GetProxyAddress

func GetProxyAddress(ctx context.Context, url string) (rtn string, err error)

获取单个SOCKS5代理地址

func GetProxyAddressArray

func GetProxyAddressArray(ctx context.Context, url string) (rtn []string, err error)

获取一组SOCKS5代理地址

func GetProxyAddressString

func GetProxyAddressString(obj ProxyInfo) (result string)

func GetProxyArrayWithConvertFunc

func GetProxyArrayWithConvertFunc(ctx context.Context, url string, convertFunc ConvertFunc) (rtn []string, err error)

获取一组SOCKS5代理地址

func GetSocks5Proxy

func GetSocks5Proxy(ctx context.Context, url string) (rtn string, err error)

获取单个SOCKS5代理地址

func GetSocks5ProxyArray

func GetSocks5ProxyArray(ctx context.Context, url string) (rtn []string, err error)

获取一组SOCKS5代理地址

func GetSocks5ProxyUrl

func GetSocks5ProxyUrl(obj ProxyInfo) (result string)

func GetTbodyDom

func GetTbodyDom(content string) (*goquery.Document, error)

如果是tbody类型的元素，则必须进行此替换，goquery才能正常读取解析，且实际读取时，依然保持tbody标识

func GetTheadDom

func GetTheadDom(content string) (*goquery.Document, error)

如果是thead类型的元素，则必须进行此替换，goquery才能正常读取解析，且实际读取时，需要调整为以tbody进行标识

func NewChromedpContext

func NewChromedpContext(ctx context.Context) (context.Context, context.CancelFunc)

func RecordData

func RecordData(data interface{}) (err error)

写入已经json序列化好的字符数组，字符串或者是可以进行json序列化的对象

func RunWithCrawler

func RunWithCrawler(ctx context.Context, crawler Crawler)

func SetRecordWriter

func SetRecordWriter(writer ioutils.Writer)

func StartRecoder

func StartRecoder(ctx context.Context, writer ioutils.Writer)

开启存储数据的服务，便于实现全局写数据

func ToCrawl

func ToCrawl(action CrawlAction)

Types

type ConvertFunc

type ConvertFunc func(obj ProxyInfo) (result string)

type CrawlAction

type CrawlAction = func(proxyReqUrl string, isHeadless bool, customMap map[string]string, outputFile string)

func GetAction

func GetAction(crawler Crawler) CrawlAction

type Crawler

type Crawler interface {
	Crawl(ctx context.Context) error
	ParseArgs(argMap map[string]string) error
}

type ProxyInfo

type ProxyInfo struct {
	IP   string `json:"ip"`
	Port int    `json:"port"`
}

先对 json 格式进行 struct 结构定义

func GetProxyInfo

func GetProxyInfo(ctx context.Context, url string) (ret []ProxyInfo, err error)

获取一组代理地址信息

type ProxyResponse

type ProxyResponse struct {
	Code    int         `json:"code"`
	Data    []ProxyInfo `json:"data"`
	Msg     string      `json:"msg"`
	Success bool        `json:"success"`
}

type Root

type Root struct {
	// contains filtered or unexported fields
}