gathertool

README

gathertool

轻量级爬虫，接口测试，压力测试框架, 提高开发对应场景的golang程序的效率。

文档： 点击开始

开始

go get github.com/mangenotwork/gathertool

import gt "github.com/mangenotwork/gathertool"

func main(){
    caseUrl := "https://www.baidu.com"
    ctx, err := gt.Get(caseUrl)
    if err != nil {
        log.Println(err)
    }
    log.Println(ctx.RespBodyString)
}

含请求事件请求

import gt "github.com/mangenotwork/gathertool"

func main(){

    gt.NewGet(`http://192.168.0.1`).SetStartFunc(func(ctx *gt.Context){
            log.Println("请求前： 添加代理等等操作")
            ctx.Client.Transport = &http.Transport{Proxy: http.ProxyURL(proxyUrl)}
        }
    ).SetSucceedFunc(func(ctx *gt.Context){
            log.Println("请求成功： 处理数据或存储等等")
            log.Println(ctx.RespBodyString())
        }
    ).SetFailedFunc(func(ctx *gt.Context){
            log.Println("请求失败： 记录失败等等")
            log.Println(ctx.Err)
        }
    ).SetRetryFunc(func(ctx *gt.Context){
             log.Println("请求重试： 更换代理或添加等待时间等等")
             ctx.Client.Transport = &http.Transport{Proxy: http.ProxyURL(proxyUrl)}
        }
    ).SetEndFunc(func(ctx *gt.Context){
             log.Println("请求结束： 记录结束，处理数据等等")
             log.Println(ctx)
        }
    ).Do()
    
}

优雅的写法

func main(){
    gt.NewGet(`http://192.168.0.1`).SetSucceedFunc(succeed).SetFailedFunc(failed).SetRetryFunc(retry).Do()
    gt.NewGet(`http://www.baidu.com`).SetSucceedFunc(baiduSucceed).SetFailedFunc(failed).SetRetryFunc(retry).Do()
}
// 请求成功： 处理数据或存储等等
func succeed(ctx *gt.Context){
    log.Println(ctx.RespBodyString())
}
// 请求失败： 记录失败等等
func failed(ctx *gt.Context){
    log.Println(ctx.Err)
}
// 请求重试： 更换代理或添加等待时间等等
func retry(ctx *gt.Context){
    ctx.Client.Transport = &http.Transport{Proxy: http.ProxyURL(proxyUrl)}
}
// 百度请求成功后
func baiduSucceed(ctx *gt.Context){
    log.Println(ctx.RespBodyString())
}

实例

• Get请求
• 阳光高考招生章程抓取
• ip地址信息抓取
• 压力测试
• 文件下载
• 无登录微博抓取
• 百度题库抓取

BUG

• MysqlDB.NewTable() 字段参数是map, 创建的表会乱序
• client.Transport = &http.Transport{ MaxIdleConnsPerHost: 1000, } 解决连接数高的问题

常见的反爬虫策略

• User-Agent反爬
• IP访问频率限制
• 强制登录
• 漏桶、令牌桶之类的算法限制接口访问频率
• 数据接口加上庞大的加密解密和混淆算法
• JS代码执行后，会发送一个带参数key的请求，后台通过判断key的值来决定是响应真实的页面，还是响应伪造或错误的页面。因为key参数是动态生成的，每次都不一样，难以分析出其生成方法，使得无法构造对应的http请求

Documentation

Index

• Constants
• Variables
• func Any2Arr(data interface{}) []interface{}
• func Any2Float64(data interface{}) float64
• func Any2Int(data interface{}) int
• func Any2Map(data interface{}) map[string]interface{}
• func Any2String(data interface{}) string
• func Any2Strings(data interface{}) []string
• func Any2int64(data interface{}) int64
• func Base64Decode(str string) (string, error)
• func Base64Encode(str string) string
• func Base64UrlDecode(str string) (string, error)
• func Base64UrlEncode(str string) string
• func BeginDayUnix() int64
• func Byte2Str(b []byte) string
• func ByteToBinaryString(data byte) (str string)
• func CPUMax()
• func CleaningStr(str string) string
• func ConvertByte2String(byte []byte, charset Charset) string
• func Daydiff(beginDay string, endDay string) int
• func EndDayUnix() int64
• func FileMd5(path string) (string, error)
• func FileMd5sum(fileName string) string
• func FileSizeFormat(fileSize int64) (size string)
• func GetAgent(agentType UserAgentType) string
• func GetNowPath() string
• func IF(condition bool, a, b interface{}) interface{}
• func IsContainStr(items []string, item string) bool
• func IsElementStr(list []string, element string) bool
• func Json2Map(str string) map[string]interface{}
• func MD5(str string) string
• func MongoConn()
• func MongoConn1()
• func NewMysqlDB(host string, port int, user, password, database string) (err error)
• func PanicToError(fn func()) (err error)
• func PathExists(path string)
• func Ping(ip string)
• func ReadCsvFile(filename string) [][]string
• func RedisDELKeys(rds *Rds, keys string, jobNumber int)
• func RegFindAll(regStr, rest string) [][]string
• func RegHtmlA(str string) []string
• func RegHtmlATxt(str string) []string
• func RegHtmlH(str, typeH string) []string
• func RegHtmlHTxt(str, typeH string) []string
• func RegHtmlHref(str string) []string
• func RegHtmlHrefTxt(str string) []string
• func RegHtmlInput(str string) []string
• func RegHtmlInputTxt(str string) []string
• func RegHtmlP(str string) []string
• func RegHtmlPTxt(str string) []string
• func RegHtmlSpan(str string) []string
• func RegHtmlSpanTxt(str string) []string
• func RegHtmlSrc(str string) []string
• func RegHtmlSrcTxt(str string) []string
• func RegHtmlTbody(str string) []string
• func RegHtmlTd(str string) []string
• func RegHtmlTdTxt(str string) []string
• func RegHtmlTitle(str string) []string
• func RegHtmlTitleTxt(str string) []string
• func RegHtmlTr(str string) []string
• func RegHtmlTrTxt(str string) []string
• func SSHClient(user string, pass string, addr string) (*ssh.Client, error)
• func SearchDomain(ip string)
• func SearchPort(ipStr string, vs ...interface{})
• func SetAgent(agentType UserAgentType, agent string)
• func SetStatusCodeFailEvent(code int)
• func SetStatusCodeRetryEvent(code int)
• func SetStatusCodeSuccessEvent(code int)
• func StartJob(jobNumber int, queue TodoQueue, f func(task *Task))
• func StartJobGet(jobNumber int, queue TodoQueue, vs ...interface{})
• func StartJobPost(jobNumber int, queue TodoQueue, vs ...interface{})
• func Str2Float64(str string) float64
• func Str2Int64(str string) int64
• func StrDeleteSpace(str string) string
• func StringValue(i interface{}) string
• func StringValueMysql(i interface{}) string
• func StrsDuplicates(a []string) []string
• func Struct2Map(obj interface{}, hasValue bool) (map[string]interface{}, error)
• func TickerRun(t time.Duration, runFirst bool, f func())
• func Timestamp() string
• func Uint82Str(bs []uint8) string
• func UnescapeUnicode(raw []byte) ([]byte, error)
• type Charset
• type Context
  • func Delete(url string, vs ...interface{}) (*Context, error)
  • func Get(url string, vs ...interface{}) (*Context, error)
  • func NewDelete(url string, vs ...interface{}) *Context
  • func NewGet(url string, vs ...interface{}) *Context
  • func NewOptions(url string, vs ...interface{}) *Context
  • func NewPost(url string, data []byte, contentType string, vs ...interface{}) *Context
  • func NewPut(url string, data []byte, contentType string, vs ...interface{}) *Context
  • func NewRequest(url, method string, data []byte, contentType string, vs ...interface{}) *Context
  • func Options(url string, vs ...interface{}) (*Context, error)
  • func Post(url string, data []byte, contentType string, vs ...interface{}) (*Context, error)
  • func PostForm(url string, data url.Values, vs ...interface{}) (*Context, error)
  • func PostJson(url string, jsonStr string, vs ...interface{}) (*Context, error)
  • func Put(url string, data []byte, contentType string, vs ...interface{}) (*Context, error)
  • func Req(request *http.Request, vs ...interface{}) *Context
  • func Request(url, method string, data []byte, contentType string, vs ...interface{}) (*Context, error)
  • func Upload(url, savePath string, vs ...interface{}) (*Context, error)
  • func (c *Context) AddCookie(k, v string) *Context
  • func (c *Context) AddHeader(k, v string) *Context
  • func (c *Context) CheckMd5() string
  • func (c *Context) CheckReqMd5() string
  • func (c *Context) CloseLog()
  • func (c *Context) CloseRetry()
  • func (c *Context) CookieNext() error
  • func (c *Context) Do() func()
  • func (c *Context) OpenErr2Retry()
  • func (c *Context) RespBodyArr() []interface{}
  • func (c *Context) RespBodyMap() map[string]interface{}
  • func (c *Context) RespBodyString() string
  • func (c *Context) SetFailedFunc(failedFunc func(c *Context)) *Context
  • func (c *Context) SetProxy(proxyUrl string) *Context
  • func (c *Context) SetProxyFunc(f func() *http.Transport) *Context
  • func (c *Context) SetRetryFunc(retryFunc func(c *Context)) *Context
  • func (c *Context) SetRetryTimes(times int) *Context
  • func (c *Context) SetSucceedFunc(successFunc func(c *Context)) *Context
  • func (c *Context) Upload(filePath string) func()
• type Csv
  • func NewCsv(fileName string) (*Csv, error)
  • func (c *Csv) Add(data []string) error
  • func (c *Csv) ReadAll() ([][]string, error)
• type EndFunc
• type Err2Retry
• type FailedFunc
• type Header
• type IsLog
• type Mongo
  • func NewMongo(user, password, host, port string) (*Mongo, error)
  • func (m *Mongo) GetCollection(dbname, name string)
  • func (m *Mongo) GetConn() (err error)
  • func (m *Mongo) GetDB(dbname string)
  • func (m *Mongo) Insert(document interface{}) error
• type Mysql
  • func GetMysqlDBConn() (*Mysql, error)
  • func NewMysql(host string, port int, user, password, database string) (*Mysql, error)
  • func (m *Mysql) CloseLog()
  • func (m *Mysql) Conn() (err error)
  • func (m *Mysql) Delete(sql string) error
  • func (m *Mysql) Describe(table string) (*tableDescribe, error)
  • func (m *Mysql) Exec(sql string) error
  • func (m *Mysql) Insert(table string, fieldData map[string]interface{}) error
  • func (m *Mysql) InsertAt(table string, fieldData map[string]interface{}) error
  • func (m *Mysql) IsHaveTable(name string) bool
  • func (m *Mysql) NewTable(table string, fields map[string]string) error
  • func (m *Mysql) Query(sql string) ([]map[string]string, error)
  • func (m *Mysql) Select(sql string) ([]map[string]string, error)
  • func (m *Mysql) SetMaxIdleConn(number int)
  • func (m *Mysql) SetMaxOpenConn(number int)
  • func (m *Mysql) ToVarChar(data interface{}) string
  • func (m *Mysql) Update(sql string) error
• type ProxyUrl
• type Queue
  • func (q *Queue) Add(task *Task) error
  • func (q *Queue) Clear() bool
  • func (q *Queue) IsEmpty() bool
  • func (q *Queue) Poll() *Task
  • func (q *Queue) Print()
  • func (q *Queue) Size() int
• type Rds
  • func NewRedis(host, port, password string, db int, vs ...interface{}) *Rds
  • func NewRedisPool(host, port, password string, db, maxIdle, maxActive, idleTimeoutSec int, ...) *Rds
  • func (r *Rds) GetConn() redis.Conn
  • func (r *Rds) RedisConn() (err error)
  • func (r *Rds) RedisPool() error
  • func (r *Rds) SelectDB(dbNumber int) error
• type ReqTimeOut
• type ReqTimeOutMs
• type ReqUrl
• type RetryFunc
• type RetryTimes
• type SSHConnInfo
  • func NewSSHInfo(addr, user, password string) *SSHConnInfo
• type Set
  • func (s Set) Add(key string)
  • func (s Set) Delete(key string)
  • func (s Set) Has(key string) bool
• type Stack
  • func New() *Stack
  • func (s *Stack) Pop()
  • func (s *Stack) Push(data interface{})
  • func (s *Stack) String() string
• type StartFunc
• type StressUrl
  • func NewTestUrl(url, method string, sum int64, total int) *StressUrl
  • func (s *StressUrl) Run(vs ...interface{})
  • func (s *StressUrl) SetJson(str string)
• type SucceedFunc
• type TableInfo
• type Task
  • func CrawlerTask(url, jsonParam string, vs ...interface{}) *Task
  • func (task Task) AddData(key string, value interface{}) Task
  • func (task Task) GetDataStr(key string) string
• type TodoQueue
  • func NewQueue() TodoQueue
  • func NewUploadQueue() TodoQueue
• type UploadQueue
  • func (q *UploadQueue) Add(task *Task) error
  • func (q *UploadQueue) Clear() bool
  • func (q *UploadQueue) IsEmpty() bool
  • func (q *UploadQueue) Poll() *Task
  • func (q *UploadQueue) Print()
  • func (q *UploadQueue) Size() int
• type UserAgentType
• type WSClient
  • func WsClient(host, path string) (WSClient, error)

Constants

const (
	SHOW_TABLES     = "SHOW TABLES"
	TABLE_NAME_NULL = "table name is null."
)

const (
	POST    = "POST"
	GET     = "GET"
	HEAD    = "HEAD"
	PUT     = "PUT"
	DELETE  = "DELETE"
	PATCH   = "PATCH"
	OPTIONS = "OPTIONS"
	ANY     = ""
)

const (
	UTF8    = Charset("UTF-8")
	GB18030 = Charset("GB18030")
	GBK     = Charset("GBK")
	GB2312  = Charset("GB2312")
)

Variables

var (
	UrlBad error = errors.New("url is bad.")  // 错误的url
	UrlNil error = errors.New("url is null.") // 空的url
)

var AgentType = map[UserAgentType][]int{
	PCAgent:           listPCAgent,
	WindowsAgent:      listWindowsAgent,
	LinuxAgent:        listLinuxAgent,
	MacAgent:          listMacAgent,
	AndroidAgent:      listAndroidAgent,
	IosAgent:          listIosAgent,
	PhoneAgent:        listPhoneAgent,
	WindowsPhoneAgent: listWindowsPhoneAgent,
	UCAgent:           listUCAgent,
}

var CookiePool = &cookiePool{}

var MysqlDB = &Mysql{}

开放的mysql对象

var StatusCodeMap map[int]string = map[int]string{
	200: "success",
	201: "success",
	202: "success",
	203: "success",
	204: "fail",
	300: "success",
	301: "success",
	302: "success",
	400: "fail",
	401: "retry",
	402: "retry",
	403: "retry",
	404: "fail",
	405: "retry",
	406: "retry",
	407: "retry",
	408: "retry",
	500: "fail",
	501: "fail",
	502: "retry",
	503: "retry",
	504: "retry",
}

StatusCodeMap 状态码处理映射 success 该状态码对应执行成功函数 fail 该状态码对应执行失败函数 retry 该状态码对应需要重试前执行的函数

var UserAgentMap map[int]string = map[int]string{
	1:  "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0",
	2:  "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36",
	3:  "Mozilla/5.0 (compatible; WOW64; MSIE 10.0; Windows NT 6.2)",
	4:  "Opera/9.80 (Windows NT 6.1; WOW64; U; en) Presto/2.10.229 Version/11.62",
	5:  "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
	6:  "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20130331 Firefox/21.0",
	7:  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Ubuntu/11.10 Chromium/27.0.1453.93 Chrome/27.0.1453.93 Safari/537.36",
	8:  "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
	9:  "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.9.168 Version/11.52",
	10: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
	11: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:21.0) Gecko/20100101 Firefox/21.0",
	12: "Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166  Safari/535.19",
	13: "Mozilla/5.0 (Linux; U; Android 4.0.4; en-gb; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
	14: "Mozilla/5.0 (Linux; U; Android 2.2; en-gb; GT-P1000 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
	15: "Mozilla/5.0 (Android; Mobile; rv:14.0) Gecko/14.0 Firefox/14.0",
	16: "Mozilla/5.0 (Android; Tablet; rv:14.0) Gecko/14.0 Firefox/14.0",
	17: "Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19",
	18: "Mozilla/5.0 (Linux; Android 4.1.2; Nexus 7 Build/JZ054K) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19",
	19: "Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
	20: "Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
	21: "Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
	22: "Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
	23: "Mozilla/5.0 (iPhone; CPU iPhone OS 6_1_4 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) CriOS/27.0.1453.10 Mobile/10B350 Safari/8536.25",
	24: "Mozilla/5.0 (compatible; MSIE 10.0; Windows Phone 8.0; Trident/6.0; IEMobile/10.0; ARM; Touch; NOKIA; Lumia 920)",
	25: "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; SAMSUNG; SGH-i917)",
	26: "User-Agent, UCWEB7.0.2.37/28/999",
	27: "User-Agent, NOKIA5700/ UCWEB7.0.2.37/28/999",
	28: "User-Agent, Openwave/ UCWEB7.0.2.37/28/999",
	29: "User-Agent, Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999",
	30: "Mozilla/5.0 (Windows; U; Windows NT 6.1; ) AppleWebKit/534.12 (KHTML, like Gecko) Maxthon/3.0 Safari/534.12",
	31: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E)",
	32: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)",
	33: "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.33 Safari/534.3 SE 2.X MetaSr 1.0",
	34: "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E)",
	35: "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1 QQBrowser/6.9.11079.201",
}

Functions

func Any2Arr

func Any2Arr(data interface{}) []interface{}

Any2Arr interface{} -> []interface{}

func Any2Float64

func Any2Float64(data interface{}) float64

Any2Float64 interface{} -> float64

func Any2Int

func Any2Int(data interface{}) int

Any2Int interface{} -> int

func Any2Map

func Any2Map(data interface{}) map[string]interface{}

Any2Map interface{} -> map[string]interface{}

func Any2String

func Any2String(data interface{}) string

Any2String interface{} -> string

func Any2Strings

func Any2Strings(data interface{}) []string

Any2Strings interface{} -> []string

func Any2int64

func Any2int64(data interface{}) int64

Any2int64 interface{} -> int64

func Base64Decode

func Base64Decode(str string) (string, error)

base64 解码

func Base64Encode

func Base64Encode(str string) string

base64 编码

func Base64UrlDecode

func Base64UrlDecode(str string) (string, error)

base64 url 解码

func Base64UrlEncode

func Base64UrlEncode(str string) string

base64 url 编码

func BeginDayUnix

func BeginDayUnix() int64

BeginDayUnix 获取当天 0点

func Byte2Str

func Byte2Str(b []byte) string

[]byte -> string

func ByteToBinaryString

func ByteToBinaryString(data byte) (str string)

ByteToBinaryString 字节 -> 二进制字符串

func CPUMax

func CPUMax()

CPUMax 多核执行

func CleaningStr

func CleaningStr(str string) string

CleaningStr 清理字符串前后空白 和回车 换行符号

func ConvertByte2String

func ConvertByte2String(byte []byte, charset Charset) string

编码转换

func Daydiff

func Daydiff(beginDay string, endDay string) int

两个时间字符串的日期差

func EndDayUnix

func EndDayUnix() int64

EndDayUnix 获取当天 24点

func FileMd5

func FileMd5(path string) (string, error)

FileMd5 file md5 文件md5

func FileMd5sum

func FileMd5sum(fileName string) string

文件 Md5

func FileSizeFormat

func FileSizeFormat(fileSize int64) (size string)

字节的单位转换 保留两位小数

func GetAgent

func GetAgent(agentType UserAgentType) string

GetAgent 随机获取那种类型的 user-agent

func GetNowPath

func GetNowPath() string

获取当前运行路径

func IF

func IF(condition bool, a, b interface{}) interface{}

三元表达式 use: IF(a>b, a, b).(int)

func IsContainStr

func IsContainStr(items []string, item string) bool

IsContainStr 字符串是否等于items中的某个元素

func IsElementStr

func IsElementStr(list []string, element string) bool

判断字符串是否与数组里的某个字符串相同

func Json2Map

func Json2Map(str string) map[string]interface{}

Json2Map json转map函数，通用

func MD5

func MD5(str string) string

MD5

func MongoConn

func MongoConn()

func MongoConn1

func MongoConn1()

func NewMysqlDB

func NewMysqlDB(host string, port int, user, password, database string) (err error)

NewMysqlDB 给mysql对象进行连接

func PanicToError

func PanicToError(fn func()) (err error)

panic -> error

func PathExists

func PathExists(path string)

目录不存在则创建

func Ping

func Ping(ip string)

func ReadCsvFile

func ReadCsvFile(filename string) [][]string

csv file -> [][]string 行列

func RedisDELKeys

func RedisDELKeys(rds *Rds, keys string, jobNumber int)

Del key

func RegFindAll

func RegFindAll(regStr, rest string) [][]string

func RegHtmlA

func RegHtmlA(str string) []string

func RegHtmlATxt

func RegHtmlATxt(str string) []string

func RegHtmlH

func RegHtmlH(str, typeH string) []string

func RegHtmlHTxt

func RegHtmlHTxt(str, typeH string) []string

func RegHtmlHref

func RegHtmlHref(str string) []string

func RegHtmlHrefTxt

func RegHtmlHrefTxt(str string) []string

func RegHtmlInput

func RegHtmlInput(str string) []string

func RegHtmlInputTxt

func RegHtmlInputTxt(str string) []string

func RegHtmlP

func RegHtmlP(str string) []string

func RegHtmlPTxt

func RegHtmlPTxt(str string) []string

func RegHtmlSpan

func RegHtmlSpan(str string) []string

func RegHtmlSpanTxt

func RegHtmlSpanTxt(str string) []string

func RegHtmlSrc

func RegHtmlSrc(str string) []string

func RegHtmlSrcTxt

func RegHtmlSrcTxt(str string) []string

func RegHtmlTbody

func RegHtmlTbody(str string) []string

func RegHtmlTd

func RegHtmlTd(str string) []string

func RegHtmlTdTxt

func RegHtmlTdTxt(str string) []string

func RegHtmlTitle

func RegHtmlTitle(str string) []string

func RegHtmlTitleTxt

func RegHtmlTitleTxt(str string) []string

func RegHtmlTr

func RegHtmlTr(str string) []string

func RegHtmlTrTxt

func RegHtmlTrTxt(str string) []string

func SSHClient

func SSHClient(user string, pass string, addr string) (*ssh.Client, error)

SSHClient 连接ssh addr : 主机地址, 如: 127.0.0.1:22 user : 用户 pass : 密码 返回 ssh连接

func SearchDomain

func SearchDomain(ip string)

func SearchPort

func SearchPort(ipStr string, vs ...interface{})

扫描ip的端口

func SetAgent

func SetAgent(agentType UserAgentType, agent string)

func SetStatusCodeFailEvent

func SetStatusCodeFailEvent(code int)

将指定状态码设置为执行失败事件

func SetStatusCodeRetryEvent

func SetStatusCodeRetryEvent(code int)

将指定状态码设置为执行重试事件

func SetStatusCodeSuccessEvent

func SetStatusCodeSuccessEvent(code int)

将指定状态码设置为执行成功事件

func StartJob

func StartJob(jobNumber int, queue TodoQueue, f func(task *Task))

StartJob 开始运行并发

func StartJobGet

func StartJobGet(jobNumber int, queue TodoQueue, vs ...interface{})

StartJobGet 并发执行Get,直到队列任务为空 @jobNumber 并发数， @queue 全局队列， @client 单个并发任务的client， @SucceedFunc 成功方法， @ RetryFunc重试方法， @FailedFunc 失败方法

func StartJobPost

func StartJobPost(jobNumber int, queue TodoQueue, vs ...interface{})

StartJobPost 开始运行并发Post

func Str2Float64

func Str2Float64(str string) float64

string -> float64

func Str2Int64

func Str2Int64(str string) int64

string -> int64

func StrDeleteSpace

func StrDeleteSpace(str string) string

StrDeleteSpace 删除字符串前后的空格

func StringValue

func StringValue(i interface{}) string

StringValue 任何类型返回值字符串形式

func StringValueMysql

func StringValueMysql(i interface{}) string

func StrsDuplicates

func StrsDuplicates(a []string) []string

数组，切片去重和去空串

func Struct2Map

func Struct2Map(obj interface{}, hasValue bool) (map[string]interface{}, error)

Struct -> map hasValue=true表示字段值不管是否存在都转换成map hasValue=false表示字段为空或者不为0则转换成map

func TickerRun

func TickerRun(t time.Duration, runFirst bool, f func())

间隔运行 t: 间隔时间， f: 运行的方法

func Timestamp

func Timestamp() string

func Uint82Str

func Uint82Str(bs []uint8) string

[]uint8 -> string

func UnescapeUnicode

func UnescapeUnicode(raw []byte) ([]byte, error)

Unicode 转码

Types

type Charset

type Charset string

type Context

type Context struct {
	// Token
	Token string

	// client
	Client *http.Client

	// Request
	Req *http.Request

	// Response
	Resp *http.Response

	// Error
	Err error

	// Ctx context.Context
	Ctx context.Context

	// 最大允许重试次数
	MaxTimes RetryTimes

	// 请求成功了需要处理的事件
	SucceedFunc SucceedFunc

	// 请求失败了需要做的事
	FailedFunc FailedFunc

	// 请求状态码设置了重试，在重试前的事件
	RetryFunc RetryFunc

	// 请求开始前的方法
	StartFunc StartFunc

	// 请求完成后的方法
	EndFunc EndFunc

	// 本次请求的任务
	// 用于有步骤的请求和并发执行请求
	Task *Task

	// 请求返回的结果
	RespBody []byte

	// job编号
	// 在执行多并发执行抓取任务，每个并发都有一个编号
	// 这个编号是递增分配的
	JobNumber int

	// 请求的响应时间 单位ms
	Ms time.Duration

	// 是否显示日志, 默认是显示的
	IsLog IsLog
	// contains filtered or unexported fields
}

请求上下文

func Delete

func Delete(url string, vs ...interface{}) (*Context, error)

Delete

func Get

func Get(url string, vs ...interface{}) (*Context, error)

Get 请求

func NewDelete

func NewDelete(url string, vs ...interface{}) *Context

NewDelete

func NewGet

func NewGet(url string, vs ...interface{}) *Context

func NewOptions

func NewOptions(url string, vs ...interface{}) *Context

NewOptions

func NewPost

func NewPost(url string, data []byte, contentType string, vs ...interface{}) *Context

func NewPut

func NewPut(url string, data []byte, contentType string, vs ...interface{}) *Context

NewPut

func NewRequest

func NewRequest(url, method string, data []byte, contentType string, vs ...interface{}) *Context

NewRequest 请求

func Options

func Options(url string, vs ...interface{}) (*Context, error)

Options

func Post

func Post(url string, data []byte, contentType string, vs ...interface{}) (*Context, error)

POST 请求

func PostForm

func PostForm(url string, data url.Values, vs ...interface{}) (*Context, error)

POST Form

func PostJson

func PostJson(url string, jsonStr string, vs ...interface{}) (*Context, error)

POST json 请求

func Put

func Put(url string, data []byte, contentType string, vs ...interface{}) (*Context, error)

Put

func Req

func Req(request *http.Request, vs ...interface{}) *Context

Req 初始化请求 @url 请求链接 @maxTimes 重试次数 @sf 请求成功后做的事情, 200等 @ff 请求失败后做的事情, 403等，502等 @vs 可变参数 @vs UserAgentType 设置指定类型 user agent 如 AndroidAgent

func Request

func Request(url, method string, data []byte, contentType string, vs ...interface{}) (*Context, error)

Request 请求

func Upload

func Upload(url, savePath string, vs ...interface{}) (*Context, error)

Upload

func (*Context) AddCookie

func (c *Context) AddCookie(k, v string) *Context

add Cookie

func (*Context) AddHeader

func (c *Context) AddHeader(k, v string) *Context

add header

func (*Context) CheckMd5

func (c *Context) CheckMd5() string

func (*Context) CheckReqMd5

func (c *Context) CheckReqMd5() string

func (*Context) CloseLog

func (c *Context) CloseLog()

close log

func (*Context) CloseRetry

func (c *Context) CloseRetry()

关闭重试

func (*Context) CookieNext

func (c *Context) CookieNext() error

CookieNext

func (*Context) Do

func (c *Context) Do() func()

Do 执行请求

func (*Context) OpenErr2Retry

func (c *Context) OpenErr2Retry()

开启请求失败都执行retry

func (*Context) RespBodyArr

func (c *Context) RespBodyArr() []interface{}

func (*Context) RespBodyMap

func (c *Context) RespBodyMap() map[string]interface{}

func (*Context) RespBodyString

func (c *Context) RespBodyString() string

func (*Context) SetFailedFunc

func (c *Context) SetFailedFunc(failedFunc func(c *Context)) *Context

SetFailed 设置错误后的方法

func (*Context) SetProxy

func (c *Context) SetProxy(proxyUrl string) *Context

set proxy

func (*Context) SetProxyFunc

func (c *Context) SetProxyFunc(f func() *http.Transport) *Context

set proxy func

func (*Context) SetRetryFunc

func (c *Context) SetRetryFunc(retryFunc func(c *Context)) *Context

SetRetryFunc 设置重试，在重试前的方法

func (*Context) SetRetryTimes

func (c *Context) SetRetryTimes(times int) *Context

SetRetryTimes 设置重试次数

func (*Context) SetSucceedFunc

func (c *Context) SetSucceedFunc(successFunc func(c *Context)) *Context

SetSucceedFunc 设置成功后的方法

func (*Context) Upload

func (c *Context) Upload(filePath string) func()

Upload 下载

type Csv

type Csv struct {
	FileName string
	W        *csv.Writer
	R        *csv.Reader
}

func NewCsv

func NewCsv(fileName string) (*Csv, error)

新创建一个csv对象

func (*Csv) Add

func (c *Csv) Add(data []string) error

写入csv

func (*Csv) ReadAll

func (c *Csv) ReadAll() ([][]string, error)

读取所有

type EndFunc

type EndFunc func(c *Context)

请求结束后的方法类型

type Err2Retry

type Err2Retry bool

设置遇到错误执行 Retry 事件

type FailedFunc

type FailedFunc func(c *Context)

失败后的方法类型

type Header

type Header map[string]string

type IsLog

type IsLog bool

是否开启日志

type Mongo

type Mongo struct {
	User        string
	Password    string
	Host        string
	Port        string
	Conn        *mongo.Client
	Database    *mongo.Database
	Collection  *mongo.Collection
	MaxPoolSize int
	TimeOut     time.Duration
}

func NewMongo

func NewMongo(user, password, host, port string) (*Mongo, error)

func (*Mongo) GetCollection

func (m *Mongo) GetCollection(dbname, name string)

连接mongodb 的db的集合 @dbname DB名 @name 集合名

func (*Mongo) GetConn

func (m *Mongo) GetConn() (err error)

建立mongodb 连接

func (*Mongo) GetDB

func (m *Mongo) GetDB(dbname string)

连接mongodb 的db @dbname DB名

func (*Mongo) Insert

func (m *Mongo) Insert(document interface{}) error

插入数据 @document 可以是 Struct, 是 Slice，

type Mysql

type Mysql struct {
	DB *sql.DB
	// contains filtered or unexported fields
}

func GetMysqlDBConn

func GetMysqlDBConn() (*Mysql, error)

GetMysqlDBConn 获取mysql 连接

func NewMysql

func NewMysql(host string, port int, user, password, database string) (*Mysql, error)

func (*Mysql) CloseLog

func (m *Mysql) CloseLog()

CloseLog 关闭日志

func (*Mysql) Conn

func (m *Mysql) Conn() (err error)

Conn 连接mysql

func (*Mysql) Delete

func (m *Mysql) Delete(sql string) error

Delete

func (*Mysql) Describe

func (m *Mysql) Describe(table string) (*tableDescribe, error)

Describe 获取表结构

func (*Mysql) Exec

func (m *Mysql) Exec(sql string) error

Exec

func (*Mysql) Insert

func (m *Mysql) Insert(table string, fieldData map[string]interface{}) error

Insert 新增数据

func (*Mysql) InsertAt

func (m *Mysql) InsertAt(table string, fieldData map[string]interface{}) error

InsertAt 新增数据 如果没有表则先创建表

func (*Mysql) IsHaveTable

func (m *Mysql) IsHaveTable(name string) bool

IsHaveTable 表是否存在

func (*Mysql) NewTable

func (m *Mysql) NewTable(table string, fields map[string]string) error

NewTable 创建表 字段顺序不固定 fields 字段:类型； name:varchar(10);

func (*Mysql) Query

func (m *Mysql) Query(sql string) ([]map[string]string, error)

Query

func (*Mysql) Select

func (m *Mysql) Select(sql string) ([]map[string]string, error)

Select 查询语句 返回 map

func (*Mysql) SetMaxIdleConn

func (m *Mysql) SetMaxIdleConn(number int)

SetMaxIdleConn

func (*Mysql) SetMaxOpenConn

func (m *Mysql) SetMaxOpenConn(number int)

SetMaxOpenConn

func (*Mysql) ToVarChar

func (m *Mysql) ToVarChar(data interface{}) string

ToVarChar 写入mysql 的字符类型

func (*Mysql) Update

func (m *Mysql) Update(sql string) error

Update

type ProxyUrl

type ProxyUrl string

代理地址

type Queue

type Queue struct {
	// contains filtered or unexported fields
}

队列

func (*Queue) Add

func (q *Queue) Add(task *Task) error

Add 向队列中添加元素

func (*Queue) Clear

func (q *Queue) Clear() bool

func (*Queue) IsEmpty

func (q *Queue) IsEmpty() bool

func (*Queue) Poll

func (q *Queue) Poll() *Task

Poll 移除队列中最前面的额元素

func (*Queue) Print

func (q *Queue) Print()

func (*Queue) Size

func (q *Queue) Size() int

type Rds

type Rds struct {
	SSHUser       string
	SSHPassword   string
	SSHAddr       string
	RedisHost     string
	RedisPost     string
	RedisPassword string

	// redis DB
	RedisDB int

	// 单个连接
	Conn redis.Conn

	//	最大闲置数，用于redis连接池
	RedisMaxIdle int

	//	最大连接数
	RedisMaxActive int

	//	单条连接Timeout
	RedisIdleTimeoutSec int

	// 连接池
	Pool *redis.Pool
}

func NewRedis

func NewRedis(host, port, password string, db int, vs ...interface{}) *Rds

func NewRedisPool

func NewRedisPool(host, port, password string, db, maxIdle, maxActive, idleTimeoutSec int, vs ...interface{}) *Rds

func (*Rds) GetConn

func (r *Rds) GetConn() redis.Conn

func (*Rds) RedisConn

func (r *Rds) RedisConn() (err error)

redis连接

func (*Rds) RedisPool

func (r *Rds) RedisPool() error

RPool 连接池连接 返回redis连接池 *redis.Pool.Get() 获取redis连接

func (*Rds) SelectDB

func (r *Rds) SelectDB(dbNumber int) error

type ReqTimeOut

type ReqTimeOut int

type ReqTimeOutMs

type ReqTimeOutMs int

type ReqUrl

type ReqUrl struct {
	Url    string
	Method string
	Params map[string]interface{}
}

单个请求地址对象

type RetryFunc

type RetryFunc func(c *Context)

重试前的方法类型

type RetryTimes

type RetryTimes int

重试次数

type SSHConnInfo

type SSHConnInfo struct {
	SSHUser     string
	SSHPassword string
	SSHAddr     string
}

func NewSSHInfo

func NewSSHInfo(addr, user, password string) *SSHConnInfo

type Set

type Set map[string]struct{}

======== Set 可以用于去重

func (Set) Add

func (s Set) Add(key string)

func (Set) Delete

func (s Set) Delete(key string)

func (Set) Has

func (s Set) Has(key string) bool

type Stack

type Stack struct {
	// contains filtered or unexported fields
}

======== Stack

func New

func New() *Stack

func (*Stack) Pop

func (s *Stack) Pop()

func (*Stack) Push

func (s *Stack) Push(data interface{})

func (*Stack) String

func (s *Stack) String() string

type StartFunc

type StartFunc func(c *Context)

请求开始前的方法类型

type StressUrl

type StressUrl struct {
	Url    string
	Method string
	Sum    int64
	Total  int
	TQueue TodoQueue

	// 接口传入的json
	JsonData string

	// 接口传入类型
	ContentType string
	// contains filtered or unexported fields
}

StressUrl 压力测试一个url

func NewTestUrl

func NewTestUrl(url, method string, sum int64, total int) *StressUrl

NewTestUrl 实例化一个新的url压测

func (*StressUrl) Run

func (s *StressUrl) Run(vs ...interface{})

Run 运行压测

func (*StressUrl) SetJson

func (s *StressUrl) SetJson(str string)

type SucceedFunc

type SucceedFunc func(c *Context)

成功后的方法类型

type TableInfo

type TableInfo struct {
	Field   string
	Type    string
	Null    string
	Key     string
	Default interface{}
	Extra   string
}

表信息

type Task

type Task struct {
	Url       string
	JsonParam string
	HeaderMap *http.Header
	// 上下文传递的数据
	Data     map[string]interface{}
	Urls     []*ReqUrl // 多步骤使用
	Type     string    // "", "upload", "do"
	SavePath string
	SaveDir  string
	FileName string
	// contains filtered or unexported fields
}

任务对象

func CrawlerTask

func CrawlerTask(url, jsonParam string, vs ...interface{}) *Task

CrawlerTask

func (Task) AddData

func (task Task) AddData(key string, value interface{}) Task

func (Task) GetDataStr

func (task Task) GetDataStr(key string) string

type TodoQueue

type TodoQueue interface {
	Add(task *Task) error //向队列中添加元素
	Poll() *Task          //移除队列中最前面的元素
	Clear() bool          //清空队列
	Size() int            //获取队列的元素个数
	IsEmpty() bool        //判断队列是否是空
	Print()               // 打印
}

func NewQueue

func NewQueue() TodoQueue

NewQueue 新建一个队列

func NewUploadQueue

func NewUploadQueue() TodoQueue

NewQueue 新建一个下载队列

type UploadQueue

type UploadQueue struct {
	// contains filtered or unexported fields
}

下载队列

func (*UploadQueue) Add

func (q *UploadQueue) Add(task *Task) error

Add 向队列中添加元素

func (*UploadQueue) Clear

func (q *UploadQueue) Clear() bool

func (*UploadQueue) IsEmpty

func (q *UploadQueue) IsEmpty() bool

func (*UploadQueue) Poll

func (q *UploadQueue) Poll() *Task

Poll 移除队列中最前面的额元素

func (*UploadQueue) Print

func (q *UploadQueue) Print()

func (*UploadQueue) Size

func (q *UploadQueue) Size() int

type UserAgentType

type UserAgentType int

const (
	PCAgent UserAgentType = iota + 1
	WindowsAgent
	LinuxAgent
	MacAgent
	AndroidAgent
	IosAgent
	PhoneAgent
	WindowsPhoneAgent
	UCAgent
)

type WSClient

type WSClient interface {
	Send(body []byte) error
	Read(data []byte) error
	Close()
}

func WsClient

func WsClient(host, path string) (WSClient, error)