gathertool

package module
v0.0.9 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 27, 2021 License: MIT Imports: 15 Imported by: 9

README

gathertool

轻量级爬虫，接口测试，压力测试框架, 提高开发对应场景的golang程序。

概念

  1. 一个请求含有 请求成功后方法， 请求重试前方法，请求失败后方法；举个例子 请一个URL 请求遇到状态200或302等算成功，则执行成功后方法。 请求遇到403或502 等需要执行重试，则执行重试方法，重试方法主要含添加等待时间更换代理IP等。 遇到404 或500等算失败则执行失败方法。

  2. 并发执行，第一需要创建TODO队列，等TODO队列加载完后，每个队列对象含有上下文， 在创建时应该富裕上文数据或对象，开始执行并发任务，每个并发任务是一个独立的cilet， 当队列任务取完后则整个并发结束。注意这里的每个并发任务都是独立的，没有chan操作。

请求

Get

简单的get请求实例, 写法一： 方法做为请求函数的参数；

func SimpleGet1(){
	// 创建请求
	c, err := gt.Get("http://192.168.0.1",
		//设置请求成功后的方法： 请求的数据
		gt.SucceedFunc(func(ctx *gt.Context){
			log.Println(string(ctx.RespBody))
		}),

		//设置请求失败后的方法： 打印失败信息
		gt.FailedFunc(func(ctx *gt.Context){
			log.Println(ctx.Err)
		}),

		//设置重试前的方法（遇到403,502 等状态码会重试）： 睡眠1s再重试
		gt.RetryFunc(func(ctx *gt.Context){
			time.Sleep(1*time.Second)
		}),
	)
	if err != nil {
		log.Println("请求创建失败: ", err)
		return
	}
	// 执行创建的请求
	c.Do()
}

最简单的get请求实例， 写法二： 上下文处理；

  func SimpleGet2(){
  	// 创建请求
  	c, err := gt.Get("http://192.168.0.1")
  	if err != nil {
  		log.Println("请求创建失败: ", err)
  		return
  	}
  	// 执行创建的请求
  	c.Do()
  	// 打印请求结果与请求错误
  	log.Println(string(c.RespBody), c.Err)
  }

简单的get请求实例, 写法三： 给请求设置方法；

func SimpleGet3()  {
	// 创建请求
	c, err := gt.Get("http://192.168.0.1")
	if err != nil {
		log.Println("请求创建失败: ", err)
		return
	}
	//设置请求成功后的方法
	c.SetSucceedFunc(func(ctx *gt.Context){
		log.Println(string(ctx.RespBody))
	})
	//设置请求失败后的方法
	c.SetFailedFunc(func(ctx *gt.Context){
		log.Println(ctx.Err)
	})
	//设置重试次数
	c.SetRetryTimes(5)
	//设置重试前的方法
	c.SetRetryFunc(func(*gt.Context) {
		time.Sleep(1*time.Second)
	})
	// 执行创建的请求
	c.Do()
}

简单的get请求实例, 写法四： 外部函数为请求方法；

func SimpleGet4(){
	c, err := gt.Get("http://192.168.0.1",
		gt.SucceedFunc(succeed),
		gt.FailedFunc(fail),
		gt.RetryFunc(retry),
		)
	if err != nil {
		log.Println("请求创建失败: ", err)
		return
	}
	// 执行创建的请求
	c.Do()
}

// 成功后的方法
func succeed(ctx *gt.Context){
	log.Println(string(ctx.RespBody))
	//处理数据
}

// 设置需要重试状态码， 重试前的方法
func retry(ctx *gt.Context){
	ctx.Client = &http.Client{
		Timeout: 1*time.Second,
	}
	log.Println("休息1s")
	time.Sleep(1*time.Second)
}

// 失败后的方法
func fail(ctx *gt.Context) {
	log.Println("请求失败: ", ctx.Err)
}

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var StatusCodeMap map[int]string = map[int]string{
	200: "success",
	201: "success",
	202: "success",
	203: "success",
	204: "fail",
	300: "success",
	301: "success",
	302: "success",
	400: "fail",
	401: "retry",
	402: "retry",
	403: "retry",
	404: "fail",
	405: "retry",
	406: "retry",
	407: "retry",
	408: "retry",
	500: "fail",
	501: "fail",
	502: "retry",
	503: "retry",
	504: "retry",
}

StatusCodeMap 状态码处理映射 success 该状态码对应执行成功函数 fail 该状态码对应执行失败函数 retry 该状态码对应需要重试前执行的函数

View Source 
var (
	UrlBad error = errors.New("url is bad.") // 错误的url
)
View Source 
var UserAgentMap map[int]string = map[int]string{
	1:  "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0",
	2:  "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36",
	3:  "Mozilla/5.0 (compatible; WOW64; MSIE 10.0; Windows NT 6.2)",
	4:  "Opera/9.80 (Windows NT 6.1; WOW64; U; en) Presto/2.10.229 Version/11.62",
	5:  "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
	6:  "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20130331 Firefox/21.0",
	7:  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Ubuntu/11.10 Chromium/27.0.1453.93 Chrome/27.0.1453.93 Safari/537.36",
	8:  "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
	9:  "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.9.168 Version/11.52",
	10: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
	11: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:21.0) Gecko/20100101 Firefox/21.0",
	12: "Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166  Safari/535.19",
	13: "Mozilla/5.0 (Linux; U; Android 4.0.4; en-gb; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
	14: "Mozilla/5.0 (Linux; U; Android 2.2; en-gb; GT-P1000 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
	15: "Mozilla/5.0 (Android; Mobile; rv:14.0) Gecko/14.0 Firefox/14.0",
	16: "Mozilla/5.0 (Android; Tablet; rv:14.0) Gecko/14.0 Firefox/14.0",
	17: "Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19",
	18: "Mozilla/5.0 (Linux; Android 4.1.2; Nexus 7 Build/JZ054K) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19",
	19: "Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
	20: "Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
	21: "Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
	22: "Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
	23: "Mozilla/5.0 (iPhone; CPU iPhone OS 6_1_4 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) CriOS/27.0.1453.10 Mobile/10B350 Safari/8536.25",
	24: "Mozilla/5.0 (compatible; MSIE 10.0; Windows Phone 8.0; Trident/6.0; IEMobile/10.0; ARM; Touch; NOKIA; Lumia 920)",
	25: "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; SAMSUNG; SGH-i917)",
	26: "User-Agent, UCWEB7.0.2.37/28/999",
	27: "User-Agent, NOKIA5700/ UCWEB7.0.2.37/28/999",
	28: "User-Agent, Openwave/ UCWEB7.0.2.37/28/999",
	29: "User-Agent, Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999",
	30: "Mozilla/5.0 (Windows; U; Windows NT 6.1; ) AppleWebKit/534.12 (KHTML, like Gecko) Maxthon/3.0 Safari/534.12",
	31: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E)",
	32: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)",
	33: "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.33 Safari/534.3 SE 2.X MetaSr 1.0",
	34: "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E)",
	35: "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1 QQBrowser/6.9.11079.201",
}

Functions

func GetAgent added in v0.0.4 

func GetAgent(agentType UserAgentType) string

GetAgent 随机获取那种类型的 user-agent

func NewGoquery added in v0.0.4 

func NewGoquery(html string) (*goquery.Document, error)

func StartJob added in v0.0.7 

func StartJob()

TODO: StartJob 开始运行并发

func StartJobGet added in v0.0.7 

func StartJobGet(jobNumber int, queue TodoQueue, vs ...interface{})

StartJobGet 并发执行Get,直到队列任务为空 @jobNumber 并发数， @queue 全局队列， @client 单个并发任务的client， @SucceedFunc 成功方法， @ RetryFunc重试方法， @FailedFunc 失败方法

func StartJobPost added in v0.0.7 

func StartJobPost()

TODO: StartJobPost 开始运行并发Post

func StringValue added in v0.0.9 

func StringValue(i interface{}) string

StringValue 任何类型返回值字符串形式

Types

type Context added in v0.0.7 

type Context struct {
	// Token
	Token string

	// client
	Client *http.Client

	// Request
	Req *http.Request

	// Response
	Resp *http.Response

	// Error
	Err error

	// 最大允许重试次数
	MaxTimes RetryTimes

	// 请求成功了需要处理的事件
	SucceedFunc SucceedFunc

	// 请求失败了需要做的事
	FailedFunc FailedFunc

	// 请求状态码设置了重试，在重试前的事件
	RetryFunc RetryFunc

	// 请求开始前的方法
	StartFunc StartFunc

	// 请求完成后的方法
	EndFunc EndFunc

	// 本次请求的任务
	Task *Task

	RespBody []byte

	// job 编号
	JobNumber int

	// 请求的响应时间 单位ms
	Ms time.Duration
	// contains filtered or unexported fields
}

请求上下文

func Get 

func Get(url string, vs ...interface{}) (*Context, error)

Get 请求, 当请求失败或状态码是失败的则会先执行 ff 再回调

func Post added in v0.0.7 

func Post(url string, data []byte, contentType string, vs ...interface{}) (*Context, error)

POST 请求

func PostJson added in v0.0.7 

func PostJson(url string, jsonStr string, vs ...interface{}) (*Context, error)

POST json 请求

func Req 

func Req(request *http.Request, vs ...interface{}) (*Context, error)

Req 初始化请求 @url 请求链接 @maxTimes 重试次数 @sf 请求成功后做的事情, 200等 @ff 请求失败后做的事情, 403等，502等 @vs 可变参数 @vs UserAgentType 设置指定类型 user agent 如 AndroidAgent

func Request added in v0.0.7 

func Request(url, method string, data []byte, contentType string, vs ...interface{}) (*Context, error)

Request 请求

func (*Context) Do added in v0.0.7 

func (c *Context) Do() func()

Do 执行请求

func (*Context) SetFailedFunc added in v0.0.7 

func (c *Context) SetFailedFunc(failedFunc func(c *Context))

SetFailed 设置错误后的方法

func (*Context) SetRetryFunc added in v0.0.7 

func (c *Context) SetRetryFunc(retryFunc func(c *Context))

SetRetryFunc 设置重试，在重试前的方法

func (*Context) SetRetryTimes added in v0.0.7 

func (c *Context) SetRetryTimes(times int)

SetRetryTimes 设置重试次数

func (*Context) SetSucceedFunc added in v0.0.7 

func (c *Context) SetSucceedFunc(successFunc func(c *Context))

SetSucceedFunc 设置成功后的方法

type EndFunc added in v0.0.7 

type EndFunc func(c *Context)

请求结束后的方法

type FailedFunc added in v0.0.7 

type FailedFunc func(c *Context)

失败后的方法

type Mysql added in v0.0.9 

type Mysql struct {
	Host        string
	Port        int
	User        string
	Password    string
	DataBase    string
	MaxOpenConn int
	MaxIdleConn int
	DB          *sql.DB
}

func NewMysql added in v0.0.9 

func NewMysql(host string, port int, user, password, database string) (*Mysql, error)

func (*Mysql) Conn added in v0.0.9 

func (m *Mysql) Conn() (err error)

连接mysql

func (*Mysql) Describe added in v0.0.9 

func (m *Mysql) Describe(table string) (map[string]string, error)

Describe 获取表结构

func (*Mysql) Insert added in v0.0.9 

func (m *Mysql) Insert(table string, fieldData map[string]interface{}) error

Insert 新增数据

func (*Mysql) NewTable added in v0.0.9 

func (m *Mysql) NewTable(table string, fields map[string]string) error

NewTable 创建表

func (*Mysql) Select added in v0.0.9 

func (m *Mysql) Select(sql string) ([]map[string]string, error)

Select 查询语句 返回 map

type Queue added in v0.0.4 

type Queue struct {
	// contains filtered or unexported fields
}

队列

func (*Queue) Add added in v0.0.4 

func (q *Queue) Add(task *Task)

Add 向队列中添加元素

func (*Queue) Clear added in v0.0.4 

func (q *Queue) Clear() bool

func (*Queue) IsEmpty added in v0.0.4 

func (q *Queue) IsEmpty() bool

func (*Queue) Poll added in v0.0.4 

func (q *Queue) Poll() *Task

Poll 移除队列中最前面的额元素

func (*Queue) Print added in v0.0.4 

func (q *Queue) Print()

func (*Queue) Size added in v0.0.4 

func (q *Queue) Size() int

type ReqTimeOut added in v0.0.7 

type ReqTimeOut int

type ReqTimeOutMs added in v0.0.7 

type ReqTimeOutMs int

type ReqUrl added in v0.0.7 

type ReqUrl struct {
	Url    string
	Method string
	Params map[string]interface{}
}

单个请求地址对象

type RetryFunc added in v0.0.7 

type RetryFunc func(c *Context)

重试前的方法

type RetryTimes added in v0.0.4 

type RetryTimes int

重试次数

type StartFunc added in v0.0.7 

type StartFunc func(c *Context)

请求开始前的方法

type StressUrl added in v0.0.7 

type StressUrl struct {
	Url    string
	Method string
	Sum    int64
	Total  int
	TQueue TodoQueue
	// contains filtered or unexported fields
}

StressUrl 压力测试一个url

func NewTestUrl added in v0.0.7 

func NewTestUrl(url, method string, sum int64, total int) *StressUrl

NewTestUrl 实例化一个新的url压测

func (*StressUrl) Run added in v0.0.7 

func (s *StressUrl) Run(vs ...interface{})

Run 运行压测

type SucceedFunc added in v0.0.7 

type SucceedFunc func(c *Context)

成功后的方法

type TableInfo added in v0.0.9 

type TableInfo struct {
	Field   string
	Type    string
	Null    string
	Key     string
	Default interface{}
	Extra   string
}

表信息

type Task added in v0.0.5 

type Task struct {
	Url     string
	Context map[string]interface{}
	Urls    []*ReqUrl // 多步骤使用
}

任务对象

type TodoQueue added in v0.0.4 

type TodoQueue interface {
	Add(task *Task) //向队列中添加元素
	Poll() *Task    //移除队列中最前面的元素
	Clear() bool    //清空队列
	Size() int      //获取队列的元素个数
	IsEmpty() bool  //判断队列是否是空
	Print()         // 打印
}

func NewQueue added in v0.0.4 

func NewQueue() TodoQueue

NewQueue 新建一个队列

type UserAgentType added in v0.0.4 

type UserAgentType int
const (
	PCAgent UserAgentType = iota + 1
	WindowsAgent
	LinuxAgent
	MacAgent
	AndroidAgent
	IosAgent
	PhoneAgent
	WindowsPhoneAgent
	UCAgent
)

Source Files

View all Source files

Directories

Path Synopsis
examples
get
ip_bczs_cn
stress_testing

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.