CoreHttp

Documentation

Index

• Variables
• func GetData(getURL string, params url.Values, proxyIP string, isSetHeader bool) ([]byte, error)
• func GetGoquery(getURL string, params url.Values, proxyIP string, isSetHeader bool) (*goquery.Document, error)
• func GetGoqueryStrInHtml(doc *goquery.Document, findStr string) int
• func GetResp(getURL string, params url.Values, proxyIP string, isSetHeader bool) (*http.Response, error)
• func GetURLEncode(sendURL string) string
• func GetUserAgentRand() string
• func HttpGet(getURL string) ([]byte, error)
• func HttpPost(getURL string, params url.Values) ([]byte, error)
• type HttpGetType

Variables

var (
	HTTPGetUserAgents = []string{
		"Mozilla/5.0 (compatible, MSIE 10.0, Windows NT, DigExt)",
		"Mozilla/4.0 (compatible, MSIE 7.0, Windows NT 5.1, 360SE)",
		"Mozilla/4.0 (compatible, MSIE 8.0, Windows NT 6.0, Trident/4.0)",
		"Mozilla/5.0 (compatible, MSIE 9.0, Windows NT 6.1, Trident/5.0,",
		"Opera/9.80 (Windows NT 6.1, U, en) Presto/2.8.131 Version/11.11",
		"Mozilla/4.0 (compatible, MSIE 7.0, Windows NT 5.1, TencentTraveler 4.0)",
		"Mozilla/5.0 (Windows, U, Windows NT 6.1, en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
		"Mozilla/5.0 (Macintosh, Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
		"Mozilla/5.0 (Macintosh, U, Intel Mac OS X 10_6_8, en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
		"Mozilla/5.0 (Linux, U, Android 3.0, en-us, Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
		"Mozilla/5.0 (iPad, U, CPU OS 4_3_3 like Mac OS X, en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
		"Mozilla/4.0 (compatible, MSIE 7.0, Windows NT 5.1, Trident/4.0, SE 2.X MetaSr 1.0, SE 2.X MetaSr 1.0, .NET CLR 2.0.50727, SE 2.X MetaSr 1.0)",
		"Mozilla/5.0 (iPhone, U, CPU iPhone OS 4_3_3 like Mac OS X, en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
		"MQQBrowser/26 Mozilla/5.0 (Linux, U, Android 2.3.7, zh-cn, MB200 Build/GRJ22, CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
	}
)

Functions

func GetData

func GetData(getURL string, params url.Values, proxyIP string, isSetHeader bool) ([]byte, error)

GetData 通过GET或POST获取URL数据 param getURL string 获取URL地址 param params url.Values 表单参数，只有post给定，留空则认定为get模式 param proxyIP string 代理IP地址，如果留空跳过 param isSetHeader bool 是否加入头信息加密，建议爬虫使用 return []byte,error 数据，错误

func GetGoquery

func GetGoquery(getURL string, params url.Values, proxyIP string, isSetHeader bool) (*goquery.Document, error)

GetGoquery 通过goquery获取HTML param getURL string 获取URL地址 param params url.Values POST参数，如果留空则GET param proxyIP string 代理IP地址，如果留空跳过 param isSetHeader bool 是否加入头信息加密，建议爬虫使用 return *goquery.Document , error 文档操作句柄，是否成功

func GetGoqueryStrInHtml

func GetGoqueryStrInHtml(doc *goquery.Document, findStr string) int

GetGoqueryStrInHtml 查询HTML中存在多少个字符串 param sendURL string URL地址 param findStr string 要搜索的字符串 return int 存在多少个，-1为失败，0没找到

func GetResp

func GetResp(getURL string, params url.Values, proxyIP string, isSetHeader bool) (*http.Response, error)

GetResp 获取RESP信息源 注意自行增加关闭机制 param getURL string 获取URL地址 param params url.Values 表单参数，只有post给定，留空则认定为get模式 param proxyIP string 代理IP地址，如果留空跳过 param isSetHeader bool 是否加入头信息加密，建议爬虫使用 return *http.Response,error 数据，错误

func GetURLEncode

func GetURLEncode(sendURL string) string

GetURLEncode URL编码工具

func GetUserAgentRand

func GetUserAgentRand() string

GetUserAgentRand 随机获取一个UserAgent头信息 return string 伪造头信息

func HttpGet

func HttpGet(getURL string) ([]byte, error)

HttpGet 发送请求封装 配置下不需要提供密钥，直接就能获取到配置信息 param getURL string 获取URL return error 是否成功

func HttpPost

func HttpPost(getURL string, params url.Values) ([]byte, error)

HttpPost 发送请求封装 配置下不需要提供密钥，直接就能获取到配置信息 param getURL string 获取URL param params url.Values 参数集合 return error 是否成功

Types

type HttpGetType

type HttpGetType struct {
	//Header头预定义信息
	HTTPGetUserAgents []string
}