requests

package module
v0.0.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 12, 2024 License: Apache-2.0 Imports: 26 Imported by: 0

README

requests

ja3指纹,http2 指纹,主流代理协议,类型自动转换,覆盖python requests 所有的功能

功能概述

  • 基于net/http 二次封装
  • cookies 开关，连接池开关，http2，ja3
  • 自实现socks五,http代理,https代理
  • 自动解压缩,解码
  • dns缓存
  • 类型自动转化
  • 尝试重试，请求回调

发送http请求

package main

import (
    "log"

    "github.com/xmapst/requests"
)

func main() {
    reqCli, err := requests.NewClient() //创建请求客户端
    if err != nil {
        log.Panic(err)
    }
    response, err := reqCli.Get("http://myip.top") //发送get请求
    if err != nil {
        log.Panic(err)
    }
    log.Print(response.Text())    //获取内容,解析为字符串
    log.Print(response.Content()) //获取内容,解析为字节
    log.Print(response.Json())    //获取json,解析为gjson
    log.Print(response.Html())    //获取内容,解析为html
    log.Print(response.Cookies()) //获取cookies
}

发送websocket 请求

package main

import (
	"context"
	"log"

	"github.com/xmapst/requests"
	"github.com/xmapst/requests/websocket"
)

func main() {
	reqCli, err := requests.NewClient() //创建请求客户端
	if err != nil {
		log.Panic(err)
	}
	response, err := reqCli.Get("ws://82.157.123.54:9010/ajaxchattest", requests.RequestOption{Headers: map[string]string{
		"Origin": "http://coolaf.com",
	}}) //发送websocket请求
	if err != nil {
		log.Panic(err)
	}
	defer response.Close()
	wsCli := response.WebSocket()
	if err = wsCli.Send(context.TODO(), websocket.MessageText, []byte("测试")); err != nil { //发送txt 消息
		log.Panic(err)
	}
	msgType, con, err := wsCli.Recv(context.TODO()) //接收消息
	if err != nil {
		log.Panic(err)
	}
	log.Print(msgType)     //消息类型
	log.Print(string(con)) //消息内容
}

ipv4,ipv6 地址控制解析

func main() {
	reqCli, err := requests.NewClient(requests.ClientOption{
		AddrType: requests.Ipv4, //优先解析ipv4地址
		// AddrType: requests.Ipv6,//优先解析ipv6地址
	})
	if err != nil {
		log.Panic(err)
	}
	href := "https://test.ipw.cn"
	resp, err := reqCli.Get(href)
	if err != nil {
		log.Panic(err)
	}
	log.Print(resp.Text())
	log.Print(resp.StatusCode())
}

ja3 伪造指纹

根据字符串生成指纹

func main() {
	reqCli, err := requests.NewClient()
	if err != nil {
		log.Panic(err)
	}
	ja3Str := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0"
	Ja3Spec, err := ja3.CreateSpecWithStr(ja3Str)//根据字符串生成指纹
	if err != nil {
		log.Panic(err)
	}
	response, err := reqCli.Get("https://tools.scrapfly.io/api/fp/ja3?extended=1", requests.RequestOption{Ja3Spec: Ja3Spec})
	if err != nil {
		log.Panic(err)
	}
	log.Print(response.Json().Get("ja3").String())
	log.Print(response.Json().Get("ja3").String() == ja3Str)
}

根据id 生成指纹

func main() {
	reqCli, err := requests.NewClient()
	if err != nil {
		log.Panic(err)
	}
	Ja3Spec, err := ja3.CreateSpecWithId(ja3.HelloChrome_Auto)//根据id 生成指纹
	if err != nil {
		log.Panic(err)
	}
	response, err := reqCli.Get("https://tools.scrapfly.io/api/fp/ja3?extended=1", requests.RequestOption{Ja3Spec: Ja3Spec})
	if err != nil {
		log.Panic(err)
	}
	log.Print(response.Json().Get("ja3").String())
}

ja3 开关

func main() {
	reqCli, err := requests.NewClient()
	if err != nil {
		log.Panic(err)
	}
	response, err := reqCli.Get("https://tools.scrapfly.io/api/fp/ja3?extended=1", requests.RequestOption{Ja3: true})//使用最新chrome 指纹
	if err != nil {
		log.Panic(err)
	}
	log.Print(response.Json().Get("ja3").String())
}

h2 指纹开关

func main() {
	reqCli, err := requests.NewClient(requests.ClientOption{
		H2Ja3: true,
	})
	if err != nil {
		log.Panic(err)
	}
	href := "https://tools.scrapfly.io/api/fp/anything"
	resp, err := reqCli.Get(href)
	if err != nil {
		log.Panic(err)
	}
	log.Print(resp.Text())
}

修改h2指纹

func main() {
	reqCli, err := requests.NewClient(requests.ClientOption{
		H2Ja3Spec: ja3.H2Ja3Spec{
			InitialSetting: []ja3.Setting{
				{Id: 1, Val: 65555},
				{Id: 2, Val: 1},
				{Id: 3, Val: 2000},
				{Id: 4, Val: 6291457},
				{Id: 6, Val: 262145},
			},
			ConnFlow: 15663106,
			OrderHeaders: []string{
				":method",
				":path",
				":scheme",
				":authority",
			},
		},
	})
	if err != nil {
		log.Panic(err)
	}
	href := "https://tools.scrapfly.io/api/fp/anything"
	resp, err := reqCli.Get(href)
	if err != nil {
		log.Panic(err)
	}
	log.Print(resp.Text())
}

采集全国公共资源网和中国政府采购网的列表页的标题

package main
import (
	"log"
	
	"github.com/xmapst/requests"
)
func main() {
	reqCli, err := requests.NewClient()
	if err != nil {
		log.Panic(err)
	}
	resp, err := reqCli.Get("http://www.ccgp.gov.cn/cggg/zygg/")
	if err != nil {
		log.Panic(err)
	}
	html := resp.Html()
	lis := html.Finds("ul.c_list_bid li")
	for _, li := range lis {
		title := li.Find("a").Get("title")
		log.Print(title)
	}
	resp, err = reqCli.GET("http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp", requests.RequestOption{
		Data: map[string]string{
			"TIMEBEGIN_SHOW": "2023-04-26",
			"TIMEEND_SHOW":   "2023-05-05",
			"TIMEBEGIN":      "2023-04-26",
			"TIMEEND":        "2023-05-05",
			"SOURCE_TYPE":    "1",
			"DEAL_TIME":      "02",
			"DEAL_CLASSIFY":  "01",
			"DEAL_STAGE":     "0100",
			"DEAL_PROVINCE":  "0",
			"DEAL_CITY":      "0",
			"DEAL_PLATFORM":  "0",
			"BID_PLATFORM":   "0",
			"DEAL_TRADE":     "0",
			"isShowAll":      "1",
			"PAGENUMBER":     "2",
			"FINDTXT":        "",
		},
	})
	if err != nil {
		log.Panic(err)
	}
	jsonData := resp.Json()
	lls := jsonData.Get("data").Array()
	for _, ll := range lls {
		log.Print(ll.Get("title"))
	}
}

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var AcceptLanguage = `"zh-CN,zh;q=0.9"`
View Source 
var (
	ErrFatal = errors.New("致命错误")
)
View Source 
var UserAgent = `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36`

Functions

This section is empty.

Types

type AddrType 

type AddrType int
const (
	Auto AddrType = 0
	Ipv4 AddrType = 4
	Ipv6 AddrType = 6
)

type Client 

type Client struct {
	RedirectNum   int                                         // 重定向次数
	DisDecode     bool                                        // 关闭自动编码
	DisRead       bool                                        // 关闭默认读取请求体
	DisUnZip      bool                                        // 变比自动解压
	TryNum        int64                                       // 重试次数
	BeforCallBack func(context.Context, *RequestOption) error // 请求前回调的方法
	AfterCallBack func(context.Context, *Response) error      // 请求后回调的方法
	ErrCallBack   func(context.Context, error) bool           // 请求error回调
	Timeout       int64                                       // 请求超时时间
	Headers       any                                         // 请求头
	// contains filtered or unexported fields
}

func NewClient 

func NewClient(clientOptinos ...ClientOption) (*Client, error)

NewClient 新建一个请求客户端,发送请求必须创建哈

func NewClientWithContext 

func NewClientWithContext(preCtx context.Context, clientOptinos ...ClientOption) (*Client, error)

NewClientWithContext 新建一个请求客户端,发送请求必须创建哈

func (*Client) ClearCookies 

func (obj *Client) ClearCookies() error

ClearCookies 清除cookies

func (*Client) Clone 

func (obj *Client) Clone() *Client

Clone 克隆请求客户端,返回一个由相同option 构造的客户端，但是不会克隆:连接池,ookies

func (*Client) Close 

func (obj *Client) Close()

Close 关闭客户端

func (*Client) CloseIdleConnections 

func (obj *Client) CloseIdleConnections()

CloseIdleConnections 关闭客户端中的空闲连接

func (*Client) Closed 

func (obj *Client) Closed() bool

func (*Client) Cookies 

func (obj *Client) Cookies(href string, cookies ...*http.Cookie) Cookies

Cookies 返回url 的cookies,也可以设置url 的cookies

func (*Client) Get added in v0.0.2 

func (obj *Client) Get(href string, options ...RequestOption) (resp *Response, err error)

Get 发送GET请求

func (*Client) Request 

func (obj *Client) Request(method string, href string, options ...RequestOption) (resp *Response, err error)

Request 发送请求

func (*Client) RequestWithContext 

func (obj *Client) RequestWithContext(preCtx context.Context, method string, href string, options ...RequestOption) (resp *Response, err error)

func (*Client) Reset 

func (obj *Client) Reset() error

Reset 重置客户端至初始状态

type ClientOption 

type ClientOption struct {
	GetProxy              func(ctx context.Context, url *url.URL) (string, error) // 根据url 返回代理，支持https,http,socks5 代理协议
	Proxy                 string                                                  // 设置代理,支持https,http,socks5 代理协议
	TLSHandshakeTimeout   time.Duration                                           // tls 超时时间,default:15
	ResponseHeaderTimeout int64                                                   // 第一个response headers 接收超时时间,default:30
	DisCookie             bool                                                    // 关闭cookies管理
	DisAlive              bool                                                    // 关闭长连接
	DisCompression        bool                                                    // 关闭请求头中的压缩功能
	LocalAddr             string                                                  // 本地网卡出口ip
	IdleConnTimeout       int64                                                   // 空闲连接在连接池中的超时时间,default:30
	KeepAlive             int64                                                   // keepalive保活检测定时,default:15
	DnsCacheTime          int64                                                   // dns解析缓存时间60*30
	DisDnsCache           bool                                                    // 是否关闭dns 缓存,影响dns 解析
	AddrType              AddrType                                                // 优先使用的addr 类型
	GetAddrType           func(string) AddrType                                   // 地址类型
	Dns                   string                                                  // dns
	Ja3                   bool                                                    // 开启ja3
	Ja3Spec               ja3.ClientHelloSpec                                     // 指定ja3Spec,使用ja3.CreateSpecWithStr 或者ja3.CreateSpecWithId 生成
	H2Ja3                 bool                                                    // 开启h2指纹
	H2Ja3Spec             ja3.H2Ja3Spec                                           // h2指纹
}

type Cookies 

type Cookies []*http.Cookie

func ReadCookies 

func ReadCookies(val any) Cookies

支持json,map,[]string,http.Header,string

func ReadSetCookies 

func ReadSetCookies(val any) Cookies

func (Cookies) Get 

func (obj Cookies) Get(name string) *http.Cookie

Get 获取符合key 条件的cookies

func (Cookies) GetVal 

func (obj Cookies) GetVal(name string) string

GetVal 获取符合key 条件的cookies的值

func (Cookies) GetVals 

func (obj Cookies) GetVals(name string) []string

GetVals 获取符合key 条件的所有cookies的值

func (Cookies) Gets 

func (obj Cookies) Gets(name string) Cookies

Gets 获取符合key 条件的所有cookies

func (Cookies) String 

func (obj Cookies) String() string

返回cookies 的字符串形式

type DialClient 

type DialClient struct {
	// contains filtered or unexported fields
}

func (*DialClient) AddrToIp 

func (obj *DialClient) AddrToIp(ctx context.Context, addr string) (string, error)

func (*DialClient) DialContext 

func (obj *DialClient) DialContext(ctx context.Context, netword string, addr string) (net.Conn, error)

func (*DialClient) DialContextForProxy 

func (obj *DialClient) DialContextForProxy(ctx context.Context, netword string, scheme string, addr string, host string, proxyUrl *url.URL) (net.Conn, error)

func (*DialClient) DialTlsProxyContext 

func (obj *DialClient) DialTlsProxyContext(ctx context.Context, netword string, addr string, host string) (net.Conn, error)

func (*DialClient) DnsDialContext 

func (obj *DialClient) DnsDialContext(ctx context.Context, netword string, addr string) (net.Conn, error)

func (*DialClient) GetProxy 

func (obj *DialClient) GetProxy(ctx context.Context, url *url.URL) (*url.URL, error)

func (*DialClient) Http2HttpProxy 

func (obj *DialClient) Http2HttpProxy(ctx context.Context, network string, proxyUrl *url.URL) (conn net.Conn, err error)

func (*DialClient) Http2HttpsProxy 

func (obj *DialClient) Http2HttpsProxy(ctx context.Context, network string, addr string, host string, proxyUrl *url.URL) (conn net.Conn, err error)

func (*DialClient) Http2Socks5Proxy 

func (obj *DialClient) Http2Socks5Proxy(ctx context.Context, network string, addr string, proxyUrl *url.URL) (conn net.Conn, err error)

func (*DialClient) Https2HttpProxy 

func (obj *DialClient) Https2HttpProxy(ctx context.Context, network string, addr string, host string, proxyUrl *url.URL) (conn net.Conn, err error)

func (*DialClient) Https2HttpsProxy 

func (obj *DialClient) Https2HttpsProxy(ctx context.Context, network string, addr string, host string, proxyUrl *url.URL) (conn net.Conn, err error)

func (*DialClient) Https2Socks5Proxy 

func (obj *DialClient) Https2Socks5Proxy(ctx context.Context, network string, addr string, host string, proxyUrl *url.URL) (conn net.Conn, err error)

type DialOption 

type DialOption struct {
	TLSHandshakeTimeout time.Duration
	DnsCacheTime        int64
	KeepAlive           int64
	GetProxy            func(ctx context.Context, url *url.URL) (string, error)
	Proxy               string   // 代理
	LocalAddr           string   // 使用本地网卡
	AddrType            AddrType // 优先使用的地址类型,ipv4,ipv6 ,或自动选项
	GetAddrType         func(string) AddrType
	Ja3                 bool                // 是否启用ja3
	Ja3Spec             ja3.ClientHelloSpec // 指定ja3Spec,使用ja3.CreateSpecWithStr 或者ja3.CreateSpecWithId 生成
	DisDnsCache         bool                // 是否关闭dns 缓存
	Dns                 string              // dns
}

type Event 

type Event struct {
	Data    string
	Event   string
	Id      string
	Retry   int
	Comment string
}

type File 

type File struct {
	Key     string // 字段的key
	Name    string // 文件名
	Content []byte // 文件的内容
	Type    string // 文件类型
}

File 构造一个文件

type RequestOption 

type RequestOption struct {
	Method  string   // method
	Url     *url.URL // 请求的url
	Host    string   // 网站的host
	Proxy   string   // 代理,支持http,https,socks5协议代理,例如：http://127.0.0.1:7005
	Timeout int64    // 请求超时时间
	Headers any      // 请求头,支持：json,map，header
	Cookies any      // cookies,支持json,map,str
	Files   []File   // 文件
	Params  any      // url 中的参数，用以拼接url,支持json,map
	Form    any      // 发送multipart/form-data,适用于文件上传,支持json,map
	Data    any      // 发送application/x-www-form-urlencoded,适用于key,val,支持string,[]bytes,json,map

	Body          io.Reader
	Json          any                                         // 发送application/json,支持：string,[]bytes,json,map
	Text          any                                         // 发送text/xml,支持string,[]bytes,json,map
	Raw           any                                         // 不设置context-type,支持string,[]bytes,json,map
	TempData      any                                         // 临时变量，用于回调存储或自由度更高的用法
	DisCookie     bool                                        // 关闭cookies管理,这个请求不用cookies池
	DisDecode     bool                                        // 关闭自动解码
	DisProxy      bool                                        // 是否关闭代理,强制关闭代理
	Ja3           bool                                        // 是否开启ja3
	Ja3Spec       ja3.ClientHelloSpec                         // 指定ja3Spec,使用ja3.CreateSpecWithStr 或者ja3.CreateSpecWithId 生成
	TryNum        int64                                       // 重试次数
	BeforCallBack func(context.Context, *RequestOption) error // 请求之前回调
	AfterCallBack func(context.Context, *Response) error      // 请求之后回调
	ErrCallBack   func(context.Context, error) bool           // 返回true 中断重试请求
	RedirectNum   int                                         // 重定向次数,小于零 关闭重定向
	DisAlive      bool                                        // 关闭长连接,这次请求不会复用之前的连接
	DisRead       bool                                        // 关闭默认读取请求体,不会主动读取body里面的内容，需用你自己读取
	DisUnZip      bool                                        // 关闭自动解压
	WsOption      websocket.Option                            // websocket option,使用websocket 请求的option
	// contains filtered or unexported fields
}

RequestOption 请求参数选项

type Response 

type Response struct {
	// contains filtered or unexported fields
}

func Get added in v0.0.2 

func Get(href string, options ...RequestOption) (resp *Response, err error)

Get 发送GET请求

func Request added in v0.0.2 

func Request(method string, href string, options ...RequestOption) (resp *Response, err error)

Request 发送请求

func RequestWithContext added in v0.0.2 

func RequestWithContext(preCtx context.Context, method string, href string, options ...RequestOption) (resp *Response, err error)

func (*Response) Close 

func (obj *Response) Close() error

Close 关闭response ,当disRead 为true 请一定要手动关闭

func (*Response) Content 

func (obj *Response) Content(val ...[]byte) []byte

Content 返回内容的二进制，也可设置内容

func (*Response) ContentEncoding 

func (obj *Response) ContentEncoding() string

ContentEncoding 获取headers 的Content-Encoding

func (*Response) ContentLength 

func (obj *Response) ContentLength() int64

ContentLength 获取response 的内容长度

func (*Response) ContentType 

func (obj *Response) ContentType() string

ContentType 获取headers 的Content-Type

func (*Response) Cookies 

func (obj *Response) Cookies() Cookies

Cookies 返回这个请求的cookies

func (*Response) Decode 

func (obj *Response) Decode(encoding string)

Decode 对内容进行解码

func (*Response) Headers 

func (obj *Response) Headers() http.Header

Headers 返回response 的请求头

func (*Response) Json 

func (obj *Response) Json(path ...string) gjson.Result

Json 尝试将请求解析成json

func (*Response) Location 

func (obj *Response) Location() (*url.URL, error)

Location 返回当前的Location

func (*Response) Map 

func (obj *Response) Map() map[string]any

Map 尝试将内容解析成map

func (*Response) Read 

func (obj *Response) Read(con []byte) (int, error)

func (*Response) Response 

func (obj *Response) Response() *http.Response

Response 返回原始http.Response

func (*Response) SseClient 

func (obj *Response) SseClient() *SseClient

func (*Response) Status 

func (obj *Response) Status() string

Status 返回这个请求的状态

func (*Response) StatusCode 

func (obj *Response) StatusCode() int

StatusCode 返回这个请求的状态码

func (*Response) Text 

func (obj *Response) Text(val ...string) string

Text 返回内容的字符串形式，也可设置内容

func (*Response) Url 

func (obj *Response) Url() *url.URL

Url 返回这个请求的url

func (*Response) WebSocket 

func (obj *Response) WebSocket() *websocket.Conn

WebSocket 返回websocket 对象,当发送websocket 请求时使用

type SseClient 

type SseClient struct {
	// contains filtered or unexported fields
}

func (*SseClient) Recv 

func (obj *SseClient) Recv() (Event, error)

Source Files

View all Source files

Directories

Path Synopsis
Package http2 implements the HTTP/2 protocol.
 Package http2 implements the HTTP/2 protocol.
h2c
Package h2c implements the unencrypted "h2c" form of HTTP/2.
 Package h2c implements the unencrypted "h2c" form of HTTP/2.
h2i
The h2i command is an interactive HTTP/2 console.
 The h2i command is an interactive HTTP/2 console.
hpack
Package hpack implements HPACK, a compression format for efficiently representing HTTP header fields in the context of HTTP/2.
 Package hpack implements HPACK, a compression format for efficiently representing HTTP header fields in the context of HTTP/2.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.