requests

package module
v0.0.0-...-b9d8d3e Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 7, 2025 License: GPL-3.0 Imports: 41 Imported by: 0

README

Requests - A next-generation HTTP client for Golang.

A request library designed specifically for web crawlers

Innovative Features

gospider007/requests Other Request Libraries
Unlimited chained proxy Not supported
HTTP/3 fingerprint spoofing protection Not supported
Arbitrary closure of underlying connections Not supported
Genuine request-level proxy settings Not supported
Unique transport layer management mechanism, fully unifying HTTP/1, HTTP/2, HTTP/3, WebSocket, SSE, UTLS, QUIC protocol handling Not supported

Features

Supported Go Versions

Recommended to use go1.23.0 and above. Initially Requests started supporting go modules

Installation

go get github.com/gospider007/requests

Usage

import "github.com/gospider007/requests"

Quickly Send Requests

package main

import (
	"log"
	"time"
	"github.com/gospider007/requests"
)

func main() {
    resp, err := requests.Get(nil, "http://httpbin.org/anything")
    if err != nil {
      log.Panic(err)
    }
    log.Print(resp.Text())    // Get content and parse as string
    log.Print(resp.Content()) // Get content as bytes
    log.Print(resp.Json())    // Get content and parse as gjson JSON
    log.Print(resp.Html())    // Get content and parse as goquery DOM
    log.Print(resp.Cookies()) // Get cookies
}

Contributing

If you have a bug report or feature request, you can open an issue

Contact

If you have questions, feel free to reach out to us in the following ways:

  • QQ Group (Chinese): 939111384 -
  • WeChat (Chinese): gospider007

Sponsors

If you like and it really helps you, feel free to reward me with a cup of coffee, and don't forget to mention your github id.


Wechat
Alipay

Documentation

Index

Constants

View Source 
const MaxUdpPacket int = math.MaxUint16 - 28

Variables

View Source 
var ErrUseLastResponse = http.ErrUseLastResponse

Functions

func NewJar 

func NewJar() *jar

new cookies jar

func NewRequestWithContext 

func NewRequestWithContext(ctx context.Context, method string, u *url.URL, body io.Reader) (*http.Request, error)

func WriteUdpAddr 

func WriteUdpAddr(w io.Writer, addr Address) error

Types

type Address 

type Address struct {
	User     string
	Password string
	Name     string
	Host     string
	IP       net.IP
	Port     int
	NetWork  string
	Scheme   string
}

func GetAddressWithAddr 

func GetAddressWithAddr(addrS string) (addr Address, err error)

func GetAddressWithUrl 

func GetAddressWithUrl(uurl *url.URL) (addr Address, err error)

func ReadUdpAddr 

func ReadUdpAddr(r io.Reader) (Address, error)

func (Address) Network 

func (a Address) Network() string

func (Address) String 

func (a Address) String() string

type Client 

type Client struct {
	ClientOption ClientOption
	// contains filtered or unexported fields
}

Connection Management

func NewClient 

func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error)

New Connection Management

func (*Client) ClearCookies 

func (obj *Client) ClearCookies()

clear cookies

func (*Client) Close 

func (obj *Client) Close()

Close the client and cannot be used again after shutdown

func (*Client) CloseConns 

func (obj *Client) CloseConns()

Close idle connections. If the connection is in use, wait until it ends before closing

func (*Client) Connect 

func (obj *Client) Connect(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Connect request and returns the response.

func (*Client) Delete 

func (obj *Client) Delete(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Delete request and returns the response.

func (*Client) ForceCloseConns 

func (obj *Client) ForceCloseConns()

Close the connection, even if it is in use, it will be closed

func (*Client) Get 

func (obj *Client) Get(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Get request and returns the response.

func (*Client) GetCookies 

func (obj *Client) GetCookies(href *url.URL) Cookies

get cookies

func (*Client) Head 

func (obj *Client) Head(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Head request and returns the response.

func (*Client) Options 

func (obj *Client) Options(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Options request and returns the response.

func (*Client) Patch 

func (obj *Client) Patch(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Patch request and returns the response.

func (*Client) Post 

func (obj *Client) Post(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Post request and returns the response.

func (*Client) Put 

func (obj *Client) Put(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Put request and returns the response.

func (*Client) Request 

func (obj *Client) Request(ctx context.Context, method string, href string, options ...RequestOption) (response *Response, err error)

Define a function named Request that takes in four parameters:

func (*Client) SetCookies 

func (obj *Client) SetCookies(href *url.URL, cookies ...any) error

set cookies

func (*Client) Trace 

func (obj *Client) Trace(ctx context.Context, href string, options ...RequestOption) (*Response, error)

sends a Trace request and returns the response.

type ClientOption 

type ClientOption struct {
	Logger                func(Log)                             //debuggable
	H3                    bool                                  //开启http3
	OrderHeaders          []string                              //order headers
	Ja3Spec               ja3.Spec                              //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
	H2Ja3Spec             ja3.H2Spec                            //h2 fingerprint
	UJa3Spec              ja3.USpec                             //h3 fingerprint
	Proxy                 string                                //proxy,support https,http,socks5
	Proxys                []string                              //proxy list,support https,http,socks5
	ForceHttp1            bool                                  //force  use http1 send requests
	Ja3                   bool                                  //enable ja3 fingerprint
	DisCookie             bool                                  //disable cookies
	DisDecode             bool                                  //disable auto decode
	DisUnZip              bool                                  //disable auto zip decode
	Bar                   bool                                  ////enable bar display
	OptionCallBack        func(ctx *Response) error             //option callback,if error is returnd, break request
	ResultCallBack        func(ctx *Response) error             //result callback,if error is returnd,next errCallback
	ErrCallBack           func(ctx *Response) error             //error callback,if error is returnd,break request
	RequestCallBack       func(ctx *Response) error             //request and response callback,if error is returnd,reponse is error
	MaxRetries            int                                   //try num
	MaxRedirect           int                                   //redirect num ,<0 no redirect,==0 no limit
	Headers               any                                   //default headers
	Timeout               time.Duration                         //request timeout
	ResponseHeaderTimeout time.Duration                         //ResponseHeaderTimeout ,default:300
	TlsHandshakeTimeout   time.Duration                         //tls timeout,default:15
	UserAgent             string                                //headers User-Agent value
	GetProxy              func(ctx *Response) (string, error)   //proxy callback:support https,http,socks5 proxy
	GetProxys             func(ctx *Response) ([]string, error) //proxys callback:support https,http,socks5 proxy
	DialOption            DialOption
	Jar                   Jar //custom cookies
	TlsConfig             *tls.Config
	UtlsConfig            *utls.Config

	QuicConfig  *quic.Config
	UquicConfig *uquic.Config
}

Connection Management Options

type Conn 

type Conn interface {
	CloseWithError(err error) error
	DoRequest(*http.Request, []string) (*http.Response, error)
}

type Cookies 

type Cookies []*http.Cookie

cookies

func ReadCookies 

func ReadCookies(val any) (Cookies, error)

read cookies or parse cookies,support json,map,[]string,http.Header,string

func (Cookies) Get 

func (obj Cookies) Get(name string) *http.Cookie

get cookie by name

func (Cookies) GetVal 

func (obj Cookies) GetVal(name string) string

get cookie value by name,return string

func (Cookies) GetVals 

func (obj Cookies) GetVals(name string) []string

get cookie values by name, return []string

func (Cookies) Gets 

func (obj Cookies) Gets(name string) Cookies

get cookies by name

func (Cookies) String 

func (obj Cookies) String() string

return cookies with string,join with '; '

type DialOption 

type DialOption struct {
	DialTimeout time.Duration
	KeepAlive   time.Duration
	LocalAddr   *net.TCPAddr  //network card ip
	AddrType    gtls.AddrType //first ip type
	Dns         *net.UDPAddr
	GetAddrType func(host string) gtls.AddrType
}

type Dialer 

type Dialer struct {
	// contains filtered or unexported fields
}

自定义dialer

func (*Dialer) DialContext 

func (obj *Dialer) DialContext(ctx *Response, network string, addr Address) (net.Conn, error)

func (*Dialer) DialProxyContext 

func (obj *Dialer) DialProxyContext(ctx *Response, network string, proxyTlsConfig *tls.Config, proxyUrls ...Address) (net.PacketConn, net.Conn, error)

func (*Dialer) ProxyDialContext 

func (obj *Dialer) ProxyDialContext(ctx *Response, network string, addr Address) (net.Conn, error)

func (*Dialer) Socks5TcpProxy 

func (obj *Dialer) Socks5TcpProxy(ctx *Response, proxyAddr Address, remoteAddr Address) (conn net.Conn, err error)

func (*Dialer) Socks5UdpProxy 

func (obj *Dialer) Socks5UdpProxy(ctx *Response, proxyAddress Address, remoteAddress Address) (udpConn net.PacketConn, err error)

type Event 

type Event struct {
	Data    string //data
	Event   string //event
	Id      string //id
	Retry   int    //retry num
	Comment string //comment info
}

type File 

type File struct {
	FileName    string
	ContentType string
	Content     any
}

Upload files with form-data,

type Jar 

type Jar interface {
	ClearCookies()
	GetCookies(u *url.URL) Cookies
	SetCookies(*url.URL, Cookies)
}

type Log 

type Log struct {
	Id   string  `json:"id"`
	Type LogType `json:"type"`
	Time time.Time
	Msg  any `json:"msg"`
}

type LogType 

type LogType string
const (
	LogType_DNSLookup LogType = "DNSLookup"

	LogType_TCPConnect LogType = "TCPConnect"

	LogType_TLSHandshake LogType = "TLSHandshake"

	LogType_ProxyDNSLookup LogType = "ProxyDNSLookup"

	LogType_ProxyTCPConnect LogType = "ProxyTCPConnect"

	LogType_ProxyTLSHandshake LogType = "ProxyTLSHandshake"

	LogType_ProxyConnectRemote LogType = "ProxyConnectRemote"

	LogType_ResponseHeader LogType = "ResponseHeader"

	LogType_ResponseBody LogType = "ResponseBody"
)

type OrderMap 

type OrderMap struct {
	// contains filtered or unexported fields
}

func NewOrderMap 

func NewOrderMap() *OrderMap

func (*OrderMap) Del 

func (obj *OrderMap) Del(key string)

func (*OrderMap) Keys 

func (obj *OrderMap) Keys() []string

func (*OrderMap) MarshalJSON 

func (obj *OrderMap) MarshalJSON() ([]byte, error)

func (*OrderMap) Set 

func (obj *OrderMap) Set(key string, val any)

type RequestOption 

type RequestOption struct {
	ClientOption
	// other option
	Method      string //method
	Url         *url.URL
	Host        string
	Referer     string //set headers referer value
	ContentType string //headers Content-Type value
	Cookies     any    // cookies,support :json,map,str，http.Header

	Params any //url params，join url query,json,map
	Json   any //send application/json,support io.Reader,：string,[]bytes,json,map
	Data   any //send application/x-www-form-urlencoded, support io.Reader, string,[]bytes,json,map
	Form   any //send multipart/form-data,file upload,support io.Reader, json,map
	Text   any //send text/xml,support: io.Reader, string,[]bytes,json,map
	Body   any //not setting context-type,support io.Reader, string,[]bytes,json,map

	Stream   bool             //disable auto read
	WsOption websocket.Option //websocket option
	DisProxy bool             //force disable proxy
	// contains filtered or unexported fields
}

Options for sending requests

type Response 

type Response struct {
	// contains filtered or unexported fields
}

func Connect 

func Connect(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a Connect request and returns the response.

func Delete 

func Delete(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a Delete request and returns the response.

func Get 

func Get(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a GET request and returns the response. 

func Head(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a Head request and returns the response.

func NewResponse 

func NewResponse(ctx context.Context, option RequestOption) *Response

func Options 

func Options(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a Options request and returns the response.

func Patch 

func Patch(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a Patch request and returns the response.

func Post 

func Post(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a Post request and returns the response.

func Put 

func Put(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a Put request and returns the response.

func Request 

func Request(ctx context.Context, method string, href string, options ...RequestOption) (resp *Response, err error)

Define a function named Request that takes in four parameters:

func Trace 

func Trace(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error)

sends a Trace request and returns the response.

func (*Response) Body 

func (obj *Response) Body() io.ReadCloser

func (*Response) Client 

func (obj *Response) Client() *Client

func (*Response) CloseBody 

func (obj *Response) CloseBody()

close body

func (*Response) CloseConn 

func (obj *Response) CloseConn()

safe close conn

func (*Response) Content 

func (obj *Response) Content() []byte

return content with []byte

func (*Response) ContentEncoding 

func (obj *Response) ContentEncoding() string

return content encoding

func (*Response) ContentLength 

func (obj *Response) ContentLength() int64

return content length

func (*Response) ContentType 

func (obj *Response) ContentType() string

return content type

func (*Response) Context 

func (obj *Response) Context() context.Context

func (*Response) Cookies 

func (obj *Response) Cookies() Cookies

return response cookies

func (*Response) Decode 

func (obj *Response) Decode(encoding string)

change decoding with content

func (*Response) Err 

func (obj *Response) Err() error

func (*Response) ForceCloseConn 

func (obj *Response) ForceCloseConn()

force close conn

func (*Response) Headers 

func (obj *Response) Headers() http.Header

return response headers

func (*Response) Html 

func (obj *Response) Html() *bs4.Client

return content with parse html

func (*Response) IsNewConn 

func (obj *Response) IsNewConn() bool

conn is new conn

func (*Response) IsSSE 

func (obj *Response) IsSSE() bool

func (*Response) IsStream 

func (obj *Response) IsStream() bool

return true if response is stream

func (*Response) IsWebSocket 

func (obj *Response) IsWebSocket() bool

return true if response is other stream

func (*Response) Json 

func (obj *Response) Json(vals ...any) (*gson.Client, error)

return content with json and you can parse struct

func (*Response) Location 

func (obj *Response) Location() (*url.URL, error)

return URL redirected address

func (*Response) Map 

func (obj *Response) Map() (data map[string]any, err error)

return content with map[string]any

func (*Response) Option 

func (obj *Response) Option() *RequestOption

func (*Response) Proto 

func (obj *Response) Proto() string

return response Proto

func (*Response) Proxys 

func (obj *Response) Proxys() []Address

conn proxy

func (*Response) ReadBody 

func (obj *Response) ReadBody() (err error)

read body

func (*Response) Request 

func (obj *Response) Request() *http.Request

func (*Response) Response 

func (obj *Response) Response() *http.Response

func (*Response) SSE 

func (obj *Response) SSE() *SSE

return SSE client

func (*Response) SetContent 

func (obj *Response) SetContent(val []byte)

set response content with []byte

func (*Response) Status 

func (obj *Response) Status() string

return response status

func (*Response) StatusCode 

func (obj *Response) StatusCode() int

return response status code

func (*Response) Text 

func (obj *Response) Text() string

return content with string

func (*Response) Url 

func (obj *Response) Url() *url.URL

return response url

func (*Response) WebSocket 

func (obj *Response) WebSocket() *websocket.Conn

return websocket client

type SSE 

type SSE struct {
	// contains filtered or unexported fields
}

func (*SSE) Close 

func (obj *SSE) Close()

close SSE

func (*SSE) Range 

func (obj *SSE) Range() iter.Seq2[Event, error]

func (*SSE) Recv 

func (obj *SSE) Recv() (Event, error)

recv SSE envent data

type UDPConn 

type UDPConn struct {
	net.PacketConn
	UDPConn *net.UDPConn
	// contains filtered or unexported fields
}

func NewUDPConn 

func NewUDPConn(packConn net.PacketConn, proxyAddress net.Addr) (*UDPConn, error)

func (*UDPConn) ReadFrom 

func (c *UDPConn) ReadFrom(p []byte) (n int, addr net.Addr, err error)

ReadFrom implements the net.PacketConn ReadFrom method.

func (*UDPConn) ReadFromUDP 

func (c *UDPConn) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error)

func (*UDPConn) ReadMsgUDP 

func (c *UDPConn) ReadMsgUDP(b, oob []byte) (n, oobn, flags int, addr *net.UDPAddr, err error)

func (*UDPConn) SetDeadline 

func (c *UDPConn) SetDeadline(t time.Time) error

func (*UDPConn) SetReadBuffer 

func (c *UDPConn) SetReadBuffer(bytes int) error

func (*UDPConn) SetReadDeadline 

func (c *UDPConn) SetReadDeadline(t time.Time) error

func (*UDPConn) SetWriteBuffer 

func (c *UDPConn) SetWriteBuffer(bytes int) error

func (*UDPConn) SetWriteDeadline 

func (c *UDPConn) SetWriteDeadline(t time.Time) error

func (*UDPConn) WriteMsgUDP 

func (c *UDPConn) WriteMsgUDP(b, oob []byte, addr *net.UDPAddr) (n, oobn int, err error)

func (*UDPConn) WriteTo 

func (c *UDPConn) WriteTo(p []byte, addr net.Addr) (n int, err error)

WriteTo implements the net.PacketConn WriteTo method.

func (*UDPConn) WriteToUDP 

func (c *UDPConn) WriteToUDP(b []byte, addr *net.UDPAddr) (int, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.