tls_client

package module
v0.0.0-...-c1bc5a5 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 18, 2024 License: BSD-4-Clause Imports: 24 Imported by: 0

README

TLS-Client

Preface

This TLS Client is built upon https://github.com/Carcraftz/fhttp and https://github.com/Carcraftz/utls. Big thanks to all contributors so far. Sadly it seems that the original repositories are not maintained anymore.

What is TLS Fingerprinting?

Some people think it is enough to change the user-agent header of a request to let the server think that the client requesting a resource is a specific browser. Nowadays this is not enough, because the server might use a technique to detect the client browser which is called TLS Fingerprinting.

Even though this article is about TLS Fingerprinting in NodeJS it well describes the technique in general. https://httptoolkit.tech/blog/tls-fingerprinting-node-js/#how-does-tls-fingerprinting-work

Why is this library needed?

With this library you are able to create a http client implementing an interface which is similar to golangs net/http client interface. This TLS Client allows you to specify the Client (Browser and Version) you want to use, when requesting a server.

The Interface of the HTTP Client looks like the following and extends the base net/http Client Interface by some useful functions. Most likely you will use the Do() function like you did before with net/http Client.

type HttpClient interface {
    GetCookies(u *url.URL) []*http.Cookie
    SetCookies(u *url.URL, cookies []*http.Cookie)
    SetCookieJar(jar http.CookieJar)
    GetCookieJar() http.CookieJar
    SetProxy(proxyUrl string) error
    GetProxy() string
    SetFollowRedirect(followRedirect bool)
    GetFollowRedirect() bool
    CloseIdleConnections()
    Do(req *http.Request) (*http.Response, error)
    Get(url string) (resp *http.Response, err error)
    Head(url string) (resp *http.Response, err error)
    Post(url, contentType string, body io.Reader) (resp *http.Response, err error)
}
Quick Usage Example
package main

import (
	"fmt"
	"io"
	"log"

	http "github.com/bogdanfinn/fhttp"
	tls_client "github.com/Enven-LLC/enven-tls"
	"github.com/Enven-LLC/enven-tls/profiles"
)

func main() {
    jar := tls_client.NewCookieJar()
	options := []tls_client.HttpClientOption{
		tls_client.WithTimeoutSeconds(30),
		tls_client.WithClientProfile(profiles.Chrome_105),
		tls_client.WithNotFollowRedirects(),
		tls_client.WithCookieJar(jar), // create cookieJar instance and pass it as argument
	}

	client, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...)
	if err != nil {
		log.Println(err)
		return
	}

	req, err := http.NewRequest(http.MethodGet, "https://tls.peet.ws/api/all", nil)
	if err != nil {
		log.Println(err)
		return
	}

	req.Header = http.Header{
		"accept":                    {"*/*"},
		"accept-language":           {"de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7"},
		"user-agent":                {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36"},
		http.HeaderOrderKey: {
			"accept",
			"accept-language",
			"user-agent",
		},
	}

	resp, err := client.Do(req)
	if err != nil {
		log.Println(err)
		return
	}

	defer resp.Body.Close()

	log.Println(fmt.Sprintf("status code: %d", resp.StatusCode))

	readBytes, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Println(err)
		return
	}

	log.Println(string(readBytes))
}
Detailed Documentation

https://bogdanfinn.gitbook.io/open-source-oasis/

Questions?

Join my discord support server for free: https://discord.gg/7Ej9eJvHqk No Support in DMs!

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var DefaultBadPinHandler = func(req *http.Request) {
	fmt.Println("this is the default bad pin handler")
}
View Source 
var DefaultOptions = []HttpClientOption{
	WithTimeoutSeconds(DefaultTimeoutSeconds),
	WithClientProfile(profiles.DefaultClientProfile),
	WithRandomTLSExtensionOrder(),
	WithNotFollowRedirects(),
}
View Source 
var DefaultTimeoutSeconds = 30
View Source 
var ErrBadPinDetected = errors.New("bad ssl pin detected")
View Source 
var H2SettingsMap = map[string]http2.SettingID{
	"HEADER_TABLE_SIZE":      http2.SettingHeaderTableSize,
	"ENABLE_PUSH":            http2.SettingEnablePush,
	"MAX_CONCURRENT_STREAMS": http2.SettingMaxConcurrentStreams,
	"INITIAL_WINDOW_SIZE":    http2.SettingInitialWindowSize,
	"MAX_FRAME_SIZE":         http2.SettingMaxFrameSize,
	"MAX_HEADER_LIST_SIZE":   http2.SettingMaxHeaderListSize,
}

Functions

func GetSpecFactoryFromJa3String 

func GetSpecFactoryFromJa3String(ja3String string, supportedSignatureAlgorithms, supportedDelegatedCredentialsAlgorithms, supportedVersions, keyShareCurves, supportedProtocolsALPN, supportedProtocolsALPS []string, echCandidateCipherSuites []CandidateCipherSuites, candidatePayloads []uint16, certCompressionAlgo string) (func() (tls.ClientHelloSpec, error), error)

Types

type BadPinHandlerFunc 

type BadPinHandlerFunc func(req *http.Request)

type BetterJar 

type BetterJar struct {
	// contains filtered or unexported fields
}

func NewBetterJar 

func NewBetterJar() *BetterJar

func (*BetterJar) GetCookieJar 

func (bj *BetterJar) GetCookieJar() map[string]string

func (*BetterJar) GetCookies 

func (bj *BetterJar) GetCookies() string

func (*BetterJar) SetCookies 

func (bj *BetterJar) SetCookies(cookieString string)

type CandidateCipherSuites 

type CandidateCipherSuites struct {
	KdfId  string
	AeadId string
}

type CertificatePinner 

type CertificatePinner interface {
	Pin(conn *tls.UConn, host string) error
}

func NewCertificatePinner 

func NewCertificatePinner(certificatePins map[string][]string) (CertificatePinner, error)

type ContextKeyHeader 

type ContextKeyHeader struct{}

Users of context.WithValue should define their own types for keys

type CookieJar 

type CookieJar interface {
	http.CookieJar
	GetAllCookies() map[string][]*http.Cookie
}

func NewCookieJar 

func NewCookieJar(options ...CookieJarOption) CookieJar

type CookieJarOption 

type CookieJarOption func(config *cookieJarConfig)

func WithAllowEmptyCookies 

func WithAllowEmptyCookies() CookieJarOption

func WithDebugLogger 

func WithDebugLogger() CookieJarOption

func WithLogger 

func WithLogger(logger Logger) CookieJarOption

func WithSkipExisting 

func WithSkipExisting() CookieJarOption

type HttpClient 

type HttpClient interface {
	GetCookies(u *url.URL) []*http.Cookie
	SetCookies(u *url.URL, cookies []*http.Cookie)
	SetCookieJar(jar http.CookieJar)
	GetCookieJar() http.CookieJar

	SetBJar(jar *BetterJar)
	GetBJar() *BetterJar

	SetProxy(proxyUrl string) error
	GetProxy() string
	SetFollowRedirect(followRedirect bool)
	GetFollowRedirect() bool
	CloseIdleConnections()
	Do(req *http.Request) (*WebResp, error)
}

func NewHttpClient 

func NewHttpClient(logger Logger, options ...HttpClientOption) (HttpClient, error)

NewHttpClient constructs a new HTTP client with the given logger and client options.

func ProvideDefaultClient 

func ProvideDefaultClient(logger Logger) (HttpClient, error)

type HttpClientOption 

type HttpClientOption func(config *httpClientConfig)

func WithBetterJar 

func WithBetterJar(bjar *BetterJar) HttpClientOption

func WithCatchPanics 

func WithCatchPanics() HttpClientOption

WithCatchPanics configures a client to catch all go panics happening during a request and not print the stacktrace.

func WithCertificatePinning 

func WithCertificatePinning(certificatePins map[string][]string, handlerFunc BadPinHandlerFunc) HttpClientOption

WithCertificatePinning enables SSL Pinning for the client and will throw an error if the SSL Pin is not matched. Please refer to https://github.com/tam7t/hpkp/#examples in order to see how to generate pins. The certificatePins are a map with the host as key. You can provide a BadPinHandlerFunc or nil as second argument. This function will be executed once a bad ssl pin is detected. BadPinHandlerFunc has to be defined like this: func(req *http.Request){}

func WithCharlesProxy 

func WithCharlesProxy(host string, port string) HttpClientOption

WithCharlesProxy configures the HTTP client to use a local running charles as proxy.

host and port can be empty, then default 127.0.0.1 and port 8888 will be used

func WithClientProfile 

func WithClientProfile(clientProfile profiles.ClientProfile) HttpClientOption

WithClientProfile configures a TLS client to use the specified client profile.

func WithCookieJar 

func WithCookieJar(jar http.CookieJar) HttpClientOption

WithCookieJar configures a HTTP client to use the specified cookie jar.

func WithCustomRedirectFunc 

func WithCustomRedirectFunc(redirectFunc func(req *http.Request, via []*http.Request) error) HttpClientOption

WithCustomRedirectFunc configures an HTTP client to use a custom redirect func. The redirect func have to look like that: func(req *http.Request, via []*http.Request) error Please only provide a custom redirect function if you know what you are doing. Check docs on net/http.Client CheckRedirect

func WithDebug 

func WithDebug() HttpClientOption

WithDebug configures a client to log debugging information.

func WithDefaultHeaders 

func WithDefaultHeaders(defaultHeaders http.Header) HttpClientOption

WithDefaultHeaders configures a TLS client to use a set of default headers if none are specified on the request.

func WithDialer 

func WithDialer(dialer net.Dialer) HttpClientOption

WithDialer configures an HTTP client to use the specified dialer. This allows the use of a custom DNS resolver

func WithDisableIPV6 

func WithDisableIPV6() HttpClientOption

WithDisableIPV6 configures a dialer to use tcp4 network argument

func WithForceHttp1 

func WithForceHttp1() HttpClientOption

WithForceHttp1 configures a client to force HTTP/1.1 as the used protocol.

func WithInsecureSkipVerify 

func WithInsecureSkipVerify() HttpClientOption

WithInsecureSkipVerify configures a client to skip SSL certificate verification.

func WithLocalAddr 

func WithLocalAddr(localAddr net.TCPAddr) HttpClientOption

WithLocalAddr configures an HTTP client to use the specified local address.

func WithNotFollowRedirects 

func WithNotFollowRedirects() HttpClientOption

WithNotFollowRedirects configures an HTTP client to not follow HTTP redirects.

func WithProxyUrl 

func WithProxyUrl(proxyUrl string) HttpClientOption

WithProxyUrl configures a HTTP client to use the specified proxy URL.

proxyUrl should be formatted as: 

"http://user:pass@host:port"

func WithRandomTLSExtensionOrder 

func WithRandomTLSExtensionOrder() HttpClientOption

WithRandomTLSExtensionOrder configures a TLS client to randomize the order of TLS extensions being sent in the ClientHello.

Placement of GREASE and padding is fixed and will not be affected by this.

func WithServerNameOverwrite 

func WithServerNameOverwrite(serverName string) HttpClientOption

WithServerNameOverwrite configures a TLS client to overwrite the server name being used for certificate verification and in the client hello. This option does only work properly if WithInsecureSkipVerify is set to true in addition

func WithTimeout 

func WithTimeout(timeout int) HttpClientOption

WithTimeout configures an HTTP client to use the specified request timeout.

timeout is the request timeout in seconds. Deprecated: use either WithTimeoutSeconds or WithTimeoutMilliseconds

func WithTimeoutMilliseconds 

func WithTimeoutMilliseconds(timeout int) HttpClientOption

WithTimeoutMilliseconds configures an HTTP client to use the specified request timeout.

timeout is the request timeout in milliseconds.

func WithTimeoutSeconds 

func WithTimeoutSeconds(timeout int) HttpClientOption

WithTimeoutSeconds configures an HTTP client to use the specified request timeout.

timeout is the request timeout in seconds.

func WithTransportOptions 

func WithTransportOptions(transportOptions *TransportOptions) HttpClientOption

WithTransportOptions configures a client to use the specified transport options.

type Logger 

type Logger interface {
	Debug(format string, args ...any)
	Info(format string, args ...any)
	Warn(format string, args ...any)
	Error(format string, args ...any)
}

func NewDebugLogger 

func NewDebugLogger(logger Logger) Logger

func NewLogger 

func NewLogger() Logger

func NewNoopLogger 

func NewNoopLogger() Logger

type TransportOptions 

type TransportOptions struct {
	DisableKeepAlives      bool
	DisableCompression     bool
	MaxIdleConns           int
	MaxIdleConnsPerHost    int
	MaxConnsPerHost        int
	MaxResponseHeaderBytes int64 // Zero means to use a default limit.
	WriteBufferSize        int   // If zero, a default (currently 4KB) is used.
	ReadBufferSize         int   // If zero, a default (currently 4KB) is used.
	// IdleConnTimeout is the maximum amount of time an idle (keep-alive)
	// connection will remain idle before closing itself. Zero means no limit.
	IdleConnTimeout *time.Duration
	// RootCAs is the set of root certificate authorities used to verify
	// the remote server's certificate.
	RootCAs *x509.CertPool
	// KeyLogWriter is an io.Writer that the TLS client will use to write the
	// TLS master secrets to. This can be used to decrypt TLS connections in
	// Wireshark and other applications.
	KeyLogWriter io.Writer
}

type WebReq 

type WebReq struct {
	BJar *BetterJar
	// NoDecodeBody bool
	// Method specifies the HTTP method (GET, POST, PUT, etc.).
	// For client requests, an empty string means GET.
	//
	// Go's HTTP client does not support sending a request with
	// the CONNECT method. See the documentation on Transport for
	// details.
	Method string

	// URL specifies either the URI being requested (for server
	// requests) or the URL to access (for client requests).
	//
	// For server requests, the URL is parsed from the URI
	// supplied on the Request-Line as stored in RequestURI.  For
	// most requests, fields other than Path and RawQuery will be
	// empty. (See RFC 7230, Section 5.3)
	//
	// For client requests, the URL's Host specifies the server to
	// connect to, while the Request's Host field optionally
	// specifies the Host header value to send in the HTTP
	// request.
	URL *url.URL

	// The protocol version for incoming server requests.
	//
	// For client requests, these fields are ignored. The HTTP
	// client code always uses either HTTP/1.1 or HTTP/2.
	// See the docs on Transport for details.
	Proto      string // "HTTP/1.0"
	ProtoMajor int    // 1
	ProtoMinor int    // 0

	// Header contains the request header fields either received
	// by the server or to be sent by the client.
	//
	// If a server received a request with header lines,
	//
	//	Host: example.com
	//	accept-encoding: gzip, deflate
	//	Accept-Language: en-us
	//	fOO: Bar
	//	foo: two
	//
	// then
	//
	//	Header = map[string][]string{
	//		"Accept-Encoding": {"gzip, deflate"},
	//		"Accept-Language": {"en-us"},
	//		"Foo": {"Bar", "two"},
	//	}
	//
	// For incoming requests, the Host header is promoted to the
	// Request.Host field and removed from the Header map.
	//
	// HTTP defines that header names are case-insensitive. The
	// request parser implements this by using CanonicalHeaderKey,
	// making the first character and any characters following a
	// hyphen uppercase and the rest lowercase.
	//
	// For client requests, certain headers such as Content-Length
	// and Connection are automatically written when needed and
	// Values in Header may be ignored. See the documentation
	// for the Request.Write method.
	Header http.Header

	// Cookies to use in place of jar
	Cookies string

	// Body is the request's body.
	//
	// For client requests, a nil body means the request has no
	// body, such as a GET request. The HTTP Client's Transport
	// is responsible for calling the Close method.
	//
	// For server requests, the Request Body is always non-nil
	// but will return EOF immediately when no body is present.
	// The Server will close the request body. The ServeHTTP
	// Handler does not need to.
	//
	// Body must allow Read to be called concurrently with Close.
	// In particular, calling Close should unblock a Read waiting
	// for input.
	Body io.ReadCloser

	// GetBody defines an optional func to return a new copy of
	// Body. It is used for client requests when a redirect requires
	// reading the body more than once. Use of GetBody still
	// requires setting Body.
	//
	// For server requests, it is unused.
	GetBody func() (io.ReadCloser, error)

	// ContentLength records the length of the associated content.
	// The value -1 indicates that the length is unknown.
	// Values >= 0 indicate that the given number of bytes may
	// be read from Body.
	//
	// For client requests, a value of 0 with a non-nil Body is
	// also treated as unknown.
	ContentLength int64

	// TransferEncoding lists the transfer encodings from outermost to
	// innermost. An empty list denotes the "identity" encoding.
	// TransferEncoding can usually be ignored; chunked encoding is
	// automatically added and removed as necessary when sending and
	// receiving requests.
	TransferEncoding []string

	// Close indicates whether to close the connection after
	// replying to this request (for servers) or after sending this
	// request and reading its response (for clients).
	//
	// For server requests, the HTTP server handles this automatically
	// and this field is not needed by Handlers.
	//
	// For client requests, setting this field prevents re-use of
	// TCP connections between requests to the same hosts, as if
	// Transport.DisableKeepAlives were set.
	Close bool

	// For server requests, Host specifies the host on which the
	// URL is sought. For HTTP/1 (per RFC 7230, section 5.4), this
	// is either the value of the "Host" header or the host name
	// given in the URL itself. For HTTP/2, it is the value of the
	// ":authority" pseudo-header field.
	// It may be of the form "host:port". For international domain
	// names, Host may be in Punycode or Unicode form. Use
	// golang.org/x/net/idna to convert it to either format if
	// needed.
	// To prevent DNS rebinding attacks, server Handlers should
	// validate that the Host header has a value for which the
	// Handler considers itself authoritative. The included
	// ServeMux supports patterns registered to particular host
	// names and thus protects its registered Handlers.
	//
	// For client requests, Host optionally overrides the Host
	// header to send. If empty, the Request.Write method uses
	// the value of URL.Host. Host may contain an international
	// domain name.
	Host string

	// Form contains the parsed form data, including both the URL
	// field's query parameters and the PATCH, POST, or PUT form data.
	// This field is only available after ParseForm is called.
	// The HTTP client ignores Form and uses Body instead.
	Form url.Values

	// PostForm contains the parsed form data from PATCH, POST
	// or PUT body parameters.
	//
	// This field is only available after ParseForm is called.
	// The HTTP client ignores PostForm and uses Body instead.
	PostForm url.Values

	// MultipartForm is the parsed multipart form, including file uploads.
	// This field is only available after ParseMultipartForm is called.
	// The HTTP client ignores MultipartForm and uses Body instead.
	MultipartForm *multipart.Form

	// Trailer specifies additional headers that are sent after the request
	// body.
	//
	// For server requests, the Trailer map initially contains only the
	// trailer keys, with nil Values. (The client declares which trailers it
	// will later send.)  While the handler is reading from Body, it must
	// not reference Trailer. After reading from Body returns EOF, Trailer
	// can be read again and will contain non-nil Values, if they were sent
	// by the client.
	//
	// For client requests, Trailer must be initialized to a map containing
	// the trailer keys to later send. The Values may be nil or their final
	// Values. The ContentLength must be 0 or -1, to send a chunked request.
	// After the HTTP request is sent the map Values can be updated while
	// the request body is read. Once the body returns EOF, the caller must
	// not mutate Trailer.
	//
	// Few HTTP clients, servers, or proxies support HTTP trailers.
	Trailer http.Header

	// RemoteAddr allows HTTP servers and other software to record
	// the network address that sent the request, usually for
	// logging. This field is not filled in by ReadRequest and
	// has no defined format. The HTTP server in this package
	// sets RemoteAddr to an "IP:port" address before invoking a
	// handler.
	// This field is ignored by the HTTP client.
	RemoteAddr string

	// RequestURI is the unmodified request-target of the
	// Request-Line (RFC 7230, Section 3.1.1) as sent by the client
	// to a server. Usually the URL field should be used instead.
	// It is an error to set this field in an HTTP client request.
	RequestURI string

	// TLS allows HTTP servers and other software to record
	// information about the TLS connection on which the request
	// was received. This field is not filled in by ReadRequest.
	// The HTTP server in this package sets the field for
	// TLS-enabled connections before invoking a handler;
	// otherwise it leaves the field nil.
	// This field is ignored by the HTTP client.
	TLS *tls.ConnectionState

	// Cancel is an optional channel whose closure indicates that the client
	// request should be regarded as canceled. Not all implementations of
	// RoundTripper may support Cancel.
	//
	// For server requests, this field is not applicable.
	//
	// Deprecated: Set the Request's context with NewRequestWithContext
	// instead. If a Request's Cancel field and context are both
	// set, it is undefined whether Cancel is respected.
	Cancel <-chan struct{}

	// Response is the redirect response which caused this request
	// to be created. This field is only populated during client
	// redirects.
	Response *http.Response
	// contains filtered or unexported fields
}

type WebResp 

type WebResp struct {
	Status     string // e.g. "200 OK"
	StatusCode int    // e.g. 200
	Proto      string // e.g. "HTTP/1.0"
	ProtoMajor int    // e.g. 1
	ProtoMinor int    // e.g. 0

	Cookies string
	// Header maps header keys to Values. If the response had multiple
	// headers with the same Key, they may be concatenated, with comma
	// delimiters.  (RFC 7230, section 3.2.2 requires that multiple headers
	// be semantically equivalent to a comma-delimited sequence.) When
	// Header Values are duplicated by other fields in this struct (e.g.,
	// ContentLength, TransferEncoding, Trailer), the field Values are
	// authoritative.
	//
	// Keys in the map are canonicalized (see CanonicalHeaderKey).
	Header http.Header

	// Body represents the response body.
	Body      string
	BodyBytes []byte

	// ContentLength records the length of the associated content. The
	// value -1 indicates that the length is unknown. Unless Request.Method
	// is "HEAD", Values >= 0 indicate that the given number of bytes may
	// be read from Body.
	ContentLength int64

	// Contains transfer encodings from outer-most to inner-most. Value is
	// nil, means that "identity" encoding is used.
	TransferEncoding []string

	// Close records whether the header directed that the connection be
	// closed after reading Body. The value is advice for clients: neither
	// ReadResponse nor Response.Write ever closes a connection.
	Close bool

	// Uncompressed reports whether the response was sent compressed but
	// was decompressed by the http package. When true, reading from
	// Body yields the uncompressed content instead of the compressed
	// content actually set from the server, ContentLength is set to -1,
	// and the "Content-Length" and "Content-Encoding" fields are deleted
	// from the responseHeader. To get the original response from
	// the server, set Transport.DisableCompression to true.
	Uncompressed bool

	// Trailer maps trailer keys to Values in the same
	// format as Header.
	//
	// The Trailer initially contains only nil Values, one for
	// each Key specified in the server's "Trailer" header
	// value. Those Values are not added to Header.
	//
	// Trailer must not be accessed concurrently with Read calls
	// on the Body.
	//
	// After Body.Read has returned io.EOF, Trailer will contain
	// any trailer Values sent by the server.
	Trailer http.Header

	// Request is the request that was sent to obtain this Response.
	// Request's Body is nil (having already been consumed).
	// This is only populated for Client requests.
	Request *http.Request

	// TLS contains information about the TLS connection on which the
	// response was received. It is nil for unencrypted responses.
	// The pointer is shared between responses and should not be
	// modified.
	TLS *tls.ConnectionState
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.