tls_client

package module
v1.7.10 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 3, 2024 License: BSD-4-Clause Imports: 27 Imported by: 134

README

TLS-Client

Preface

This TLS Client is built upon https://github.com/Carcraftz/fhttp and https://github.com/Carcraftz/utls (https://github.com/refraction-networking/utls). Big thanks to all contributors so far. Sadly it seems that the original repositories from Carcraftz are not maintained anymore.

What is TLS Fingerprinting?

Some people think it is enough to change the user-agent header of a request to let the server think that the client requesting a resource is a specific browser. Nowadays this is not enough, because the server might use a technique to detect the client browser which is called TLS Fingerprinting.

Even though this article is about TLS Fingerprinting in NodeJS it well describes the technique in general. https://httptoolkit.tech/blog/tls-fingerprinting-node-js/#how-does-tls-fingerprinting-work

Why is this library needed?

With this library you are able to create a http client implementing an interface which is similar to golangs net/http client interface. This TLS Client allows you to specify the Client (Browser and Version) you want to use, when requesting a server.

The Interface of the HTTP Client looks like the following and extends the base net/http Client Interface by some useful functions. Most likely you will use the Do() function like you did before with net/http Client.

type HttpClient interface {
    GetCookies(u *url.URL) []*http.Cookie
    SetCookies(u *url.URL, cookies []*http.Cookie)
    SetCookieJar(jar http.CookieJar)
    GetCookieJar() http.CookieJar
    SetProxy(proxyUrl string) error
    GetProxy() string
    SetFollowRedirect(followRedirect bool)
    GetFollowRedirect() bool
    CloseIdleConnections()
    Do(req *http.Request) (*http.Response, error)
    Get(url string) (resp *http.Response, err error)
    Head(url string) (resp *http.Response, err error)
    Post(url, contentType string, body io.Reader) (resp *http.Response, err error)
}
Quick Usage Example
package main

import (
	"fmt"
	"io"
	"log"

	http "github.com/bogdanfinn/fhttp"
	tls_client "github.com/bogdanfinn/tls-client"
	"github.com/bogdanfinn/tls-client/profiles"
)

func main() {
    jar := tls_client.NewCookieJar()
	options := []tls_client.HttpClientOption{
		tls_client.WithTimeoutSeconds(30),
		tls_client.WithClientProfile(profiles.Chrome_120),
		tls_client.WithNotFollowRedirects(),
		tls_client.WithCookieJar(jar), // create cookieJar instance and pass it as argument
	}

	client, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...)
	if err != nil {
		log.Println(err)
		return
	}

	req, err := http.NewRequest(http.MethodGet, "https://tls.peet.ws/api/all", nil)
	if err != nil {
		log.Println(err)
		return
	}

	req.Header = http.Header{
		"accept":                    {"*/*"},
		"accept-language":           {"de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7"},
		"user-agent":                {"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"},
		http.HeaderOrderKey: {
			"accept",
			"accept-language",
			"user-agent",
		},
	}

	resp, err := client.Do(req)
	if err != nil {
		log.Println(err)
		return
	}

	defer resp.Body.Close()

	log.Println(fmt.Sprintf("status code: %d", resp.StatusCode))

	readBytes, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Println(err)
		return
	}

	log.Println(string(readBytes))
}
Detailed Documentation

https://bogdanfinn.gitbook.io/open-source-oasis/

Questions?

Join my discord support server for free: https://discord.gg/7Ej9eJvHqk No Support in DMs!

Appreciate my work?

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var DefaultBadPinHandler = func(req *http.Request) {
	fmt.Println("this is the default bad pin handler")
}
View Source 
var DefaultOptions = []HttpClientOption{
	WithTimeoutSeconds(DefaultTimeoutSeconds),
	WithClientProfile(profiles.DefaultClientProfile),
	WithRandomTLSExtensionOrder(),
	WithNotFollowRedirects(),
}
View Source 
var DefaultTimeoutSeconds = 30
View Source 
var ErrBadPinDetected = errors.New("bad ssl pin detected")
View Source 
var H2SettingsMap = map[string]http2.SettingID{
	"HEADER_TABLE_SIZE":      http2.SettingHeaderTableSize,
	"ENABLE_PUSH":            http2.SettingEnablePush,
	"MAX_CONCURRENT_STREAMS": http2.SettingMaxConcurrentStreams,
	"INITIAL_WINDOW_SIZE":    http2.SettingInitialWindowSize,
	"MAX_FRAME_SIZE":         http2.SettingMaxFrameSize,
	"MAX_HEADER_LIST_SIZE":   http2.SettingMaxHeaderListSize,
	"UNKNOWN_SETTING_7":      0x7,
	"UNKNOWN_SETTING_8":      0x8,
	"UNKNOWN_SETTING_9":      0x9,
}

Functions

func GetSpecFactoryFromJa3String 

func GetSpecFactoryFromJa3String(ja3String string, supportedSignatureAlgorithms, supportedDelegatedCredentialsAlgorithms, supportedVersions, keyShareCurves, supportedProtocolsALPN, supportedProtocolsALPS []string, echCandidateCipherSuites []CandidateCipherSuites, candidatePayloads []uint16, certCompressionAlgo string) (func() (tls.ClientHelloSpec, error), error)

Types

type BadPinHandlerFunc 

type BadPinHandlerFunc func(req *http.Request)

type CandidateCipherSuites added in v1.7.0 

type CandidateCipherSuites struct {
	KdfId  string
	AeadId string
}

type CertificatePinner 

type CertificatePinner interface {
	Pin(conn *tls.UConn, host string) error
}

func NewCertificatePinner 

func NewCertificatePinner(certificatePins map[string][]string) (CertificatePinner, error)

type ContextKeyHeader 

type ContextKeyHeader struct{}

Users of context.WithValue should define their own types for keys

type CookieJar 

type CookieJar interface {
	http.CookieJar
	GetAllCookies() map[string][]*http.Cookie
}

func NewCookieJar 

func NewCookieJar(options ...CookieJarOption) CookieJar

type CookieJarOption 

type CookieJarOption func(config *cookieJarConfig)

func WithAllowEmptyCookies added in v1.3.6 

func WithAllowEmptyCookies() CookieJarOption

func WithDebugLogger added in v1.3.6 

func WithDebugLogger() CookieJarOption

func WithLogger 

func WithLogger(logger Logger) CookieJarOption

func WithSkipExisting 

func WithSkipExisting() CookieJarOption

type HttpClient 

type HttpClient interface {
	GetCookies(u *url.URL) []*http.Cookie
	SetCookies(u *url.URL, cookies []*http.Cookie)
	SetCookieJar(jar http.CookieJar)
	GetCookieJar() http.CookieJar
	SetProxy(proxyUrl string) error
	GetProxy() string
	SetFollowRedirect(followRedirect bool)
	GetFollowRedirect() bool
	CloseIdleConnections()
	Do(req *http.Request) (*http.Response, error)
	Get(url string) (resp *http.Response, err error)
	Head(url string) (resp *http.Response, err error)
	Post(url, contentType string, body io.Reader) (resp *http.Response, err error)

	GetBandwidthTracker() bandwidth.BandwidthTracker
}

func NewHttpClient 

func NewHttpClient(logger Logger, options ...HttpClientOption) (HttpClient, error)

NewHttpClient constructs a new HTTP client with the given logger and client options.

func ProvideDefaultClient 

func ProvideDefaultClient(logger Logger) (HttpClient, error)

type HttpClientOption 

type HttpClientOption func(config *httpClientConfig)

func WithBandwidthTracker added in v1.7.6 

func WithBandwidthTracker() HttpClientOption

WithBandwidthTracker configures a client to track the bandwidth used by the client.

func WithCatchPanics 

func WithCatchPanics() HttpClientOption

WithCatchPanics configures a client to catch all go panics happening during a request and not print the stacktrace.

func WithCertificatePinning 

func WithCertificatePinning(certificatePins map[string][]string, handlerFunc BadPinHandlerFunc) HttpClientOption

WithCertificatePinning enables SSL Pinning for the client and will throw an error if the SSL Pin is not matched. Please refer to https://github.com/tam7t/hpkp/#examples in order to see how to generate pins. The certificatePins are a map with the host as key. You can provide a BadPinHandlerFunc or nil as second argument. This function will be executed once a bad ssl pin is detected. BadPinHandlerFunc has to be defined like this: func(req *http.Request){}

func WithCharlesProxy 

func WithCharlesProxy(host string, port string) HttpClientOption

WithCharlesProxy configures the HTTP client to use a local running charles as proxy.

host and port can be empty, then default 127.0.0.1 and port 8888 will be used

func WithClientProfile 

func WithClientProfile(clientProfile profiles.ClientProfile) HttpClientOption

WithClientProfile configures a TLS client to use the specified client profile.

func WithConnectHeaders added in v1.7.7 

func WithConnectHeaders(headers http.Header) HttpClientOption

WithConnectHeaders configures a client to use the specified headers for the CONNECT request.

func WithCookieJar 

func WithCookieJar(jar http.CookieJar) HttpClientOption

WithCookieJar configures a HTTP client to use the specified cookie jar.

func WithCustomRedirectFunc 

func WithCustomRedirectFunc(redirectFunc func(req *http.Request, via []*http.Request) error) HttpClientOption

WithCustomRedirectFunc configures an HTTP client to use a custom redirect func. The redirect func have to look like that: func(req *http.Request, via []*http.Request) error Please only provide a custom redirect function if you know what you are doing. Check docs on net/http.Client CheckRedirect

func WithDebug 

func WithDebug() HttpClientOption

WithDebug configures a client to log debugging information.

func WithDefaultHeaders added in v1.6.0 

func WithDefaultHeaders(defaultHeaders http.Header) HttpClientOption

WithDefaultHeaders configures a TLS client to use a set of default headers if none are specified on the request.

func WithDialer added in v1.7.0 

func WithDialer(dialer net.Dialer) HttpClientOption

WithDialer configures an HTTP client to use the specified dialer. This allows the use of a custom DNS resolver

func WithDisableIPV4 added in v1.7.6 

func WithDisableIPV4() HttpClientOption

WithDisableIPV4 configures a dialer to use tcp6 network argument

func WithDisableIPV6 added in v1.4.0 

func WithDisableIPV6() HttpClientOption

WithDisableIPV6 configures a dialer to use tcp4 network argument

func WithForceHttp1 

func WithForceHttp1() HttpClientOption

WithForceHttp1 configures a client to force HTTP/1.1 as the used protocol.

func WithInsecureSkipVerify 

func WithInsecureSkipVerify() HttpClientOption

WithInsecureSkipVerify configures a client to skip SSL certificate verification.

func WithLocalAddr added in v1.4.0 

func WithLocalAddr(localAddr net.TCPAddr) HttpClientOption

WithLocalAddr configures an HTTP client to use the specified local address.

func WithNotFollowRedirects 

func WithNotFollowRedirects() HttpClientOption

WithNotFollowRedirects configures an HTTP client to not follow HTTP redirects.

func WithProxyUrl 

func WithProxyUrl(proxyUrl string) HttpClientOption

WithProxyUrl configures a HTTP client to use the specified proxy URL.

proxyUrl should be formatted as: 

"http://user:pass@host:port"

func WithRandomTLSExtensionOrder 

func WithRandomTLSExtensionOrder() HttpClientOption

WithRandomTLSExtensionOrder configures a TLS client to randomize the order of TLS extensions being sent in the ClientHello.

Placement of GREASE and padding is fixed and will not be affected by this.

func WithServerNameOverwrite 

func WithServerNameOverwrite(serverName string) HttpClientOption

WithServerNameOverwrite configures a TLS client to overwrite the server name being used for certificate verification and in the client hello. This option does only work properly if WithInsecureSkipVerify is set to true in addition

func WithTimeout 

func WithTimeout(timeout int) HttpClientOption

WithTimeout configures an HTTP client to use the specified request timeout.

timeout is the request timeout in seconds. Deprecated: use either WithTimeoutSeconds or WithTimeoutMilliseconds

func WithTimeoutMilliseconds 

func WithTimeoutMilliseconds(timeout int) HttpClientOption

WithTimeoutMilliseconds configures an HTTP client to use the specified request timeout.

timeout is the request timeout in milliseconds.

func WithTimeoutSeconds 

func WithTimeoutSeconds(timeout int) HttpClientOption

WithTimeoutSeconds configures an HTTP client to use the specified request timeout.

timeout is the request timeout in seconds.

func WithTransportOptions 

func WithTransportOptions(transportOptions *TransportOptions) HttpClientOption

WithTransportOptions configures a client to use the specified transport options.

type Logger 

type Logger interface {
	Debug(format string, args ...any)
	Info(format string, args ...any)
	Warn(format string, args ...any)
	Error(format string, args ...any)
}

func NewDebugLogger 

func NewDebugLogger(logger Logger) Logger

func NewLogger 

func NewLogger() Logger

func NewNoopLogger 

func NewNoopLogger() Logger

type TransportOptions 

type TransportOptions struct {
	// KeyLogWriter is an io.Writer that the TLS client will use to write the
	// TLS master secrets to. This can be used to decrypt TLS connections in
	// Wireshark and other applications.
	KeyLogWriter io.Writer
	// IdleConnTimeout is the maximum amount of time an idle (keep-alive)
	// connection will remain idle before closing itself. Zero means no limit.
	IdleConnTimeout *time.Duration
	// RootCAs is the set of root certificate authorities used to verify
	// the remote server's certificate.
	RootCAs                *x509.CertPool
	MaxIdleConns           int
	MaxIdleConnsPerHost    int
	MaxConnsPerHost        int
	MaxResponseHeaderBytes int64 // Zero means to use a default limit.
	WriteBufferSize        int   // If zero, a default (currently 4KB) is used.
	ReadBufferSize         int   // If zero, a default (currently 4KB) is used.
	DisableKeepAlives      bool
	DisableCompression     bool
}

Source Files

View all Source files

Directories

Path Synopsis
tls_client_cffi_src provides and manages a CFFI (C Foreign Function Interface) which allows code in other languages to interact with the module.
 tls_client_cffi_src provides and manages a CFFI (C Foreign Function Interface) which allows code in other languages to interact with the module.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.