tls_client

README

TLS-Client

Preface

This TLS Client is built upon https://github.com/Carcraftz/fhttp and https://github.com/Carcraftz/utls (https://github.com/refraction-networking/utls). Big thanks to all contributors so far. Sadly it seems that the original repositories from Carcraftz are not maintained anymore.

What is TLS Fingerprinting?

Some people think it is enough to change the user-agent header of a request to let the server think that the client requesting a resource is a specific browser. Nowadays this is not enough, because the server might use a technique to detect the client browser which is called TLS Fingerprinting.

Even though this article is about TLS Fingerprinting in NodeJS it well describes the technique in general. https://httptoolkit.tech/blog/tls-fingerprinting-node-js/#how-does-tls-fingerprinting-work

Why is this library needed?

With this library you are able to create a http client implementing an interface which is similar to golangs net/http client interface. This TLS Client allows you to specify the Client (Browser and Version) you want to use, when requesting a server.

The Interface of the HTTP Client looks like the following and extends the base net/http Client Interface by some useful functions. Most likely you will use the Do() function like you did before with net/http Client.

type HttpClient interface {
    GetCookies(u *url.URL) []*http.Cookie
    SetCookies(u *url.URL, cookies []*http.Cookie)
    SetCookieJar(jar http.CookieJar)
    GetCookieJar() http.CookieJar
    SetProxy(proxyUrl string) error
    GetProxy() string
    SetFollowRedirect(followRedirect bool)
    GetFollowRedirect() bool
    CloseIdleConnections()
    Do(req *http.Request) (*http.Response, error)
    Get(url string) (resp *http.Response, err error)
    Head(url string) (resp *http.Response, err error)
    Post(url, contentType string, body io.Reader) (resp *http.Response, err error)
}

Quick Usage Example

package main

import (
	"fmt"
	"io"
	"log"

	http "github.com/bogdanfinn/fhttp"
	tls_client "github.com/bogdanfinn/tls-client"
	"github.com/bogdanfinn/tls-client/profiles"
)

func main() {
    jar := tls_client.NewCookieJar()
	options := []tls_client.HttpClientOption{
		tls_client.WithTimeoutSeconds(30),
		tls_client.WithClientProfile(profiles.Chrome_120),
		tls_client.WithNotFollowRedirects(),
		tls_client.WithCookieJar(jar), // create cookieJar instance and pass it as argument
	}

	client, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...)
	if err != nil {
		log.Println(err)
		return
	}

	req, err := http.NewRequest(http.MethodGet, "https://tls.peet.ws/api/all", nil)
	if err != nil {
		log.Println(err)
		return
	}

	req.Header = http.Header{
		"accept":                    {"*/*"},
		"accept-language":           {"de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7"},
		"user-agent":                {"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"},
		http.HeaderOrderKey: {
			"accept",
			"accept-language",
			"user-agent",
		},
	}

	resp, err := client.Do(req)
	if err != nil {
		log.Println(err)
		return
	}

	defer resp.Body.Close()

	log.Println(fmt.Sprintf("status code: %d", resp.StatusCode))

	readBytes, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Println(err)
		return
	}

	log.Println(string(readBytes))
}

Detailed Documentation

https://bogdanfinn.gitbook.io/open-source-oasis/

Questions?

Join my discord support server for free: https://discord.gg/7Ej9eJvHqk No Support in DMs!

Appreciate my work?

Documentation

Index

• Variables
• func GetSpecFactoryFromJa3String(ja3String string, ...) (func() (tls.ClientHelloSpec, error), error)
• type BadPinHandlerFunc
• type CandidateCipherSuites
• type CertificatePinner
  • func NewCertificatePinner(certificatePins map[string][]string) (CertificatePinner, error)
• type ContextKeyHeader
• type CookieJar
  • func NewCookieJar(options ...CookieJarOption) CookieJar
• type CookieJarOption
  • func WithAllowEmptyCookies() CookieJarOption
  • func WithDebugLogger() CookieJarOption
  • func WithLogger(logger Logger) CookieJarOption
  • func WithSkipExisting() CookieJarOption
• type HttpClient
  • func NewHttpClient(logger Logger, options ...HttpClientOption) (HttpClient, error)
  • func ProvideDefaultClient(logger Logger) (HttpClient, error)
• type HttpClientOption
  • func WithBandwidthTracker() HttpClientOption
  • func WithCatchPanics() HttpClientOption
  • func WithCertificatePinning(certificatePins map[string][]string, handlerFunc BadPinHandlerFunc) HttpClientOption
  • func WithCharlesProxy(host string, port string) HttpClientOption
  • func WithClientProfile(clientProfile profiles.ClientProfile) HttpClientOption
  • func WithConnectHeaders(headers http.Header) HttpClientOption
  • func WithCookieJar(jar http.CookieJar) HttpClientOption
  • func WithCustomRedirectFunc(redirectFunc func(req *http.Request, via []*http.Request) error) HttpClientOption
  • func WithDebug() HttpClientOption
  • func WithDefaultHeaders(defaultHeaders http.Header) HttpClientOption
  • func WithDialer(dialer net.Dialer) HttpClientOption
  • func WithDisableIPV4() HttpClientOption
  • func WithDisableIPV6() HttpClientOption
  • func WithForceHttp1() HttpClientOption
  • func WithInsecureSkipVerify() HttpClientOption
  • func WithLocalAddr(localAddr net.TCPAddr) HttpClientOption
  • func WithNotFollowRedirects() HttpClientOption
  • func WithProxyUrl(proxyUrl string) HttpClientOption
  • func WithRandomTLSExtensionOrder() HttpClientOption
  • func WithServerNameOverwrite(serverName string) HttpClientOption
  • func WithTimeout(timeout int) HttpClientOption
  • func WithTimeoutMilliseconds(timeout int) HttpClientOption
  • func WithTimeoutSeconds(timeout int) HttpClientOption
  • func WithTransportOptions(transportOptions *TransportOptions) HttpClientOption
• type Logger
  • func NewDebugLogger(logger Logger) Logger
  • func NewLogger() Logger
  • func NewNoopLogger() Logger
• type TransportOptions

Variables

var DefaultBadPinHandler = func(req *http.Request) {
	fmt.Println("this is the default bad pin handler")
}

var DefaultOptions = []HttpClientOption{
	WithTimeoutSeconds(DefaultTimeoutSeconds),
	WithClientProfile(profiles.DefaultClientProfile),
	WithRandomTLSExtensionOrder(),
	WithNotFollowRedirects(),
}

var DefaultTimeoutSeconds = 30

var ErrBadPinDetected = errors.New("bad ssl pin detected")

var H2SettingsMap = map[string]http2.SettingID{
	"HEADER_TABLE_SIZE":      http2.SettingHeaderTableSize,
	"ENABLE_PUSH":            http2.SettingEnablePush,
	"MAX_CONCURRENT_STREAMS": http2.SettingMaxConcurrentStreams,
	"INITIAL_WINDOW_SIZE":    http2.SettingInitialWindowSize,
	"MAX_FRAME_SIZE":         http2.SettingMaxFrameSize,
	"MAX_HEADER_LIST_SIZE":   http2.SettingMaxHeaderListSize,
	"UNKNOWN_SETTING_7":      0x7,
	"UNKNOWN_SETTING_8":      0x8,
	"UNKNOWN_SETTING_9":      0x9,
}

Functions

func GetSpecFactoryFromJa3String

func GetSpecFactoryFromJa3String(ja3String string, supportedSignatureAlgorithms, supportedDelegatedCredentialsAlgorithms, supportedVersions, keyShareCurves, supportedProtocolsALPN, supportedProtocolsALPS []string, echCandidateCipherSuites []CandidateCipherSuites, candidatePayloads []uint16, certCompressionAlgo string) (func() (tls.ClientHelloSpec, error), error)

Types

type BadPinHandlerFunc

type BadPinHandlerFunc func(req *http.Request)

type CandidateCipherSuites

type CandidateCipherSuites struct {
	KdfId  string
	AeadId string
}

type CertificatePinner

type CertificatePinner interface {
	Pin(conn *tls.UConn, host string) error
}

func NewCertificatePinner

func NewCertificatePinner(certificatePins map[string][]string) (CertificatePinner, error)

type ContextKeyHeader

type ContextKeyHeader struct{}

Users of context.WithValue should define their own types for keys

type CookieJar

type CookieJar interface {
	http.CookieJar
	GetAllCookies() map[string][]*http.Cookie
}

func NewCookieJar

func NewCookieJar(options ...CookieJarOption) CookieJar

type CookieJarOption

type CookieJarOption func(config *cookieJarConfig)

func WithAllowEmptyCookies

func WithAllowEmptyCookies() CookieJarOption

func WithDebugLogger

func WithDebugLogger() CookieJarOption

func WithLogger

func WithLogger(logger Logger) CookieJarOption

func WithSkipExisting

func WithSkipExisting() CookieJarOption

type HttpClient

type HttpClient interface {
	GetCookies(u *url.URL) []*http.Cookie
	SetCookies(u *url.URL, cookies []*http.Cookie)
	SetCookieJar(jar http.CookieJar)
	GetCookieJar() http.CookieJar
	SetProxy(proxyUrl string) error
	GetProxy() string
	SetFollowRedirect(followRedirect bool)
	GetFollowRedirect() bool
	CloseIdleConnections()
	Do(req *http.Request) (*http.Response, error)
	Get(url string) (resp *http.Response, err error)
	Head(url string) (resp *http.Response, err error)
	Post(url, contentType string, body io.Reader) (resp *http.Response, err error)

	GetBandwidthTracker() bandwidth.BandwidthTracker
}

func NewHttpClient

func NewHttpClient(logger Logger, options ...HttpClientOption) (HttpClient, error)

NewHttpClient constructs a new HTTP client with the given logger and client options.

func ProvideDefaultClient

func ProvideDefaultClient(logger Logger) (HttpClient, error)

type HttpClientOption

type HttpClientOption func(config *httpClientConfig)

func WithBandwidthTracker

func WithBandwidthTracker() HttpClientOption

WithBandwidthTracker configures a client to track the bandwidth used by the client.

func WithCatchPanics

func WithCatchPanics() HttpClientOption

WithCatchPanics configures a client to catch all go panics happening during a request and not print the stacktrace.

func WithCertificatePinning

func WithCertificatePinning(certificatePins map[string][]string, handlerFunc BadPinHandlerFunc) HttpClientOption

WithCertificatePinning enables SSL Pinning for the client and will throw an error if the SSL Pin is not matched. Please refer to https://github.com/tam7t/hpkp/#examples in order to see how to generate pins. The certificatePins are a map with the host as key. You can provide a BadPinHandlerFunc or nil as second argument. This function will be executed once a bad ssl pin is detected. BadPinHandlerFunc has to be defined like this: func(req *http.Request){}

func WithCharlesProxy

func WithCharlesProxy(host string, port string) HttpClientOption

WithCharlesProxy configures the HTTP client to use a local running charles as proxy.

host and port can be empty, then default 127.0.0.1 and port 8888 will be used

func WithClientProfile

func WithClientProfile(clientProfile profiles.ClientProfile) HttpClientOption

WithClientProfile configures a TLS client to use the specified client profile.

func WithConnectHeaders

func WithConnectHeaders(headers http.Header) HttpClientOption

WithConnectHeaders configures a client to use the specified headers for the CONNECT request.

func WithCookieJar

func WithCookieJar(jar http.CookieJar) HttpClientOption

WithCookieJar configures a HTTP client to use the specified cookie jar.

func WithCustomRedirectFunc

func WithCustomRedirectFunc(redirectFunc func(req *http.Request, via []*http.Request) error) HttpClientOption

WithCustomRedirectFunc configures an HTTP client to use a custom redirect func. The redirect func have to look like that: func(req *http.Request, via []*http.Request) error Please only provide a custom redirect function if you know what you are doing. Check docs on net/http.Client CheckRedirect

func WithDebug

func WithDebug() HttpClientOption

WithDebug configures a client to log debugging information.

func WithDefaultHeaders

func WithDefaultHeaders(defaultHeaders http.Header) HttpClientOption

WithDefaultHeaders configures a TLS client to use a set of default headers if none are specified on the request.

func WithDialer

func WithDialer(dialer net.Dialer) HttpClientOption

WithDialer configures an HTTP client to use the specified dialer. This allows the use of a custom DNS resolver

func WithDisableIPV4

func WithDisableIPV4() HttpClientOption

WithDisableIPV4 configures a dialer to use tcp6 network argument

func WithDisableIPV6

func WithDisableIPV6() HttpClientOption

WithDisableIPV6 configures a dialer to use tcp4 network argument

func WithForceHttp1

func WithForceHttp1() HttpClientOption

WithForceHttp1 configures a client to force HTTP/1.1 as the used protocol.

func WithInsecureSkipVerify

func WithInsecureSkipVerify() HttpClientOption

WithInsecureSkipVerify configures a client to skip SSL certificate verification.

func WithLocalAddr

func WithLocalAddr(localAddr net.TCPAddr) HttpClientOption

WithLocalAddr configures an HTTP client to use the specified local address.

func WithNotFollowRedirects

func WithNotFollowRedirects() HttpClientOption

WithNotFollowRedirects configures an HTTP client to not follow HTTP redirects.

func WithProxyUrl

func WithProxyUrl(proxyUrl string) HttpClientOption

WithProxyUrl configures a HTTP client to use the specified proxy URL.

proxyUrl should be formatted as:

"http://user:pass@host:port"

func WithRandomTLSExtensionOrder

func WithRandomTLSExtensionOrder() HttpClientOption

WithRandomTLSExtensionOrder configures a TLS client to randomize the order of TLS extensions being sent in the ClientHello.

Placement of GREASE and padding is fixed and will not be affected by this.

func WithServerNameOverwrite

func WithServerNameOverwrite(serverName string) HttpClientOption

WithServerNameOverwrite configures a TLS client to overwrite the server name being used for certificate verification and in the client hello. This option does only work properly if WithInsecureSkipVerify is set to true in addition

func WithTimeout

func WithTimeout(timeout int) HttpClientOption

WithTimeout configures an HTTP client to use the specified request timeout.

timeout is the request timeout in seconds. Deprecated: use either WithTimeoutSeconds or WithTimeoutMilliseconds

func WithTimeoutMilliseconds

func WithTimeoutMilliseconds(timeout int) HttpClientOption

WithTimeoutMilliseconds configures an HTTP client to use the specified request timeout.

timeout is the request timeout in milliseconds.

func WithTimeoutSeconds

func WithTimeoutSeconds(timeout int) HttpClientOption

WithTimeoutSeconds configures an HTTP client to use the specified request timeout.

timeout is the request timeout in seconds.

func WithTransportOptions

func WithTransportOptions(transportOptions *TransportOptions) HttpClientOption

WithTransportOptions configures a client to use the specified transport options.

type Logger

type Logger interface {
	Debug(format string, args ...any)
	Info(format string, args ...any)
	Warn(format string, args ...any)
	Error(format string, args ...any)
}

func NewDebugLogger

func NewDebugLogger(logger Logger) Logger

func NewLogger

func NewLogger() Logger

func NewNoopLogger

func NewNoopLogger() Logger

type TransportOptions

type TransportOptions struct {
	// KeyLogWriter is an io.Writer that the TLS client will use to write the
	// TLS master secrets to. This can be used to decrypt TLS connections in
	// Wireshark and other applications.
	KeyLogWriter io.Writer
	// IdleConnTimeout is the maximum amount of time an idle (keep-alive)
	// connection will remain idle before closing itself. Zero means no limit.
	IdleConnTimeout *time.Duration
	// RootCAs is the set of root certificate authorities used to verify
	// the remote server's certificate.
	RootCAs                *x509.CertPool
	MaxIdleConns           int
	MaxIdleConnsPerHost    int
	MaxConnsPerHost        int
	MaxResponseHeaderBytes int64 // Zero means to use a default limit.
	WriteBufferSize        int   // If zero, a default (currently 4KB) is used.
	ReadBufferSize         int   // If zero, a default (currently 4KB) is used.
	DisableKeepAlives      bool
	DisableCompression     bool
}