proxy

Documentation

Index

• Constants
• Variables
• func GenCA(name string) (certPEM, keyPEM []byte, err error)
• func GenCert(names []string) (*tls.Certificate, error)
• func HandleHTTP(sw http.ResponseWriter, sreq *http.Request) error
• func HandleTunneling(sw http.ResponseWriter, sreq *http.Request) error
• func Handshake(w http.ResponseWriter, config *tls.Config) (net.Conn, error)
• func LoadOrGenCAFiles(caCertFile, caKeyFile string) (err error)
• func LoadProxyFile(path string) (count int, urls []string, err error)
• func NewPageFromRequest(r *http.Request, scheme string, pconf *PageConfig) (p *pages.Page, err error)
• type PageConfig

Constants

const UATypeHeader = "X-DH-UA-Type"

UATypeHeader is the custom header that the scraper calls till to set the user agent type

Variables

var (
	// Token is the Till auth token
	Token string

	// InstanceName is the name of this till instance
	InstanceName string

	// ForceUA indicates whether to overwrite all incoming user-agent with a random one
	ForceUA = true

	// UAType specifies what kind of user-agent to generate
	UAType = "desktop"

	// ProxyFile points to the path of the txt file that contains a list of proxies
	ProxyFile = ""

	// ProxyURLs are external proxies that will be randomized
	ProxyURLs = []string{}

	// ProxyCount is the total count of proxies used.
	ProxyCount int

	// ReleaseVersion is the version of Till release
	ReleaseVersion = "dev"

	StatMu *tillclient.InstanceStatMutex

	// Cache is the cache specific config
	CacheConfig cache.Config

	// LoggerConfig is the logger specific config
	LoggerConfig logger.Config

	// SessionsConfig is the sessions specific config
	SessionsConfig sessions.Config
)

Functions

func GenCA

func GenCA(name string) (certPEM, keyPEM []byte, err error)

func GenCert

func GenCert(names []string) (*tls.Certificate, error)

GenCert generates cert

func HandleHTTP

func HandleHTTP(sw http.ResponseWriter, sreq *http.Request) error

HandleHTTP proxies the request from source to target

func HandleTunneling

func HandleTunneling(sw http.ResponseWriter, sreq *http.Request) error

func Handshake

func Handshake(w http.ResponseWriter, config *tls.Config) (net.Conn, error)

Handshake hijacks w's underlying net.Conn, responds to the CONNECT request and manually performs the TLS handshake. It returns the net.Conn or and error if any.

func LoadOrGenCAFiles

func LoadOrGenCAFiles(caCertFile, caKeyFile string) (err error)

LoadOrGenCAFiles loads CA from file, or generates it into a file and use it

func LoadProxyFile

func LoadProxyFile(path string) (count int, urls []string, err error)

LoadProxyFile will load the file

func NewPageFromRequest

func NewPageFromRequest(r *http.Request, scheme string, pconf *PageConfig) (p *pages.Page, err error)

Types

type PageConfig

type PageConfig struct {
	ForceUA  bool   // if true, overrides the User-Agent header
	UaType   string // default to "desktop"
	UseProxy bool

	// StickySession features
	SessionID     string
	StickyCookies bool
	StickyUA      bool

	// Interceptions feature
	IgnoreInterceptions    []string
	IgnoreAllInterceptions bool

	// Cache feature
	CacheFreshness     freshness.Type
	CacheServeFailures bool
}

PageConfig is where the page configuration is set