httpx

Documentation

Overview

Package httpx containst the httpx common funcionality

Index

• Constants
• Variables
• func Decodebig5(s []byte) ([]byte, error)
• func Decodegbk(s []byte) ([]byte, error)
• func Encodebig5(s []byte) ([]byte, error)
• func ExtractTitle(r *Response) (title string)
• type CSPData
• type ChainItem
• type CustomCallback
• type Filter
• type FilterCustom
  • func (f FilterCustom) Filter(response *Response) (bool, error)
• type FilterRegex
  • func (f FilterRegex) Filter(response *Response) (bool, error)
• type FilterString
  • func (f FilterString) Filter(response *Response) (bool, error)
• type HTTPX
  • func New(options *Options) (*HTTPX, error)
  • func (h *HTTPX) AddFilter(f Filter)
  • func (h *HTTPX) CSPGrab(r *http.Response) *CSPData
  • func (h *HTTPX) CdnCheck(ip string) (bool, error)
  • func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error)
  • func (h *HTTPX) IsVirtualHost(req *retryablehttp.Request) (bool, error)
  • func (h *HTTPX) NewRequest(method, targetURL string) (req *retryablehttp.Request, err error)
  • func (h *HTTPX) SetCustomHeaders(r *retryablehttp.Request, headers map[string]string)
  • func (h *HTTPX) SupportHTTP2(protocol, method, targetURL string) bool
  • func (h *HTTPX) SupportPipeline(protocol, method, host string, port int) bool
  • func (h *HTTPX) TLSGrab(r *http.Response) *TLSData
  • func (h *HTTPX) Verify(req *retryablehttp.Request) (bool, error)
• type Options
• type RequestOverride
• type Response
  • func (r *Response) GetChain() string
  • func (r *Response) GetChainAsSlice() (chain []ChainItem)
  • func (r *Response) GetChainStatusCodes() []int
  • func (r *Response) GetHeader(name string) string
  • func (r *Response) GetHeaderPart(name, sep string) string
  • func (r *Response) HasChain() bool
• type TLSData

Constants

const (
	// HTTP defines the plain http scheme
	HTTP = "http"
	// HTTPS defines the secure http scheme
	HTTPS = "https"
	// HTTPorHTTPS defines the both http and https scheme
	HTTPorHTTPS = "http|https"
)

Variables

var CSPHeaders []string = []string{
	"Content-Security-Policy",
	"Content-Security-Policy-Report-Only",
	"X-Content-Security-Policy-Report-Only",
	"X-Webkit-Csp-Report-Only",
}

CSPHeaders is an incomplete list of most common CSP headers

var DefaultOptions = Options{
	RandomAgent: true,
	Threads:     25,
	Timeout:     30 * time.Second,
	RetryMax:    5,
	Unsafe:      false,
	CdnCheck:    true,

	VHostIgnoreStatusCode:    false,
	VHostIgnoreContentLength: true,
	VHostIgnoreNumberOfWords: false,
	VHostIgnoreNumberOfLines: false,
	VHostStripHTML:           false,
	VHostSimilarityRatio:     85,
	DefaultUserAgent:         "httpx - Open-source project (github.com/projectdiscovery/httpx)",
}

DefaultOptions contains the default options

Functions

func Decodebig5

func Decodebig5(s []byte) ([]byte, error)

Decodebig5 converts BIG5 to UTF-8

func Decodegbk

func Decodegbk(s []byte) ([]byte, error)

Decodegbk converts GBK to UTF-8

func Encodebig5

func Encodebig5(s []byte) ([]byte, error)

Encodebig5 converts UTF-8 to BIG5

func ExtractTitle

func ExtractTitle(r *Response) (title string)

ExtractTitle from a response

Types

type CSPData

type CSPData struct {
	Domains []string `json:"domains,omitempty"`
}

CSPData contains the Content-Security-Policy domain list

type ChainItem

type ChainItem struct {
	Request    string `json:"request,omitempty"`
	Response   string `json:"response,omitempty"`
	StatusCode int    `json:"status_code,omitempty"`
}

ChainItem request=>response

type CustomCallback

type CustomCallback func(response *Response) (bool, error)

CustomCallback used in custom filters

type Filter

type Filter interface {
	Filter(response *Response) (bool, error)
}

Filter defines a generic filter interface to apply to responses

type FilterCustom

type FilterCustom struct {
	CallBacks []CustomCallback
}

FilterCustom defines a filter with callback functions applied

func (FilterCustom) Filter

func (f FilterCustom) Filter(response *Response) (bool, error)

Filter a response with custom callbacks

type FilterRegex

type FilterRegex struct {
	Regexs []string
}

FilterRegex defines a filter of type regex

func (FilterRegex) Filter

func (f FilterRegex) Filter(response *Response) (bool, error)

Filter a response with regexes

type FilterString

type FilterString struct {
	Keywords []string
}

FilterString defines a filter of type string

func (FilterString) Filter

func (f FilterString) Filter(response *Response) (bool, error)

Filter a response with strings filtering

type HTTPX

type HTTPX struct {
	Filters []Filter
	Options *Options

	CustomHeaders   map[string]string
	RequestOverride *RequestOverride

	Dialer *fastdialer.Dialer
	// contains filtered or unexported fields
}

HTTPX represent an instance of the library client

func New

func New(options *Options) (*HTTPX, error)

New httpx instance

func (*HTTPX) AddFilter

func (h *HTTPX) AddFilter(f Filter)

AddFilter cascade

func (*HTTPX) CSPGrab

func (h *HTTPX) CSPGrab(r *http.Response) *CSPData

CSPGrab fills the CSPData

func (*HTTPX) CdnCheck

func (h *HTTPX) CdnCheck(ip string) (bool, error)

CdnCheck verifies if the given ip is part of Cdn ranges

func (*HTTPX) Do

func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error)

Do http request

func (*HTTPX) IsVirtualHost

func (h *HTTPX) IsVirtualHost(req *retryablehttp.Request) (bool, error)

IsVirtualHost checks if the target endpoint is a virtual host

func (*HTTPX) NewRequest

func (h *HTTPX) NewRequest(method, targetURL string) (req *retryablehttp.Request, err error)

NewRequest from url

func (*HTTPX) SetCustomHeaders

func (h *HTTPX) SetCustomHeaders(r *retryablehttp.Request, headers map[string]string)

SetCustomHeaders on the provided request

func (*HTTPX) SupportHTTP2

func (h *HTTPX) SupportHTTP2(protocol, method, targetURL string) bool

SupportHTTP2 checks if the target host supports HTTP2

func (*HTTPX) SupportPipeline

func (h *HTTPX) SupportPipeline(protocol, method, host string, port int) bool

SupportPipeline checks if the target host supports HTTP1.1 pipelining by sending x probes and reading back responses expecting at least 2 with HTTP/1.1 or HTTP/1.0

func (*HTTPX) TLSGrab

func (h *HTTPX) TLSGrab(r *http.Response) *TLSData

TLSGrab fills the TLSData

func (*HTTPX) Verify

func (h *HTTPX) Verify(req *retryablehttp.Request) (bool, error)

Verify the http calls and apply-cascade all the filters, as soon as one matches it returns true

type Options

type Options struct {
	RandomAgent      bool
	DefaultUserAgent string
	RequestOverride  RequestOverride
	HTTPProxy        string
	SocksProxy       string
	Threads          int
	CdnCheck         bool
	// Timeout is the maximum time to wait for the request
	Timeout time.Duration
	// RetryMax is the maximum number of retries
	RetryMax      int
	CustomHeaders map[string]string
	// VHostSimilarityRatio 1 - 100
	VHostSimilarityRatio int
	FollowRedirects      bool
	FollowHostRedirects  bool
	Unsafe               bool
	TLSGrab              bool
	// VHOSTs options
	VHostIgnoreStatusCode    bool
	VHostIgnoreContentLength bool
	VHostIgnoreNumberOfWords bool
	VHostIgnoreNumberOfLines bool
	VHostStripHTML           bool
	Allow                    []string
	Deny                     []string
}

Options contains configuration options for the client

type RequestOverride

type RequestOverride struct {
	URIPath string
}

RequestOverride contains the URI path to override the request

type Response

type Response struct {
	StatusCode    int
	Headers       map[string][]string
	Data          []byte
	ContentLength int
	Raw           string
	RawHeaders    string
	Words         int
	Lines         int
	TLSData       *TLSData
	CSPData       *CSPData
	HTTP2         bool
	Pipeline      bool
	Duration      time.Duration
	Chain         []httputil.ChainItem
}

Response contains the response to a server

func (*Response) GetChain

func (r *Response) GetChain() string

GetChain dump the whole redirect chain as string

func (*Response) GetChainAsSlice

func (r *Response) GetChainAsSlice() (chain []ChainItem)

GetChainAsSlice dump the whole redirect chain as structuerd slice

func (*Response) GetChainStatusCodes

func (r *Response) GetChainStatusCodes() []int

GetChainStatusCodes from redirects

func (*Response) GetHeader

func (r *Response) GetHeader(name string) string

GetHeader value

func (*Response) GetHeaderPart

func (r *Response) GetHeaderPart(name, sep string) string

GetHeaderPart with offset

func (*Response) HasChain

func (r *Response) HasChain() bool

HasChain redirects

type TLSData

type TLSData struct {
	DNSNames         []string `json:"dns_names,omitempty"`
	Emails           []string `json:"emails,omitempty"`
	CommonName       []string `json:"common_name,omitempty"`
	Organization     []string `json:"organization,omitempty"`
	IssuerCommonName []string `json:"issuer_common_name,omitempty"`
	IssuerOrg        []string `json:"issuer_organization,omitempty"`
}

TLSData contains the relevant Transport Layer Security information