runner

package
v1.6.10 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 27, 2025 License: MIT Imports: 86 Imported by: 36

Documentation

Overview

TODO: move this to internal package

Package runner executes the enumeration process.

Index

Constants

View Source 
const (
	DefaultResumeFile      = "resume.cfg"
	DefaultOutputDirectory = "output"
)
View Source 
const Version = `v1.6.10`

Version is the current Version of httpx

Variables

View Source 
var (
	PDCPApiKey = ""
	TeamIDEnv  = env.GetEnvOrDefault("PDCP_TEAM_ID", "")
)

Functions

func AuthWithPDCP added in v1.5.0 

func AuthWithPDCP()

AuthWithPDCP is used to authenticate with PDCP

func DoHealthCheck added in v1.2.3 

func DoHealthCheck(options *Options, flagSet *goflags.FlagSet) string

func GetUpdateCallback added in v1.2.9 

func GetUpdateCallback() func()

GetUpdateCallback returns a callback function that updates httpx

func MustDisableSandbox added in v1.3.0 

func MustDisableSandbox() bool

MustDisableSandbox determines if the current os and user needs sandbox mode disabled

Types

type AsnResponse added in v1.2.1 

type AsnResponse struct {
	AsNumber  string   `json:"as_number" csv:"as_number"`
	AsName    string   `json:"as_name" csv:"as_name"`
	AsCountry string   `json:"as_country" csv:"as_country"`
	AsRange   []string `json:"as_range" csv:"as_range"`
}

func (AsnResponse) String added in v1.2.1 

func (o AsnResponse) String() string

type Browser added in v1.3.0 

type Browser struct {
	// contains filtered or unexported fields
}

func NewBrowser added in v1.3.0 

func NewBrowser(proxy string, useLocal bool, optionalArgs map[string]string) (*Browser, error)

func (*Browser) Close added in v1.3.0 

func (b *Browser) Close()

func (*Browser) ScreenshotWithBody added in v1.3.0 

func (b *Browser) ScreenshotWithBody(url string, timeout time.Duration, idle time.Duration, headers []string) ([]byte, string, error)

type Concurrency added in v1.6.4 

type Concurrency struct {
	Threads int `json:"threads"`
}

type FilterOperator added in v1.2.2 

type FilterOperator struct {
	// contains filtered or unexported fields
}

func (FilterOperator) Parse added in v1.2.2 

func (f FilterOperator) Parse(flagValue string) (string, time.Duration, error)

Parse the given value into operator and value pair

type OnResultCallback added in v1.2.5 

type OnResultCallback func(Result)

OnResultCallback (hostResult)

type Options 

type Options struct {
	CustomHeaders customheader.CustomHeaders
	CustomPorts   customport.CustomPorts

	Output           string
	OutputAll        bool
	StoreResponseDir string
	OmitBody         bool
	// Deprecated: use Proxy
	HTTPProxy string
	// Deprecated: use Proxy
	SocksProxy      string
	Proxy           string
	InputFile       string
	InputTargetHost goflags.StringSlice
	Methods         string
	RequestURI      string
	RequestURIs     string

	OutputMatchStatusCode     string
	OutputMatchContentLength  string
	OutputFilterStatusCode    string
	OutputFilterErrorPage     bool
	FilterOutDuplicates       bool
	OutputFilterContentLength string
	InputRawRequest           string

	RequestBody        string
	OutputFilterString goflags.StringSlice
	OutputMatchString  goflags.StringSlice
	OutputFilterRegex  goflags.StringSlice
	OutputMatchRegex   goflags.StringSlice
	Retries            int
	Threads            int
	Timeout            int
	Delay              time.Duration

	VHost                   bool
	VHostInput              bool
	Smuggling               bool
	ExtractTitle            bool
	StatusCode              bool
	Location                bool
	ContentLength           bool
	FollowRedirects         bool
	RespectHSTS             bool
	StoreResponse           bool
	JSONOutput              bool
	CSVOutput               bool
	CSVOutputEncoding       string
	PdcpAuth                string
	PdcpAuthCredFile        string
	Silent                  bool
	Version                 bool
	Verbose                 bool
	NoColor                 bool
	OutputServerHeader      bool
	OutputWebSocket         bool
	ResponseHeadersInStdout bool
	ResponseInStdout        bool
	Base64ResponseInStdout  bool
	ChainInStdout           bool
	FollowHostRedirects     bool
	MaxRedirects            int
	OutputMethod            bool
	TLSProbe                bool
	CSPProbe                bool
	OutputContentType       bool
	OutputIP                bool
	OutputCName             bool
	ExtractFqdn             bool
	Unsafe                  bool
	Debug                   bool
	DebugRequests           bool
	DebugResponse           bool
	Pipeline                bool
	HTTP2Probe              bool
	OutputCDN               string
	OutputResponseTime      bool
	NoFallback              bool
	NoFallbackScheme        bool
	TechDetect              bool
	TLSGrab                 bool

	ShowStatistics            bool
	StatsInterval             int
	RandomAgent               bool
	StoreChain                bool
	StoreVisionReconClusters  bool
	Deny                      customlist.CustomList
	Allow                     customlist.CustomList
	MaxResponseBodySizeToSave int
	MaxResponseBodySizeToRead int
	ResponseBodyPreviewSize   int
	OutputExtractRegexs       goflags.StringSlice
	OutputExtractPresets      goflags.StringSlice
	RateLimit                 int
	RateLimitMinute           int
	Probe                     bool
	Resume                    bool

	Exclude               goflags.StringSlice
	HostMaxErrors         int
	Stream                bool
	SkipDedupe            bool
	ProbeAllIPS           bool
	Resolvers             goflags.StringSlice
	Favicon               bool
	OutputFilterFavicon   goflags.StringSlice
	OutputMatchFavicon    goflags.StringSlice
	LeaveDefaultPorts     bool
	ZTLS                  bool
	OutputLinesCount      bool
	OutputMatchLinesCount string

	OutputFilterLinesCount string
	Memprofile             string

	OutputWordsCount      bool
	OutputMatchWordsCount string

	OutputFilterWordsCount string

	Hashes                   string
	Jarm                     bool
	Asn                      bool
	OutputMatchCdn           goflags.StringSlice
	OutputFilterCdn          goflags.StringSlice
	SniName                  string
	OutputMatchResponseTime  string
	OutputFilterResponseTime string
	HealthCheck              bool
	ListDSLVariable          bool
	OutputFilterCondition    string
	OutputMatchCondition     string
	StripFilter              string
	//The OnResult callback function is invoked for each result. It is important to check for errors in the result before using Result.Err.
	OnResult           OnResultCallback
	DisableUpdateCheck bool
	NoDecode           bool
	Screenshot         bool
	UseInstalledChrome bool
	TlsImpersonate     bool
	DisableStdin       bool
	HttpApiEndpoint    string
	NoScreenshotBytes  bool
	NoHeadlessBody     bool
	ScreenshotTimeout  time.Duration
	ScreenshotIdle     time.Duration
	// HeadlessOptionalArguments specifies optional arguments to pass to Chrome
	HeadlessOptionalArguments goflags.StringSlice
	Protocol                  string
	OutputFilterErrorPagePath string
	DisableStdout             bool
	// AssetUpload
	AssetUpload bool
	// AssetName
	AssetName string
	// AssetID
	AssetID string
	// AssetFileUpload
	AssetFileUpload string
	TeamID          string
	// OnClose adds a callback function that is invoked when httpx is closed
	// to be exact at end of existing closures
	OnClose func()

	Trace bool

	// Optional pre-created objects to reduce allocations
	Wappalyzer     *wappalyzer.Wappalyze
	Networkpolicy  *networkpolicy.NetworkPolicy
	CDNCheckClient *cdncheck.Client
	// contains filtered or unexported fields
}

Options contains configuration options for httpx.

func ParseOptions 

func ParseOptions() *Options

ParseOptions parses the command line options for application

func (*Options) ParseHeadlessOptionalArguments added in v1.3.8 

func (options *Options) ParseHeadlessOptionalArguments() map[string]string

redundant with katana

func (*Options) ShouldLoadResume added in v1.1.1 

func (options *Options) ShouldLoadResume() bool

ShouldLoadResume resume file

func (*Options) ShouldSaveResume added in v1.1.1 

func (options *Options) ShouldSaveResume() bool

ShouldSaveResume file

func (*Options) ValidateOptions added in v1.2.3 

func (options *Options) ValidateOptions() error

type Result 

type Result struct {
	Timestamp    time.Time              `json:"timestamp,omitempty" csv:"timestamp" mapstructure:"timestamp"`
	ASN          *AsnResponse           `json:"asn,omitempty" csv:"asn" mapstructure:"asn"`
	Err          error                  `json:"-" csv:"-" mapstructure:"-"`
	CSPData      *httpx.CSPData         `json:"csp,omitempty" csv:"csp" mapstructure:"csp"`
	TLSData      *clients.Response      `json:"tls,omitempty" csv:"tls" mapstructure:"tls"`
	Hashes       map[string]interface{} `json:"hash,omitempty" csv:"hash" mapstructure:"hash"`
	ExtractRegex []string               `json:"extract_regex,omitempty" csv:"extract_regex" mapstructure:"extract_regex"`
	CDNName      string                 `json:"cdn_name,omitempty" csv:"cdn_name" mapstructure:"cdn_name"`
	CDNType      string                 `json:"cdn_type,omitempty" csv:"cdn_type" mapstructure:"cdn_type"`
	SNI          string                 `json:"sni,omitempty" csv:"sni" mapstructure:"sni"`
	Port         string                 `json:"port,omitempty" csv:"port" mapstructure:"port"`
	Raw          string                 `json:"-" csv:"-" mapstructure:"-"`
	URL          string                 `json:"url,omitempty" csv:"url" mapstructure:"url"`
	Input        string                 `json:"input,omitempty" csv:"input" mapstructure:"input"`
	Location     string                 `json:"location,omitempty" csv:"location" mapstructure:"location"`
	Title        string                 `json:"title,omitempty" csv:"title" mapstructure:"title"`

	Scheme             string                        `json:"scheme,omitempty" csv:"scheme" mapstructure:"scheme"`
	Error              string                        `json:"error,omitempty" csv:"error" mapstructure:"error"`
	WebServer          string                        `json:"webserver,omitempty" csv:"webserver" mapstructure:"webserver"`
	ResponseBody       string                        `json:"body,omitempty" csv:"-" mapstructure:"body"`
	BodyPreview        string                        `json:"body_preview,omitempty" csv:"body_preview" mapstructure:"body_preview"`
	ContentType        string                        `json:"content_type,omitempty" csv:"content_type" mapstructure:"content_type"`
	Method             string                        `json:"method,omitempty" csv:"method" mapstructure:"method"`
	Host               string                        `json:"host,omitempty" csv:"host" mapstructure:"host"`
	Path               string                        `json:"path,omitempty" csv:"path" mapstructure:"path"`
	FavIconMMH3        string                        `json:"favicon,omitempty" csv:"favicon" mapstructure:"favicon"`
	FavIconMD5         string                        `json:"favicon_md5,omitempty" csv:"favicon_md5" mapstructure:"favicon_md5"`
	FaviconPath        string                        `json:"favicon_path,omitempty" csv:"favicon_path" mapstructure:"favicon_path"`
	FaviconURL         string                        `json:"favicon_url,omitempty" csv:"favicon_url" mapstructure:"favicon_url"`
	FinalURL           string                        `json:"final_url,omitempty" csv:"final_url" mapstructure:"final_url"`
	ResponseHeaders    map[string]interface{}        `json:"header,omitempty" csv:"-" mapstructure:"header"`
	RawHeaders         string                        `json:"raw_header,omitempty" csv:"-" mapstructure:"raw_header"`
	Request            string                        `json:"request,omitempty" csv:"-" mapstructure:"request"`
	ResponseTime       string                        `json:"time,omitempty" csv:"time" mapstructure:"time"`
	JarmHash           string                        `json:"jarm_hash,omitempty" csv:"jarm_hash" mapstructure:"jarm_hash"`
	ChainStatusCodes   []int                         `json:"chain_status_codes,omitempty" csv:"chain_status_codes" mapstructure:"chain_status_codes"`
	A                  []string                      `json:"a,omitempty" csv:"a" mapstructure:"a"`
	AAAA               []string                      `json:"aaaa,omitempty" csv:"aaaa" mapstructure:"aaaa"`
	CNAMEs             []string                      `json:"cname,omitempty" csv:"cname" mapstructure:"cname"`
	Technologies       []string                      `json:"tech,omitempty" csv:"tech" mapstructure:"tech"`
	Extracts           map[string][]string           `json:"extracts,omitempty" csv:"extracts" mapstructure:"extracts"`
	Chain              []httpx.ChainItem             `json:"chain,omitempty" csv:"chain" mapstructure:"chain"`
	Words              int                           `json:"words" csv:"words" mapstructure:"words"`
	Lines              int                           `json:"lines" csv:"lines" mapstructure:"lines"`
	StatusCode         int                           `json:"status_code" csv:"status_code" mapstructure:"status_code"`
	ContentLength      int                           `json:"content_length" csv:"content_length" mapstructure:"content_length"`
	Failed             bool                          `json:"failed" csv:"failed" mapstructure:"failed"`
	VHost              bool                          `json:"vhost,omitempty" csv:"vhost" mapstructure:"vhost"`
	WebSocket          bool                          `json:"websocket,omitempty" csv:"websocket" mapstructure:"websocket"`
	CDN                bool                          `json:"cdn,omitempty" csv:"cdn" mapstructure:"cdn"`
	HTTP2              bool                          `json:"http2,omitempty" csv:"http2" mapstructure:"http2"`
	Pipeline           bool                          `json:"pipeline,omitempty" csv:"pipeline" mapstructure:"pipeline"`
	HeadlessBody       string                        `json:"headless_body,omitempty" csv:"headless_body" mapstructure:"headless_body"`
	ScreenshotBytes    []byte                        `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes" mapstructure:"screenshot_bytes"`
	StoredResponsePath string                        `json:"stored_response_path,omitempty" csv:"stored_response_path" mapstructure:"stored_response_path"`
	ScreenshotPath     string                        `json:"screenshot_path,omitempty" csv:"screenshot_path" mapstructure:"screenshot_path"`
	ScreenshotPathRel  string                        `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel" mapstructure:"screenshot_path_rel"`
	KnowledgeBase      map[string]interface{}        `json:"knowledgebase,omitempty" csv:"knowledgebase" mapstructure:"knowledgebase"`
	Resolvers          []string                      `json:"resolvers,omitempty" csv:"resolvers" mapstructure:"resolvers"`
	Fqdns              []string                      `json:"body_fqdn,omitempty" mapstructure:"body_fqdn"`
	Domains            []string                      `json:"body_domains,omitempty" mapstructure:"body_domains"`
	TechnologyDetails  map[string]wappalyzer.AppInfo `json:"-" csv:"-" mapstructure:"-"`
	RequestRaw         []byte                        `json:"-" csv:"-" mapstructure:"-"`
	Response           *httpx.Response               `json:"-" csv:"-" mapstructure:"-"`
	FaviconData        []byte                        `json:"-" csv:"-" mapstructure:"-"`
	Trace              *retryablehttp.TraceInfo      `json:"trace,omitempty" csv:"trace"  mapstructure:"trace"`
	// contains filtered or unexported fields
}

Result of a scan

func (Result) CSVHeader added in v1.1.3 

func (r Result) CSVHeader() string

CSVHeader the CSV headers

func (Result) CSVRow added in v1.1.3 

func (r Result) CSVRow(scanopts *ScanOptions) string

CSVRow the CSV Row

func (Result) JSON 

func (r Result) JSON(scanopts *ScanOptions) string

JSON the result

type ResumeCfg added in v1.1.1 

type ResumeCfg struct {
	ResumeFrom string
	Index      int
	// contains filtered or unexported fields
}

type Runner 

type Runner struct {
	HostErrorsCache gcache.Cache[string, int]
	// contains filtered or unexported fields
}

Runner is a client for running the enumeration process.

func New 

func New(options *Options) (*Runner, error)

New creates a new client for running enumeration process.

func (*Runner) Close 

func (r *Runner) Close()

Close closes the httpx scan instance

func (*Runner) GetScanOpts added in v1.2.4 

func (r *Runner) GetScanOpts() ScanOptions

func (*Runner) HTTPX added in v1.6.5 

func (r *Runner) HTTPX() *httpx.HTTPX

func (*Runner) HandleFaviconHash added in v1.6.5 

func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, currentResp []byte, defaultProbe bool) (string, string, string, []byte, string, error)

func (*Runner) Process added in v1.2.4 

func (r *Runner) Process(t string, wg *syncutil.AdaptiveWaitGroup, protocol string, scanopts *ScanOptions, output chan Result)

func (*Runner) RunEnumeration 

func (r *Runner) RunEnumeration()

RunEnumeration on targets for httpx client

func (*Runner) SaveResumeConfig added in v1.1.1 

func (r *Runner) SaveResumeConfig() error

SaveResumeConfig to file

type ScanOptions added in v1.3.1 

type ScanOptions struct {
	Methods                   []string
	StoreResponseDirectory    string
	RequestURI                string
	RequestBody               string
	VHost                     bool
	OutputTitle               bool
	OutputStatusCode          bool
	OutputLocation            bool
	OutputContentLength       bool
	StoreResponse             bool
	OmitBody                  bool
	OutputServerHeader        bool
	OutputWebSocket           bool
	OutputWithNoColor         bool
	OutputMethod              bool
	ResponseHeadersInStdout   bool
	ResponseInStdout          bool
	Base64ResponseInStdout    bool
	ChainInStdout             bool
	TLSProbe                  bool
	CSPProbe                  bool
	VHostInput                bool
	OutputContentType         bool
	Unsafe                    bool
	Pipeline                  bool
	HTTP2Probe                bool
	OutputIP                  bool
	OutputCName               bool
	OutputCDN                 string
	OutputResponseTime        bool
	PreferHTTPS               bool
	NoFallback                bool
	NoFallbackScheme          bool
	TechDetect                bool
	StoreChain                bool
	StoreVisionReconClusters  bool
	MaxResponseBodySizeToSave int
	MaxResponseBodySizeToRead int
	OutputExtractRegex        string

	ExcludeCDN         bool
	HostMaxErrors      int
	ProbeAllIPS        bool
	Favicon            bool
	LeaveDefaultPorts  bool
	OutputLinesCount   bool
	OutputWordsCount   bool
	Hashes             string
	Screenshot         bool
	UseInstalledChrome bool
	DisableStdin       bool
	NoScreenshotBytes  bool
	NoHeadlessBody     bool
	ScreenshotTimeout  time.Duration
	ScreenshotIdle     time.Duration
	// contains filtered or unexported fields
}

func (*ScanOptions) Clone added in v1.3.1 

func (s *ScanOptions) Clone() *ScanOptions

type Server added in v1.6.4 

type Server struct {
	// contains filtered or unexported fields
}

Server represents the HTTP server that handles the concurrency settings endpoints.

func NewServer added in v1.6.4 

func NewServer(addr string, config *Options) *Server

New creates a new instance of Server.

func (*Server) Start added in v1.6.4 

func (s *Server) Start() error

Start initializes the server and its routes, then starts listening on the specified address.

func (*Server) Stop added in v1.6.8 

func (s *Server) Stop() error

Stop gracefully shuts down the server.

type Trace added in v1.6.9 

type Trace struct {
	GetConn              time.Time `json:"get_conn,omitempty"`
	GotConn              time.Time `json:"got_conn,omitempty"`
	PutIdleConn          time.Time `json:"put_idle_conn,omitempty"`
	GotFirstResponseByte time.Time `json:"got_first_response_byte,omitempty"`
	Got100Continue       time.Time `json:"got_100_continue,omitempty"`
	DNSStart             time.Time `json:"dns_start,omitempty"`
	DNSDone              time.Time `json:"dns_done,omitempty"`
	ConnectStart         time.Time `json:"connect_start,omitempty"`
	ConnectDone          time.Time `json:"connect_done,omitempty"`
	TLSHandshakeStart    time.Time `json:"tls_handshake_start,omitempty"`
	TLSHandshakeDone     time.Time `json:"tls_handshake_done,omitempty"`
	WroteHeaderField     time.Time `json:"wrote_header_field,omitempty"`
	WroteHeaders         time.Time `json:"wrote_headers,omitempty"`
	Wait100Continue      time.Time `json:"wait_100_continue,omitempty"`
	WroteRequest         time.Time `json:"wrote_request,omitempty"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.