extractors

Documentation

Overview

Package extractors implements extractors for http response data retrieval.

Index

• func SupportsMap(extractor *Extractor) bool
• type Extractor
  • func (e *Extractor) CompileExtractors() error
  • func (e *Extractor) ExtractDSL(data map[string]interface{}) map[string]struct{}
  • func (e *Extractor) ExtractHTML(corpus string) map[string]struct{}
  • func (e *Extractor) ExtractJSON(corpus string) map[string]struct{}
  • func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{}
  • func (e *Extractor) ExtractRegex(corpus string) map[string]struct{}
  • func (e *Extractor) ExtractXML(corpus string) map[string]struct{}
  • func (e *Extractor) ExtractXPath(corpus string) map[string]struct{}
  • func (e *Extractor) GetType() ExtractorType
  • func (e *Extractor) SaveToFile(data map[string]struct{})
• type ExtractorType
  • func GetSupportedExtractorTypes() []ExtractorType
  • func (t ExtractorType) String() string
• type ExtractorTypeHolder
  • func (holder ExtractorTypeHolder) JSONSchemaType() *jsonschema.Type
  • func (holder *ExtractorTypeHolder) MarshalJSON() ([]byte, error)
  • func (holder ExtractorTypeHolder) MarshalYAML() (interface{}, error)
  • func (holder *ExtractorTypeHolder) UnmarshalJSON(data []byte) error
  • func (holder *ExtractorTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error

Functions

func SupportsMap

func SupportsMap(extractor *Extractor) bool

SupportsMap determines if the extractor type requires a map

Types

type Extractor

type Extractor struct {
	// description: |
	//   Name of the extractor. Name should be lowercase and must not contain
	//   spaces or underscores (_).
	// examples:
	//   - value: "\"cookie-extractor\""
	Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"title=name of the extractor,description=Name of the extractor"`
	// description: |
	//   Type is the type of the extractor.
	Type ExtractorTypeHolder `json:"type" yaml:"type"`

	// description: |
	//   Regex contains the regular expression patterns to extract from a part.
	//
	//   Go regex engine does not support lookaheads or lookbehinds, so as a result
	//   they are also not supported in nuclei.
	// examples:
	//   - name: Braintree Access Token Regex
	//     value: >
	//       []string{"access_token\\$production\\$[0-9a-z]{16}\\$[0-9a-f]{32}"}
	//   - name: Wordpress Author Extraction regex
	//     value: >
	//       []string{"Author:(?:[A-Za-z0-9 -\\_=\"]+)?<span(?:[A-Za-z0-9 -\\_=\"]+)?>([A-Za-z0-9]+)<\\/span>"}
	Regex []string `` /* 130-byte string literal not displayed */
	// description: |
	//   Group specifies a numbered group to extract from the regex.
	// examples:
	//   - name: Example Regex Group
	//     value: "1"
	RegexGroup int `` /* 132-byte string literal not displayed */

	// description: |
	//   kval contains the key-value pairs present in the HTTP response header.
	//   kval extractor can be used to extract HTTP response header and cookie key-value pairs.
	//   kval extractor inputs are case-insensitive, and does not support dash (-) in input which can replaced with underscores (_)
	// 	 For example, Content-Type should be replaced with content_type
	//
	//   A list of supported parts is available in docs for request types.
	// examples:
	//   - name: Extract Server Header From HTTP Response
	//     value: >
	//       []string{"server"}
	//   - name: Extracting value of PHPSESSID Cookie
	//     value: >
	//       []string{"phpsessid"}
	//   - name: Extracting value of Content-Type Cookie
	//     value: >
	//       []string{"content_type"}
	KVal []string `` /* 146-byte string literal not displayed */

	// description: |
	//   JSON allows using jq-style syntax to extract items from json response
	//
	// examples:
	//   - value: >
	//       []string{".[] | .id"}
	//   - value: >
	//       []string{".batters | .batter | .[] | .id"}
	JSON []string `` /* 161-byte string literal not displayed */
	// description: |
	//   XPath allows using xpath expressions to extract items from html response
	//
	// examples:
	//   - value: >
	//       []string{"/html/body/div/p[2]/a"}
	XPath []string `` /* 188-byte string literal not displayed */
	// description: |
	//   Attribute is an optional attribute to extract from response XPath.
	//
	// examples:
	//   - value: "\"href\""
	Attribute string `` /* 175-byte string literal not displayed */

	// description: |
	//   Extracts using DSL expressions.
	DSL []string `` /* 147-byte string literal not displayed */

	// description: |
	//   Part is the part of the request response to extract data from.
	//
	//   Each protocol exposes a lot of different parts which are well
	//   documented in docs for each request type.
	// examples:
	//   - value: "\"body\""
	//   - value: "\"raw\""
	Part string `` /* 162-byte string literal not displayed */
	// description: |
	//   Internal, when set to true will allow using the value extracted
	//   in the next request for some protocols (like HTTP).
	Internal bool `` /* 231-byte string literal not displayed */

	// description: |
	//   CaseInsensitive enables case-insensitive extractions. Default is false.
	// values:
	//   - false
	//   - true
	CaseInsensitive bool `` /* 156-byte string literal not displayed */
	// description: |
	//  ToFile (to) saves extracted requests to file and if file is present values are appended to file.
	ToFile string `` /* 130-byte string literal not displayed */
	// contains filtered or unexported fields
}

Extractor is used to extract part of response using a regex.

func (*Extractor) CompileExtractors

func (e *Extractor) CompileExtractors() error

CompileExtractors performs the initial setup operation on an extractor

func (*Extractor) ExtractDSL

func (e *Extractor) ExtractDSL(data map[string]interface{}) map[string]struct{}

ExtractDSL execute the expression and returns the results

func (*Extractor) ExtractHTML

func (e *Extractor) ExtractHTML(corpus string) map[string]struct{}

ExtractHTML extracts items from HTML using XPath selectors

func (*Extractor) ExtractJSON

func (e *Extractor) ExtractJSON(corpus string) map[string]struct{}

ExtractJSON extracts text from a corpus using JQ queries and returns it

func (*Extractor) ExtractKval

func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{}

ExtractKval extracts key value pairs from a data map

func (*Extractor) ExtractRegex

func (e *Extractor) ExtractRegex(corpus string) map[string]struct{}

ExtractRegex extracts text from a corpus and returns it

func (*Extractor) ExtractXML

func (e *Extractor) ExtractXML(corpus string) map[string]struct{}

ExtractXML extracts items from XML using XPath selectors

func (*Extractor) ExtractXPath

func (e *Extractor) ExtractXPath(corpus string) map[string]struct{}

ExtractXPath extracts items from text using XPath selectors

func (*Extractor) GetType

func (e *Extractor) GetType() ExtractorType

GetType returns the type of the matcher

func (*Extractor) SaveToFile

func (e *Extractor) SaveToFile(data map[string]struct{})

SaveToFile saves extracted values to file if `to` is present and valid

type ExtractorType

type ExtractorType int

ExtractorType is the type of the extractor specified

const (
	// name:regex
	RegexExtractor ExtractorType = iota + 1
	// name:kval
	KValExtractor
	// name:xpath
	XPathExtractor
	// name:json
	JSONExtractor
	// name:dsl
	DSLExtractor
)

name:ExtractorType

func GetSupportedExtractorTypes

func GetSupportedExtractorTypes() []ExtractorType

GetSupportedExtractorTypes returns list of supported types

func (ExtractorType) String

func (t ExtractorType) String() string

type ExtractorTypeHolder

type ExtractorTypeHolder struct {
	ExtractorType ExtractorType `mapping:"true"`
}

ExtractorTypeHolder is used to hold internal type of the extractor

func (ExtractorTypeHolder) JSONSchemaType

func (holder ExtractorTypeHolder) JSONSchemaType() *jsonschema.Type

func (*ExtractorTypeHolder) MarshalJSON

func (holder *ExtractorTypeHolder) MarshalJSON() ([]byte, error)

func (ExtractorTypeHolder) MarshalYAML

func (holder ExtractorTypeHolder) MarshalYAML() (interface{}, error)

func (*ExtractorTypeHolder) UnmarshalJSON

func (holder *ExtractorTypeHolder) UnmarshalJSON(data []byte) error

func (*ExtractorTypeHolder) UnmarshalYAML

func (holder *ExtractorTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error