dsl

package module
v0.0.25 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 2, 2023 License: MIT Imports: 50 Imported by: 39

README

DSL

DSL ease the creation of expressions by providing a set of built-in helper functions.

What is DSL?

DSL is a library that allows to create and evaluate expressions. You can define expressions to compare, filter, or transform data, and then evaluate them against a set of data.

Example
package main

import (
	"fmt"
	"github.com/Knetic/govaluate"
	"github.com/projectdiscovery/dsl"
)

func main() {
	// Define the data to evaluate
	data := map[string]interface{}{
		"username": "johndoe",
		"email":    "johndoe@example.com",
		"password": "12345",
		"ip":       "127.0.0.1",
		"url":      "https://www.example.com",
		"date":     "2022-05-01",
	}

	// Define the expressions to evaluate
	expressions := map[string]string{
		"username_and_email_matcher": "contains(username, 'john') && contains(email, 'example.com')",
		"password_criteria":          "contains_any(password, '0123456789') && regex_any(password, '[A-Z]')",
		"sha256_username_matcher":    "sha256(username) == 'a0d95c8b32fa9b05a7d790a08e221c384b317ca05f66a7b84978d22c9838bb2a'",
		"ip_format_matcher":        "ip_format(ip, '1') == '127.0.0.1'",
		"url_valid_matcher":       "startswith(url, 'http') && contains(url, '://') && contains(url, '.') && !contains_any(url, ':@')",
	}

	for matcherName, expression := range expressions {
		compiledExpression, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.DefaultHelperFunctions)
		if err != nil {
			fmt.Printf("Failed to compile expresion: %v\n", expression)
		}

		result, err := compiledExpression.Evaluate(data)
		if err != nil {
			fmt.Printf("Failed to evaluate expresion: %v\n", expression)
		}

		if result == true {
			fmt.Printf("[%v] matches data\n", matcherName)
		} else {
			fmt.Printf("[%v] not matches data\n", matcherName)
		}
	}
}
Default Helper functions list
Helper function Description Example Output
aes_gcm(key, plaintext interface{}) []byte AES GCM encrypts a string with key {{hex_encode(aes_gcm("AES256Key-32Characters1234567890", "exampleplaintext"))}} ec183a153b8e8ae7925beed74728534b57a60920c0b009eaa7608a34e06325804c096d7eebccddea3e5ed6c4
base64(src interface{}) string Base64 encodes a string base64("Hello") SGVsbG8=
base64_decode(src interface{}) []byte Base64 decodes a string base64_decode("SGVsbG8=") Hello
base64_py(src interface{}) string Encodes string to base64 like python (with new lines) base64_py("Hello") SGVsbG8=\n
bin_to_dec(binaryNumber number | string) float64 Transforms the input binary number into a decimal format bin_to_dec("0b1010")
bin_to_dec(1010)		 10
compare_versions(versionToCheck string, constraints ...string) bool Compares the first version argument with the provided constraints compare_versions('v1.0.0', '>v0.0.1', '<v1.0.1') true
concat(arguments ...interface{}) string Concatenates the given number of arguments to form a string concat("Hello", 123, "world) Hello123world
contains(input, substring interface{}) bool Verifies if a string contains a substring contains("Hello", "lo") true
contains_all(input interface{}, substrings ...string) bool Verifies if any input contains all of the substrings contains("Hello everyone", "lo", "every") true
contains_any(input interface{}, substrings ...string) bool Verifies if an input contains any of substrings contains("Hello everyone", "abc", "llo") true
date_time(dateTimeFormat string, optionalUnixTime interface{}) string Returns the formatted date time using simplified or go style layout for the current or the given unix time date_time("%Y-%M-%D %H:%m")
date_time("%Y-%M-%D %H:%m", 1654870680)
date_time("2006-01-02 15:04", unix_time())		 2022-06-10 14:18
dec_to_hex(number number | string) string Transforms the input number into hexadecimal format dec_to_hex(7001)" 1b59
deflate(input string) string Compresses the input using DEFLATE deflate("Hello") "\xf2\x48\xcd\xc9\xc9\x07\x04\x00\x00\xff\xff"
ends_with(str string, suffix ...string) bool Checks if the string ends with any of the provided substrings ends_with("Hello", "lo") true
generate_java_gadget(gadget, cmd, encoding interface{}) string Generates a Java Deserialization Gadget generate_java_gadget("dns", "{{interactsh-url}}", "base64") rO0ABXNyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABc3IADGphdmEubmV0LlVSTJYlNzYa/ORyAwAHSQAIaGFzaENvZGVJAARwb3J0TAAJYXV0aG9yaXR5dAASTGphdmEvbGFuZy9TdHJpbmc7TAAEZmlsZXEAfgADTAAEaG9zdHEAfgADTAAIcHJvdG9jb2xxAH4AA0wAA3JlZnEAfgADeHD//////////3QAAHQAAHEAfgAFdAAFcHh0ACpjYWhnMmZiaW41NjRvMGJ0MHRzMDhycDdlZXBwYjkxNDUub2FzdC5mdW54
generate_jwt(json, algorithm, signature, unixMaxAge) []byte Generates a JSON Web Token (JWT) using the claims provided in a JSON string, the signature, and the specified algorithm generate_jwt("{\"name\":\"John Doe\",\"foo\":\"bar\"}", "HS256", "hello-world") eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJmb28iOiJiYXIiLCJuYW1lIjoiSm9obiBEb2UifQ.EsrL8lIcYJR_Ns-JuhF3VCllCP7xwbpMCCfHin_WT6U
gzip(input string) string Compresses the input using GZip base64(gzip("Hello")) +H4sIAAAAAAAA//JIzcnJBwQAAP//gonR9wUAAAA=
gzip_decode(input string) string Decompresses the input using GZip gzip_decode(hex_decode("1f8b08000000000000fff248cdc9c907040000ffff8289d1f705000000")) Hello
hex_decode(input interface{}) []byte Hex decodes the given input hex_decode("6161") aa
hex_encode(input interface{}) string Hex encodes the given input hex_encode("aa") 6161
hex_to_dec(hexNumber number | string) float64 Transforms the input hexadecimal number into decimal format hex_to_dec("ff")
hex_to_dec("0xff")		 255
hmac(algorithm, data, secret) string hmac function that accepts a hashing function type with data and secret hmac("sha1", "test", "scrt") 8856b111056d946d5c6c92a21b43c233596623c6
html_escape(input interface{}) string HTML escapes the given input html_escape("<body>test</body>") &lt;body&gt;test&lt;/body&gt;
html_unescape(input interface{}) string HTML un-escapes the given input html_unescape("&lt;body&gt;test&lt;/body&gt;") <body>test</body>
index(slice, index) interface{} Select item at index from slice or string (zero based) index("test",0) t
inflate(input string) string Decompresses the input using DEFLATE inflate(hex_decode("f348cdc9c90700")) Hello
join(separator string, elements ...interface{}) string Joins the given elements using the specified separator join("_", 123, "hello", "world") 123_hello_world
jarm(hostport string) string Calculate the jarm hash for the host:port combination jarm("127.0.0.1:443") 29d29d00029d29d00041d41d000000ad9bf51cc3f5a1e29eecb81d0c7b06eb
json_minify(json) string Minifies a JSON string by removing unnecessary whitespace json_minify("{ \"name\": \"John Doe\", \"foo\": \"bar\" }") {"foo":"bar","name":"John Doe"}
json_prettify(json) string Prettifies a JSON string by adding indentation json_prettify("{\"foo\":\"bar\",\"name\":\"John Doe\"}") {\n \"foo\": \"bar\",\n \"name\": \"John Doe\"\n}
len(arg interface{}) int Returns the length of the input len("Hello") 5
line_ends_with(str string, suffix ...string) bool Checks if any line of the string ends with any of the provided substrings line_ends_with("Hello\nHi", "lo") true
line_starts_with(str string, prefix ...string) bool Checks if any line of the string starts with any of the provided substrings line_starts_with("Hi\nHello", "He") true
llm_prompt(str string) string Query OpenAI LLM (default GPT 3.5) with the provided text prompt and result the result as string (requires api token as environment variable OPENAI_API_KEY) llm_prompt("produce a generic json") {'a':'b'}
md5(input interface{}) string Calculates the MD5 (Message Digest) hash of the input md5("Hello") 8b1a9953c4611296a827abf8c47804d7
mmh3(input interface{}) string Calculates the MMH3 (MurmurHash3) hash of an input mmh3("Hello") 316307400
oct_to_dec(octalNumber number | string) float64 Transforms the input octal number into a decimal format oct_to_dec("0o1234567")
oct_to_dec(1234567)		 342391
print_debug(args ...interface{}) Prints the value of a given input or expression. Used for debugging. print_debug(1+2, "Hello") 3 Hello
rand_base(length uint, optionalCharSet string) string Generates a random sequence of given length string from an optional charset (defaults to letters and numbers) rand_base(5, "abc") caccb
rand_char(optionalCharSet string) string Generates a random character from an optional character set (defaults to letters and numbers) rand_char("abc") a
rand_int(optionalMin, optionalMax uint) int Generates a random integer between the given optional limits (defaults to 0 - MaxInt32) rand_int(1, 10) 6
rand_ip(cidr ...string) string Generates a random IP address rand_ip("192.168.0.0/24") 192.168.0.171
rand_text_alpha(length uint, optionalBadChars string) string Generates a random string of letters, of given length, excluding the optional cutset characters rand_text_alpha(10, "abc") WKozhjJWlJ
rand_text_alphanumeric(length uint, optionalBadChars string) string Generates a random alphanumeric string, of given length without the optional cutset characters rand_text_alphanumeric(10, "ab12") NthI0IiY8r
rand_text_numeric(length uint, optionalBadNumbers string) string Generates a random numeric string of given length without the optional set of undesired numbers rand_text_numeric(10, 123) 0654087985
regex(pattern, input string) bool Tests the given regular expression against the input string regex("H([a-z]+)o", "Hello") true
remove_bad_chars(input, cutset interface{}) string Removes the desired characters from the input remove_bad_chars("abcd", "bc") ad
repeat(str string, count uint) string Repeats the input string the given amount of times repeat("../", 5) ../../../../../
replace(str, old, new string) string Replaces a given substring in the given input replace("Hello", "He", "Ha") Hallo
replace_regex(source, regex, replacement string) string Replaces substrings matching the given regular expression in the input replace_regex("He123llo", "(\\d+)", "") Hello
reverse(input string) string Reverses the given input reverse("abc") cba
sha1(input interface{}) string Calculates the SHA1 (Secure Hash 1) hash of the input sha1("Hello") f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0
sha256(input interface{}) string Calculates the SHA256 (Secure Hash 256) hash of the input sha256("Hello") 185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969
starts_with(str string, prefix ...string) bool Checks if the string starts with any of the provided substrings starts_with("Hello", "He") true
to_lower(input string) string Transforms the input into lowercase characters to_lower("HELLO") hello
to_unix_time(input string, layout string) int Parses a string date time using default or user given layouts, then returns its Unix timestamp to_unix_time("2022-01-13T16:30:10+00:00")
to_unix_time("2022-01-13 16:30:10")
to_unix_time("13-01-2022 16:30:10". "02-01-2006 15:04:05")		 1642091410
to_upper(input string) string Transforms the input into uppercase characters to_upper("hello") HELLO
trim(input, cutset string) string Returns a slice of the input with all leading and trailing Unicode code points contained in cutset removed trim("aaaHelloddd", "ad") Hello
trim_left(input, cutset string) string Returns a slice of the input with all leading Unicode code points contained in cutset removed trim_left("aaaHelloddd", "ad") Helloddd
trim_prefix(input, prefix string) string Returns the input without the provided leading prefix string trim_prefix("aaHelloaa", "aa") Helloaa
trim_right(input, cutset string) string Returns a string, with all trailing Unicode code points contained in cutset removed trim_right("aaaHelloddd", "ad") aaaHello
trim_space(input string) string Returns a string, with all leading and trailing white space removed, as defined by Unicode trim_space(" Hello ") "Hello"
trim_suffix(input, suffix string) string Returns input without the provided trailing suffix string trim_suffix("aaHelloaa", "aa") aaHello
unix_time(optionalSeconds uint) float64 Returns the current Unix time (number of seconds elapsed since January 1, 1970 UTC) with the added optional seconds unix_time(10) 1639568278
unpack(format string, sequence string/bytes) {}interface Returns the result of python binary unpack for the first operand in the sequence (endianess+operand) unpack('>I', '\xac\xd7\t\xd0') -272646673
url_decode(input string) string URL decodes the input string url_decode("https:%2F%2Fprojectdiscovery.io%3Ftest=1") https://projectdiscovery.io?test=1
url_encode(input string) string URL encodes the input string url_encode("https://projectdiscovery.io/test?a=1") https%3A%2F%2Fprojectdiscovery.io%2Ftest%3Fa%3D1
wait_for(seconds uint) Pauses the execution for the given amount of seconds wait_for(10) true
xor(sequences ...strings/bytes) Perform xor on sequences of same length xor('abc','def') 50705 in hex
zlib(input string) string Compresses the input using Zlib base64(zlib("Hello")) eJzySM3JyQcEAAD//wWMAfU=
zlib_decode(input string) string Decompresses the input using Zlib zlib_decode(hex_decode("789cf248cdc9c907040000ffff058c01f5")) Hello
padding(input string, padding string,length int) string Adds padding char to input string until it reaches desired length padding("A","b",3) Abb

Supported encodings:

  • base64 (default)
  • gzip-base64
  • gzip
  • hex
  • raw

Deserialization helper function format:

{{generate_java_gadget(payload, cmd, encoding}}

Deserialization helper function example:

{{generate_java_gadget("commons-collections3.1", "wget http://{{interactsh-url}}", "base64")}}

Binary pack/unpack

The gostruct package provides functionality for converting Go values represented as byte slices. It uses format strings to describe the layout of the Go structs. The format characters include boolean, big endian, little endian, int, float32, float64, and string types with different packed sizes. Currently, only binary unpack of the first operand after endianess has been supported.

JSON helper functions

Nuclei allows manipulate JSON strings in different ways, here is a list of its functions:

  • generate_jwt, to generates a JSON Web Token (JWT) using the claims provided in a JSON string, the signature, and the specified algorithm.
  • json_minify, to minifies a JSON string by removing unnecessary whitespace.
  • json_prettify, to prettifies a JSON string by adding indentation.

Examples

generate_jwt

To generate a JSON Web Token (JWT), you have to supply the JSON that you want to sign, at least.

Here is a list of supported algorithms for generating JWTs with generate_jwt function (case-insensitive):

  • HS256
  • HS384
  • HS512
  • RS256
  • RS384
  • RS512
  • PS256
  • PS384
  • PS512
  • ES256
  • ES384
  • ES512
  • EdDSA
  • NONE

Empty string ("") also means NONE.

Format:

{{generate_jwt(json, algorithm, signature, maxAgeUnix)}}

Arguments other than json are optional.

Example:

variables:
  json: | # required
    {
      "foo": "bar",
      "name": "John Doe"
    }
  alg: "HS256" # optional
  sig: "this_is_secret" # optional
  age: '{{to_unix_time("2032-12-30T16:30:10+00:00")}}' # optional
  jwt: '{{generate_jwt(json, "{{alg}}", "{{sig}}", "{{age}}")}}'

The maxAgeUnix argument is to set the expiration "exp" JWT standard claim, as well as the "iat" claim when you call the function.

json_minify

Format:

{{json_minify(json)}}

Example:

variables:
  json: |
    {
      "foo": "bar",
      "name": "John Doe"
    }
  minify: '{{json_minify(json}}'

minify variable output:

{"foo":"bar","name":"John Doe"}

json_prettify

Format:

{{json_prettify(json)}}

Example:

variables:
  json: '{"foo":"bar","name":"John Doe"}'
  pretty: '{{json_prettify(json}}'

pretty variable output:

{
  "foo": "bar",
  "name": "John Doe"
}
Caching layer

DSL helpers that given the same input have deterministic output and are I/O intensive, should set the Cacheable property to true. This avoids to invoke the function multiple times and return always the same result.

Thanks

Knetic/govaluate - For providing a powerful expression evaluation package.

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	// FunctionNames is a list of function names for expression evaluation usages
	FunctionNames []string

	// DefaultHelperFunctions is a pre-compiled list of govaluate DSL functions
	DefaultHelperFunctions map[string]govaluate.ExpressionFunction

	// ErrParsingArg is error when parsing value of argument
	// Use With Caution: Nuclei ignores this error in extractors(ref: https://github.com/projectdiscovery/nuclei/issues/3950)
	ErrParsingArg = errors.New("error parsing argument value")

	DefaultCacheSize = 250
)
View Source 
var ErrInvalidDslFunction = errors.New("invalid DSL function signature")
View Source 
var PrintDebugCallback func(args ...interface{}) error
View Source 
var (
	RegexStore = &mapsutil.SyncLockMap[string, *regexp.Regexp]{Map: make(mapsutil.Map[string, *regexp.Regexp])}
)

Functions

func AddFunction added in v0.0.6 

func AddFunction(function dslFunction) error

func AddMultiSignatureHelperFunction added in v0.0.4 

func AddMultiSignatureHelperFunction(key string, signatureparts []string, cacheable bool, value func(args ...interface{}) (interface{}, error)) error

AddMultiSignatureHelperFunction allows creation of additional helper functions to be supported with templates Deprecated: Use AddFunction(NewWithMultipleSignatures(...)) - kept for backward compatibility

func EvalExpr 

func EvalExpr(expr string, vars map[string]interface{}) (interface{}, error)

func GetFunctionNames added in v0.0.4 

func GetFunctionNames(heperFunctions map[string]govaluate.ExpressionFunction) []string

func GetPrintableDslFunctionSignatures 

func GetPrintableDslFunctionSignatures(noColor bool) string

func GetPublicIP added in v0.0.11 

func GetPublicIP() string

GetPublicIp of the host

func HelperFunctions added in v0.0.4 

func HelperFunctions() map[string]govaluate.ExpressionFunction

HelperFunctions returns the dsl helper functions

func MustAddFunction added in v0.0.6 

func MustAddFunction(function dslFunction)

func NewWithMultipleSignatures added in v0.0.6 

func NewWithMultipleSignatures(name string, signatures []string, cacheable bool, expr govaluate.ExpressionFunction) dslFunction

func NewWithPositionalArgs added in v0.0.6 

func NewWithPositionalArgs(name string, numberOfArgs int, cacheable bool, expr govaluate.ExpressionFunction) dslFunction

func NewWithSingleSignature added in v0.0.6 

func NewWithSingleSignature(name, signature string, cacheable bool, logic govaluate.ExpressionFunction) dslFunction

func RandSeq 

func RandSeq(base string, n int) string

func Regex 

func Regex(regxp string) (*regexp.Regexp, error)

func TrimAll 

func TrimAll(s, cutset string) string

Types

type Engine 

type Engine struct {
	HelperFunctions map[string]govaluate.ExpressionFunction
	ExpressionStore map[string]*govaluate.EvaluableExpression
	// contains filtered or unexported fields
}

func NewEngine 

func NewEngine() (*Engine, error)

func (*Engine) EvalExpr 

func (e *Engine) EvalExpr(expr string, vars map[string]interface{}) (interface{}, error)

func (*Engine) EvalExprFromCache 

func (e *Engine) EvalExprFromCache(expr string, vars map[string]interface{}) (interface{}, error)

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.