webfingerprint

package
v1.3.4-alpha0725 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 24, 2024 License: AGPL-3.0 Imports: 28 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	CRLF = "\r\n"
)

Variables

View Source 
var (
	DefaultWebFingerprintRules = []*WebRule{

		{
			Methods: []*WebMatcherMethods{
				{
					Keywords: []*KeywordMatcher{
						{
							CPE:    CPE{Product: "phpmyadmin", Vendor: "phpmyadmin"},
							Regexp: `<h.>Welcome to.*phpMyAdmin`,
						},
					},
				},
				{
					Keywords: []*KeywordMatcher{
						{
							CPE:    CPE{Product: "phpmyadmin", Vendor: "phpmyadmin"},
							Regexp: `phpmyadmin\.css\.php\?`,
						},
					},
				},
			},
			NextStep: &WebRule{
				Path: "/README",
				Methods: []*WebMatcherMethods{
					{Keywords: []*KeywordMatcher{
						{
							CPE:          CPE{Product: "phpmyadmin", Vendor: "phpmyadmin"},
							Regexp:       `Version (?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)`,
							VersionIndex: 1,
						},
					}},
				},
			},
		},

		{
			Methods: []*WebMatcherMethods{
				{
					MD5s: []*MD5Matcher{
						{
							CPE: CPE{
								Vendor:  "oracle",
								Product: "weblogic_server",
							},
							MD5: "1af585e6c8cc77a6ca1832b608fd20aa"},
					},
				},
			},
			NextStep: &WebRule{
				Path: "/console/login/LoginForm.jsp",
				Methods: []*WebMatcherMethods{{
					Keywords: []*KeywordMatcher{{
						CPE:          CPE{Product: "weblogic", Vendor: "oracle"},
						Regexp:       `WebLogic Server.*: ([0-9\.]+)</p>`,
						VersionIndex: 1,
					}, {
						CPE:          CPE{Product: "weblogic_server", Vendor: "oracle"},
						Regexp:       `WebLogic Server.*: ([0-9\.]+)</p>`,
						VersionIndex: 1,
					}},
				}},
			},
		},

		{
			Methods: []*WebMatcherMethods{
				{
					HTTPHeaders: []*HTTPHeaderMatcher{

						{
							HeaderName: "Server",
							HeaderValue: KeywordMatcher{
								Regexp:       `(?P<product>[a-zA-Z_0-9]+)/(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)`,
								ProductIndex: 1,
								VersionIndex: 2,
							},
						},
						{
							HeaderName: "X-Powered-By",
							HeaderValue: KeywordMatcher{
								Regexp:       `(?P<product>[a-zA-Z_0-9]+)/(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)`,
								ProductIndex: 1,
								VersionIndex: 2,
							},
						},

						{
							HeaderName: "X-Powered-By",
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: "jsp", Vendor: "oracle"},
								Regexp:       `JSP/((\d+\.?)+)`,
								VersionIndex: 1,
							},
						},

						{
							HeaderName: "Server",
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: "iis"},
								Regexp:       `Microsoft-IIS/(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)`,
								VersionIndex: 1,
							},
						},

						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: "glassfish_server"},
								Regexp:       `Oracle GlassFish Server +(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)`,
								VersionIndex: 1,
							},
						},
						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: "glassfish_server"},
								Regexp:       `GlassFish Server Open Source Edition +(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)`,
								VersionIndex: 1,
							},
						},
						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: "glassfish_server"},
								Regexp:       `.*GlassFish Server Open Source Edition +(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)`,
								VersionIndex: 1,
							},
						},

						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: "php"},
								Regexp:       `PHP/(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)`,
								VersionIndex: 1,
							},
						},

						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: "openssl"},
								Regexp:       `OpenSSL/(?P<version>((\d+\.?)+)?([a-zA-Z_0-9]+)?)`,
								VersionIndex: 1,
							},
						},

						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: `mod_fcgid`},
								Regexp:       `mod_fcgid/(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)`,
								VersionIndex: 1,
							},
						},

						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: `mod_ssl`},
								Regexp:       `mod_ssl/(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)`,
								VersionIndex: 1,
							},
						},

						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: "weblogic_server"},
								Regexp:       `WebLogic Server +(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?) (SP\d+)`,
								VersionIndex: 1,
								UpdateIndex:  5,
							},
						},

						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: "nginx"},
								Regexp:       `nginx/(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)`,
								VersionIndex: 1,
							},
						},
						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: "nginx"},
								Regexp:       `Nginx/(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)`,
								VersionIndex: 1,
							},
						},

						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Vendor: "eclipse", Product: "jetty"},
								Regexp:       `Jetty\((?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)\)`,
								VersionIndex: 1,
							},
						},

						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: "medusa"},
								Regexp:       `Medusa/(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)`,
								VersionIndex: 1,
							},
						},

						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: "application_server", Vendor: "oracle"},
								Regexp:       `Oracle-Application-Server-(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)(g|c|i)`,
								VersionIndex: 1,
							},
						},

						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: "http_server", Vendor: "oracle"},
								Regexp:       `Oracle-Application-Server-\d+g/(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?) Oracle-HTTP-Server`,
								VersionIndex: 1,
							},
						},

						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: "web_cache", Vendor: "oracle"},
								Regexp:       `Oracle-Web-Cache-\d+g/(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)`,
								VersionIndex: 1,
							},
						},
						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: "web_cache", Vendor: "oracle"},
								Regexp:       `OracleAS-Web-Cache-\d+g/(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)`,
								VersionIndex: 1,
							},
						},

						{
							HeaderValue: KeywordMatcher{
								CPE:          CPE{Product: "http_server", Vendor: "apache"},
								Regexp:       `Apache/(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)`,
								VersionIndex: 1,
							},
						},
					},
					Keywords: []*KeywordMatcher{

						{
							Regexp:       `<title>Apache Tomcat/(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)</title>`,
							CPE:          CPE{Vendor: "apache", Product: "tomcat"},
							VersionIndex: 1,
						},

						{
							CPE:          CPE{Product: "supervisord"},
							Regexp:       `<a href="http://supervisord.org">Supervisor</a> <span>(?P<version>((\d+\.?)+)?([a-zA-Z_0-9-]+)?)</span>`,
							VersionIndex: 1,
						},

						{
							CPE:          CPE{Product: "wordpress", Vendor: "wordpress"},
							Regexp:       `< *meta[^>]*name *= *['"]generator['"][^>]*content *= *['"]WordPress ?([\d.]+)?`,
							VersionIndex: 1,
						},

						{
							CPE:          CPE{Product: "jquery", Vendor: "jquery"},
							Regexp:       `< *script[^>]*src *= *['"][^'"]*jquery-([0-9\.]+)(\.min)?\.js`,
							VersionIndex: 1,
						},
						{
							CPE:          CPE{Product: "jquery", Vendor: "jquery"},
							Regexp:       `< *script[^>]*src *= *['"][^'"]*jquery(\.min)?\.js\?ver=([0-9\.]+)`,
							VersionIndex: 2,
						},
						{
							CPE:          CPE{Product: "jquery", Vendor: "jquery"},
							Regexp:       `< *script[^>]*src *= *['"][^'"]*jquery\/([0-9\.]+)/jquery(\.min)?\.js`,
							VersionIndex: 1,
						},
					},
				},
			},
		},
	}
)

Functions

func CalcAbsolutelyURLWithFragment 

func CalcAbsolutelyURLWithFragment(origin *url.URL, u string) string

func CalcAbsolutelyURLWithoutFragment 

func CalcAbsolutelyURLWithoutFragment(origin *url.URL, u string) string

func DomainToURLFilter 

func DomainToURLFilter(domain string) (*regexp.Regexp, error)

func GenerateRulesFromWappalyzer 

func GenerateRulesFromWappalyzer()

func HttpGet 

func HttpGet(url string) ([]byte, error)

func HttpGetWithRetry 

func HttpGetWithRetry(retry int, url string) ([]byte, error)

func HttpInsecureGet 

func HttpInsecureGet(url string) (*http.Response, error)

func MapQueryToString 

func MapQueryToString(values map[string][]string) string

map[string][]string to query

func MockRandomWebFingerPrints added in v1.2.8 

func MockRandomWebFingerPrints() ([]string, string, int)

func MockWebFingerPrintByName added in v1.2.8 

func MockWebFingerPrintByName(name string) (string, int)

Types

type CPE 

type CPE struct {
	Part     string `yaml:"part,omitempty" json:"part"`
	Vendor   string `yaml:"vendor,omitempty" json:"vendor"`
	Product  string `yaml:"product,omitempty" json:"product"`
	Version  string `yaml:"version,omitempty" json:"version"`
	Update   string `yaml:"update,omitempty" json:"update"`
	Edition  string `yaml:"edition,omitempty" json:"edition"`
	Language string `yaml:"language,omitempty" json:"language"`
}

//////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////CPE MODEL/////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////

func ParseToCPE 

func ParseToCPE(cpe string) (*CPE, error)

func (*CPE) LikeSearchString 

func (c *CPE) LikeSearchString() string

func (*CPE) String 

func (c *CPE) String() string

type CPEAnalyzer 

type CPEAnalyzer struct {
	WebsiteURLs []string
	// contains filtered or unexported fields
}

func NewCPEAnalyzer 

func NewCPEAnalyzer(urls ...string) *CPEAnalyzer

func (*CPEAnalyzer) AvailableCPE 

func (c *CPEAnalyzer) AvailableCPE() []*CPE

func (*CPEAnalyzer) Feed 

func (c *CPEAnalyzer) Feed(url string, cpes ...*CPE)

func (*CPEAnalyzer) GetCPEsByProduct 

func (c *CPEAnalyzer) GetCPEsByProduct(product string) []*CPE

func (*CPEAnalyzer) GetVersionByProduct 

func (c *CPEAnalyzer) GetVersionByProduct(product string) (string, error)

func (*CPEAnalyzer) IsProductExisted 

func (c *CPEAnalyzer) IsProductExisted(product string) bool

func (*CPEAnalyzer) QueryOrigins 

func (c *CPEAnalyzer) QueryOrigins(cpes ...*CPE) map[*CPE]string

type Config 

type Config struct {
	// 使用哪些规则来进行 Web 指纹探测？
	Rules []*WebRule

	// 主动模式
	ActiveMode bool

	// 强制所有规则进行匹配
	ForceAllRuleMatching bool

	// 在需要主动发送 Probe 的规则探测设置中
	//    我们需要为每一个 Probe 设置 TimeoutSeconds
	ProbeTimeout time.Duration

	// 指纹大小默认 20480
	FingerprintDataSize int

	// Proxies
	Proxies []string

	// runtime Id
	RuntimeId string
}

func NewWebFingerprintConfig 

func NewWebFingerprintConfig(options ...ConfigOption) *Config

type ConfigOption 

type ConfigOption func(config *Config)

func WithActiveMode 

func WithActiveMode(b bool) ConfigOption

func WithForceAllRuleMatching 

func WithForceAllRuleMatching(b bool) ConfigOption

func WithProbeTimeout 

func WithProbeTimeout(timeout time.Duration) ConfigOption

func WithRuntimeId added in v1.3.3 

func WithRuntimeId(id string) ConfigOption

func WithWebFingerprintDataSize added in v1.2.8 

func WithWebFingerprintDataSize(size int) ConfigOption

func WithWebFingerprintRules 

func WithWebFingerprintRules(rules []*WebRule) ConfigOption

func WithWebProxy 

func WithWebProxy(proxy ...string) ConfigOption

type HTTPHeaderMatcher 

type HTTPHeaderMatcher struct {
	HeaderName  string         `yaml:"key"`
	HeaderValue KeywordMatcher `yaml:"value"`
}

//////////////////////////////////////////////////////////////////////////////////// ////////////////////////////HTTPHeader Matcher Model//////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////

func (*HTTPHeaderMatcher) Match 

func (h *HTTPHeaderMatcher) Match(name string, value string) (*CPE, error)

func (*HTTPHeaderMatcher) String 

func (h *HTTPHeaderMatcher) String() string

type HTTPResponseInfo 

type HTTPResponseInfo struct {
	Header            *http.Header
	URL               *url.URL
	Status            string
	Body              []byte
	RequestRaw        []byte
	ResponseHeaderRaw []byte
	ResponseRaw       []byte

	StatusCode int
	IsHttps    bool
	// contains filtered or unexported fields
}

func ExtractHTTPResponseInfoFromHTTPResponse 

func ExtractHTTPResponseInfoFromHTTPResponse(res *http.Response) *HTTPResponseInfo

func ExtractHTTPResponseInfoFromHTTPResponseWithBodySize 

func ExtractHTTPResponseInfoFromHTTPResponseWithBodySize(res *http.Response, size int) *HTTPResponseInfo

func (*HTTPResponseInfo) Bytes 

func (h *HTTPResponseInfo) Bytes() []byte

func (*HTTPResponseInfo) ResponseHeaderBytes 

func (h *HTTPResponseInfo) ResponseHeaderBytes() []byte

type Info 

type Info struct {
	Cats    []int
	Icon    string
	Website string

	Headers map[string]string
	Html    interface{}
	Scripts interface{}
	Meta    interface{}
	Cookies map[string]string
	Url     string
	Js      map[string]string
	Implies []string
}

type KeywordMatcher 

type KeywordMatcher struct {
	CPE `yaml:"cpe,inline,omitempty"`

	Regexp        string `yaml:"regexp,omitempty"`
	VendorIndex   int    `yaml:"vendor_index,omitempty"`
	ProductIndex  int    `yaml:"product_index,omitempty"`
	VersionIndex  int    `yaml:"version_index,omitempty"`
	UpdateIndex   int    `yaml:"update_index,omitempty"`
	EditionIndex  int    `yaml:"edition_index,omitempty"`
	LanguageIndex int    `yaml:"language_index,omitempty"`
	// contains filtered or unexported fields
}

//////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////Keyword Matcher Model//////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////

func ProcessMetaValue added in v1.2.8 

func ProcessMetaValue(v interface{}, key, name string) *KeywordMatcher

ProcessMetaValue processes and returns a keyword matcher

func (*KeywordMatcher) Match 

func (k *KeywordMatcher) Match(raw string) (*CPE, error)

type MD5Matcher 

type MD5Matcher struct {
	CPE `yaml:"cpe,inline,omitempty"`

	MD5 string `yaml:"md5"`
	// contains filtered or unexported fields
}

//////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////MD5 Matcher Model//////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////

func (*MD5Matcher) Match 

func (h *MD5Matcher) Match(raw []byte) (*CPE, error)

type Matcher 

type Matcher struct {
	// contains filtered or unexported fields
}

func NewWebFingerprintMatcher 

func NewWebFingerprintMatcher(rules []*WebRule, active bool, allRules bool) (*Matcher, error)

func NewWebFingerprintMatcherWithConfig 

func NewWebFingerprintMatcherWithConfig(config *Config) *Matcher

func (*Matcher) Match 

func (f *Matcher) Match(rsp *HTTPResponseInfo) ([]*CPE, error)

func (*Matcher) MatchWithOptions 

func (f *Matcher) MatchWithOptions(rsp *HTTPResponseInfo, options ...ConfigOption) ([]*CPE, error)

func (*Matcher) MatchWithRules 

func (f *Matcher) MatchWithRules(rsp *HTTPResponseInfo, rules []*WebRule) ([]*CPE, error)

type ResponseInfo 

type ResponseInfo struct {
	http.Response
	Body []byte
}

type ServiceRule 

type ServiceRule struct {
	Probe    string            `yaml:"probe,omitempty"`
	Proto    TransportProto    `yaml:"proto,omitempty"`
	Keywords []*KeywordMatcher `yaml:"keywords,omitempty"`
}

type TransportProto 

type TransportProto string

//////////////////////////////////////////////////////////////////////////////////// ////////////////////////////Service Rule Model ///////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////// 

var (
	TCP TransportProto = "tcp"
	UDP TransportProto = "udp"
)

type WappalyzerBase 

type WappalyzerBase struct {
	Apps map[string]Info
}

type WebMatcherMethods 

type WebMatcherMethods struct {
	Condition string `yaml:"condition,omitempty"`

	Keywords    []*KeywordMatcher    `yaml:"keywords,omitempty"`
	HTTPHeaders []*HTTPHeaderMatcher `yaml:"headers,omitempty"`
	MD5s        []*MD5Matcher        `yaml:"md5s,omitempty"`
}

//////////////////////////////////////////////////////////////////////////////////// /////////////////////////WebMatcherMethods Model /////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////

type WebRule 

type WebRule struct {
	Path     string `yaml:"path,omitempty"`
	Methods  []*WebMatcherMethods
	NextStep *WebRule `yaml:"next,omitempty"`
}

//////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////Web Rule///////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////

func FileOrDirToWebRules 

func FileOrDirToWebRules(dir string) []*WebRule

func GetYamlWebFingerprintRules 

func GetYamlWebFingerprintRules(yamlFilePath string) ([]*WebRule, error)

func LoadDefaultDataSource 

func LoadDefaultDataSource() ([]*WebRule, error)

func ParseWebFingerprintRules 

func ParseWebFingerprintRules(raw []byte) ([]*WebRule, error)

func (*WebRule) IsActiveToProbePath 

func (w *WebRule) IsActiveToProbePath() bool

type WriteCounter 

type WriteCounter struct {
	Total    uint64
	FileSize uint64
}

WriteCounter counts the number of bytes written to it. It implements to the io.Writer interface and we can pass this into io.TeeReader() which will report progress on each write cycle.

func (WriteCounter) PrintProgress 

func (wc WriteCounter) PrintProgress()

func (*WriteCounter) Write 

func (wc *WriteCounter) Write(p []byte) (int, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.