sources

Documentation

Index

• Variables
• func DefaultQuery(query string, engine string) string
• func GetHostname(u string) (string, error)
• func GetProxyFunc(proxy, auth string) (func(*http.Request) (*url.URL, error), error)
• func MatchSubdomains(domain string, html string, fuzzy bool) []string
• func NewHTTPRequest(method, url string, body io.Reader) (*retryablehttp.Request, error)
• func ParseProxyAuth(auth string) (string, string, bool)
• func ReadBody(resp *http.Response) (*bytes.Buffer, error)
• type Agent
• type Keys
  • func (keys Keys) Empty() bool
• type Provider
  • func NewProvider(location string) *Provider
  • func (provider *Provider) GetKeys() Keys
  • func (provider *Provider) HasKeys() bool
  • func (provider *Provider) LoadProviderConfig(location string) error
  • func (provider *Provider) LoadProviderKeysFromEnv()
• type Query
• type Result
  • func (result *Result) HostPort() string
  • func (result *Result) IpPort() string
  • func (result *Result) JSON() string
  • func (result *Result) RawData() string
• type Session
  • func NewSession(keys *Keys, retryMax, timeout, rateLimit int, engines []string, ...) (*Session, error)
  • func (s *Session) Do(request *retryablehttp.Request, source string) (*http.Response, error)

Variables

var (
	// UncoverConfigDir Todo: replace from utils with ConfigDirOrDefault
	UncoverConfigDir = filepath.Join(folderutil.HomeDirOrDefault("."), ".config/uncover")
	// DefaultProviderConfigLocation where keys and config of providers is stored
	DefaultProviderConfigLocation = filepath.Join(UncoverConfigDir, "provider-config.yaml")
)

var DefaultRateLimits = map[string]*ratelimit.Options{
	"shodan":             {Key: "shodan", MaxCount: 1, Duration: time.Second},
	"shodan-idb":         {Key: "shodan-idb", MaxCount: 1, Duration: time.Second},
	"fofa":               {Key: "fofa", MaxCount: 1, Duration: time.Second},
	"censys":             {Key: "censys", MaxCount: 1, Duration: 3 * time.Second},
	"quake":              {Key: "quake", MaxCount: 1, Duration: time.Second},
	"hunter":             {Key: "hunter", MaxCount: 15, Duration: time.Second},
	"zoomeye":            {Key: "zoomeye", MaxCount: 1, Duration: time.Second},
	"netlas":             {Key: "netlas", MaxCount: 1, Duration: time.Second},
	"criminalip":         {Key: "criminalip", MaxCount: 1, Duration: time.Second},
	"publicwww":          {Key: "publicwww", MaxCount: 1, Duration: time.Minute},
	"hunterhow":          {Key: "hunterhow", MaxCount: 1, Duration: 3 * time.Second},
	"binary":             {Key: "binary", MaxCount: 1, Duration: time.Second},
	"fullhunt":           {Key: "fullhunt", MaxCount: 1, Duration: time.Second},
	"fofa-spider":        {Key: "fofa-spider", MaxCount: 5, Duration: time.Second},
	"bing-spider":        {Key: "bing-spider", MaxCount: 1, Duration: time.Second},
	"google-spider":      {Key: "google-spider", MaxCount: 1, Duration: time.Second},
	"chinaz-spider":      {Key: "chinaz-spider", MaxCount: 1, Duration: time.Second},
	"ip138-spider":       {Key: "ip138-spider", MaxCount: 1, Duration: time.Second},
	"qianxun-spider":     {Key: "qianxun-spider", MaxCount: 1, Duration: time.Second},
	"anubis-spider":      {Key: "anubis-spider", MaxCount: 1, Duration: time.Second},
	"baidu-spider":       {Key: "baidu-spider", MaxCount: 5, Duration: time.Second},
	"sitedossier-spider": {Key: "sitedossier-spider", MaxCount: 2, Duration: time.Second},
	"yahoo-spider":       {Key: "yahoo-spider", MaxCount: 3, Duration: time.Second},
}

DefaultRateLimits of all/most of uncover are hardcoded by default to improve performance engine is not present in default ratelimits then user given ratelimit from cli options is used

Functions

func DefaultQuery

func DefaultQuery(query string, engine string) string

func GetHostname

func GetHostname(u string) (string, error)

func GetProxyFunc

func GetProxyFunc(proxy, auth string) (func(*http.Request) (*url.URL, error), error)

GetProxyFunc 辅助函数：获取代理设置函数

func MatchSubdomains

func MatchSubdomains(domain string, html string, fuzzy bool) []string

func NewHTTPRequest

func NewHTTPRequest(method, url string, body io.Reader) (*retryablehttp.Request, error)

func ParseProxyAuth

func ParseProxyAuth(auth string) (string, string, bool)

func ReadBody

func ReadBody(resp *http.Response) (*bytes.Buffer, error)

Types

type Agent

type Agent interface {
	Query(*Session, *Query) (chan Result, error)
	Name() string
}

type Keys

type Keys struct {
	CensysToken     string
	CensysSecret    string
	Shodan          string
	FofaEmail       string
	FofaKey         string
	QuakeToken      string
	HunterToken     string
	ZoomEyeToken    string
	NetlasToken     string
	CriminalIPToken string
	PublicwwwToken  string
	HunterHowToken  string
	BinaryToken     string
	GithubToken     string
	FullHuntToken   string
}

func (Keys) Empty

func (keys Keys) Empty() bool

type Provider

type Provider struct {
	Shodan     []string `yaml:"shodan"`
	Censys     []string `yaml:"censys"`
	Fofa       []string `yaml:"fofa"`
	Quake      []string `yaml:"quake"`
	Hunter     []string `yaml:"hunter"`
	ZoomEye    []string `yaml:"zoomeye"`
	Netlas     []string `yaml:"netlas"`
	CriminalIP []string `yaml:"criminalip"`
	Publicwww  []string `yaml:"publicwww"`
	HunterHow  []string `yaml:"hunterhow"`
	Binary     []string `yaml:"binary"`
	Github     []string `yaml:"github"`
	FullHunt   []string `json:"fullhunt"`
}

func NewProvider

func NewProvider(location string) *Provider

NewProvider loads provider keys from default location and env variables

func (*Provider) GetKeys

func (provider *Provider) GetKeys() Keys

func (*Provider) HasKeys

func (provider *Provider) HasKeys() bool

HasKeys returns true if at least one agent/source has keys

func (*Provider) LoadProviderConfig

func (provider *Provider) LoadProviderConfig(location string) error

LoadProviderConfig LoadProvidersFrom loads provider config from given location

func (*Provider) LoadProviderKeysFromEnv

func (provider *Provider) LoadProviderKeysFromEnv()

LoadProviderKeysFromEnv loads provider keys from env variables

type Query

type Query struct {
	Query string
	Limit int
}

type Result

type Result struct {
	Timestamp int64  `json:"timestamp" csv:"timestamp"`
	Source    string `json:"source" csv:"source"`
	IP        string `json:"ip" csv:"IP"`
	Port      int    `json:"port" csv:"port"`
	Host      string `json:"host" csv:"host"`
	Url       string `json:"url" csv:"url"`
	Raw       []byte `json:"-" csv:"-"`
	Error     error  `json:"-" csv:"-"`
}

func (*Result) HostPort

func (result *Result) HostPort() string

func (*Result) IpPort

func (result *Result) IpPort() string

func (*Result) JSON

func (result *Result) JSON() string

func (*Result) RawData

func (result *Result) RawData() string

type Session

type Session struct {
	Keys       *Keys
	Client     *retryablehttp.Client
	RetryMax   int
	RateLimits *ratelimit.MultiLimiter
}

Session handles session agent sessions

func NewSession

func NewSession(keys *Keys, retryMax, timeout, rateLimit int, engines []string, duration time.Duration, proxy, proxyAuth string) (*Session, error)

func (*Session) Do

func (s *Session) Do(request *retryablehttp.Request, source string) (*http.Response, error)