subscraping

Documentation

Overview

Package subscraping contains the logic of scraping agents

Index

• func Exists(values []string, key string) bool
• func NewSubdomainExtractor(domain string) (*regexp.Regexp, error)
• type BasicAuth
• type Keys
• type Result
• type ResultType
• type Session
  • func NewSession(domain string, keys *Keys, proxy string, rateLimit, timeout int) (*Session, error)
  • func (s *Session) DiscardHTTPResponse(response *http.Response)
  • func (s *Session) Get(ctx context.Context, getURL, cookies string, headers map[string]string) (*http.Response, error)
  • func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies string, ...) (*http.Response, error)
  • func (s *Session) Post(ctx context.Context, postURL, cookies string, headers map[string]string, ...) (*http.Response, error)
  • func (s *Session) SimpleGet(ctx context.Context, getURL string) (*http.Response, error)
  • func (s *Session) SimplePost(ctx context.Context, postURL, contentType string, body io.Reader) (*http.Response, error)
• type Source

Functions

func Exists

func Exists(values []string, key string) bool

Exists check if a key exist in a slice

func NewSubdomainExtractor

func NewSubdomainExtractor(domain string) (*regexp.Regexp, error)

NewSubdomainExtractor creates a new regular expression to extract subdomains from text based on the given domain.

Types

type BasicAuth

type BasicAuth struct {
	Username string
	Password string
}

BasicAuth request's Authorization header

type Keys

type Keys struct {
	Binaryedge           string   `json:"binaryedge"`
	Bufferover           string   `json:"bufferover"`
	C99                  string   `json:"c99"`
	CensysToken          string   `json:"censysUsername"`
	CensysSecret         string   `json:"censysPassword"`
	Certspotter          string   `json:"certspotter"`
	Chaos                string   `json:"chaos"`
	Chinaz               string   `json:"chinaz"`
	DNSDB                string   `json:"dnsdb"`
	GitHub               []string `json:"github"`
	IntelXHost           string   `json:"intelXHost"`
	IntelXKey            string   `json:"intelXKey"`
	PassiveTotalUsername string   `json:"passivetotal_username"`
	PassiveTotalPassword string   `json:"passivetotal_password"`
	Robtex               string   `json:"robtex"`
	Securitytrails       string   `json:"securitytrails"`
	Shodan               string   `json:"shodan"`
	Spyse                string   `json:"spyse"`
	ThreatBook           string   `json:"threatbook"`
	URLScan              string   `json:"urlscan"`
	Virustotal           string   `json:"virustotal"`
	ZoomEyeUsername      string   `json:"zoomeye_username"`
	ZoomEyePassword      string   `json:"zoomeye_password"`
	ZoomEyeKey           string   `json:"zoomeye_key"`
	FofaUsername         string   `json:"fofa_username"`
	FofaSecret           string   `json:"fofa_secret"`
	FullHunt             string   `json:"fullhunt"`
}

Keys contains the current API Keys we have in store

type Result

type Result struct {
	Type   ResultType
	Source string
	Value  string
	Error  error
}

Result is a result structure returned by a source

type ResultType

type ResultType int

ResultType is the type of result returned by the source

const (
	Subdomain ResultType = iota
	Error
)

Types of results returned by the source

type Session

type Session struct {
	// Extractor is the regex for subdomains created for each domain
	Extractor *regexp.Regexp
	// Keys is the API keys for the application
	Keys *Keys
	// Client is the current http client
	Client *http.Client
	// Rate limit instance
	RateLimiter ratelimit.Limiter
}

Session is the option passed to the source, an option is created uniquely for each source.

func NewSession

func NewSession(domain string, keys *Keys, proxy string, rateLimit, timeout int) (*Session, error)

NewSession creates a new session object for a domain

func (*Session) DiscardHTTPResponse

func (s *Session) DiscardHTTPResponse(response *http.Response)

DiscardHTTPResponse discards the response content by demand

func (*Session) Get

func (s *Session) Get(ctx context.Context, getURL, cookies string, headers map[string]string) (*http.Response, error)

Get makes a GET request to a URL with extended parameters

func (*Session) HTTPRequest

func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies string, headers map[string]string, body io.Reader, basicAuth BasicAuth) (*http.Response, error)

HTTPRequest makes any HTTP request to a URL with extended parameters

func (*Session) Post

func (s *Session) Post(ctx context.Context, postURL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error)

Post makes a POST request to a URL with extended parameters

func (*Session) SimpleGet

func (s *Session) SimpleGet(ctx context.Context, getURL string) (*http.Response, error)

SimpleGet makes a simple GET request to a URL

func (*Session) SimplePost

func (s *Session) SimplePost(ctx context.Context, postURL, contentType string, body io.Reader) (*http.Response, error)

SimplePost makes a simple POST request to a URL

type Source

type Source interface {
	// Run takes a domain as argument and a session object
	// which contains the extractor for subdomains, http client
	// and other stuff.
	Run(context.Context, string, *Session) <-chan Result
	// Name returns the name of the source
	Name() string
}

Source is an interface inherited by each passive source