subscraping

Documentation

Index

• func Exists(values []string, key string) bool
• func NewSubdomainExtractor(domain string) (*regexp.Regexp, error)
• type Keys
• type Result
• type ResultType
• type Session
  • func NewSession(domain string, keys Keys, timeout int) (*Session, error)
  • func (s *Session) Get(ctx context.Context, url string, cookies string, headers map[string]string) (*http.Response, error)
  • func (s *Session) NormalGetWithContext(ctx context.Context, url string) (*http.Response, error)
• type Source

Functions

func Exists

func Exists(values []string, key string) bool

Exists check if a key exist in a slice

func NewSubdomainExtractor

func NewSubdomainExtractor(domain string) (*regexp.Regexp, error)

NewSubdomainExtractor creates a new regular expression to extract subdomains from text based on the given domain.

Types

type Keys

type Keys struct {
	Binaryedge           string `json:"binaryedge"`
	CensysToken          string `json:"censysUsername"`
	CensysSecret         string `json:"censysPassword"`
	Certspotter          string `json:"certspotter"`
	Chaos                string `json:"chaos"`
	DNSDB                string `json:"dnsdb"`
	PassiveTotalUsername string `json:"passivetotal_username"`
	PassiveTotalPassword string `json:"passivetotal_password"`
	Securitytrails       string `json:"securitytrails"`
	Shodan               string `json:"shodan"`
	Spyse                string `json:"spyse"`
	URLScan              string `json:"urlscan"`
	Virustotal           string `json:"virustotal"`
	ZoomEyeUsername      string `json:"zoomeye_username"`
	ZoomEyePassword      string `json:"zoomeye_password"`
}

Keys contains the current API Keys we have in store

type Result

type Result struct {
	Type   ResultType
	Source string
	Value  string
	Error  error
}

Result is a result structure returned by a source

type ResultType

type ResultType int

ResultType is the type of result returned by the source

const (
	Subdomain ResultType = iota
	Error
)

Types of results returned by the source

type Session

type Session struct {
	// Extractor is the regex for subdomains created for each domain
	Extractor *regexp.Regexp
	// Keys is the API keys for the application
	Keys Keys
	// Client is the current http client
	Client *http.Client
}

Session is the option passed to the source, an option is created uniquely for eac source.

func NewSession

func NewSession(domain string, keys Keys, timeout int) (*Session, error)

NewSession creates a new session object for a domain

func (*Session) Get

func (s *Session) Get(ctx context.Context, url string, cookies string, headers map[string]string) (*http.Response, error)

Get makes a GET request to a URL

func (*Session) NormalGetWithContext

func (s *Session) NormalGetWithContext(ctx context.Context, url string) (*http.Response, error)

NormalGetWithContext makes a normal GET request to a URL with context

type Source

type Source interface {
	// Run takes a domain as argument and a session object
	// which contains the extractor for subdomains, http client
	// and other stuff.
	Run(context.Context, string, *Session) <-chan Result
	// Name returns the name of the source
	Name() string
}

Source is an interface inherited by each passive source