util

Documentation

Index

• func BodyDomainExtracter(body io.ReadCloser) chan string
• func CountNumLines(filepath string) int64
• func CrawlAllSubdomains(task Task)
• func Deduplicater(in chan string) chan string
• func DomainExtracter(response *http.Response) chan string
• func DomainToURLConverter(domain string) chan string
• func ExpandSubdomains(domain string) chan string
• func ExtractDomains(body io.ReadCloser) chan string
• func HeadersDomainExtracter(header http.Header) chan string
• func Loader(rank int, domain string, numScheduled *int64, numMaxSubdomains int64, ...) chan Task
• func Merger(cs ...chan Result) chan Result
• func Printer(path string, results chan Result) int
• func PrometheusExporter()
• func QueryCNAME(domain string) string
• func Sha1Hash(data string) string
• func StringSliceToChan(s []string) chan string
• func SubdomainFilter(in chan string, suffix string) chan string
• func URLDecode(body string) string
• func Worker(tasks chan Task, numScheduled *int64, numMaxSubdomains int64, ...) chan Result
• type DomainBuilder
  • func (db *DomainBuilder) Append(ch byte)
  • func (db *DomainBuilder) Len() int
  • func (db *DomainBuilder) Reset()
  • func (db *DomainBuilder) String() string
  • func (db *DomainBuilder) StringBuilder() string
  • func (db *DomainBuilder) StringSlow() string
  • func (db *DomainBuilder) StringUnsafe() string
• type Result
  • func NewResult(task Task) Result
  • func Processer(task Task, suffix string) (result Result)
  • func (r Result) ToJSON() []byte
• type Task
  • func NewTask(rank int, url, domain string) Task

Functions

func BodyDomainExtracter

func BodyDomainExtracter(body io.ReadCloser) chan string

BodyDomainExtracter extracts domains from response body

func CountNumLines

func CountNumLines(filepath string) int64

CountNumLines counts the number of lines in a file

func CrawlAllSubdomains

func CrawlAllSubdomains(task Task)

CrawlAllSubdomains crawls all subdomains of domain

func Deduplicater

func Deduplicater(in chan string) chan string

func DomainExtracter

func DomainExtracter(response *http.Response) chan string

DomainExtracter extracts domains from *http.Response

func DomainToURLConverter

func DomainToURLConverter(domain string) chan string

func ExpandSubdomains

func ExpandSubdomains(domain string) chan string

ExpandSubdomains returns all subdomains of domain

func ExtractDomains

func ExtractDomains(body io.ReadCloser) chan string

ExtractDomains extract domain name from the given reader

func HeadersDomainExtracter

func HeadersDomainExtracter(header http.Header) chan string

HeadersDomainExtracter extracts domains from response header (e.g., Content-Security-Policy)

func Loader

func Loader(rank int, domain string, numScheduled *int64, numMaxSubdomains int64, wg *sync.WaitGroup, scheduled *sync.Map) chan Task

func Merger

func Merger(cs ...chan Result) chan Result

func Printer

func Printer(path string, results chan Result) int

func PrometheusExporter

func PrometheusExporter()

PrometheusExporter exports metrics to Prometheus, it will starts a web server listening on the port 2112

func QueryCNAME

func QueryCNAME(domain string) string

func Sha1Hash

func Sha1Hash(data string) string

func StringSliceToChan

func StringSliceToChan(s []string) chan string

func SubdomainFilter

func SubdomainFilter(in chan string, suffix string) chan string

SubdomainFilter returns a channel in which the domain has the given suffix

func URLDecode

func URLDecode(body string) string

URLDecode decodes a string to a plain URL once

func Worker

func Worker(tasks chan Task, numScheduled *int64, numMaxSubdomains int64, scheduled *sync.Map, wg *sync.WaitGroup, suffix string) chan Result

Types

type DomainBuilder

type DomainBuilder struct {
	// contains filtered or unexported fields
}

func (*DomainBuilder) Append

func (db *DomainBuilder) Append(ch byte)

func (*DomainBuilder) Len

func (db *DomainBuilder) Len() int

func (*DomainBuilder) Reset

func (db *DomainBuilder) Reset()

func (*DomainBuilder) String

func (db *DomainBuilder) String() string

func (*DomainBuilder) StringBuilder

func (db *DomainBuilder) StringBuilder() string

func (*DomainBuilder) StringSlow

func (db *DomainBuilder) StringSlow() string

func (*DomainBuilder) StringUnsafe

func (db *DomainBuilder) StringUnsafe() string

type Result

type Result struct {
	Task
	RequestHeaders     http.Header `json:"request_headers"`
	CNAME              string      `json:"cname"`
	ResponseStatus     string      `json:"response_status"`
	ResponseStatusCode int         `json:"response_status_code"`
	ResponseProto      string      `json:"response_proto"`
	ResponseProtoMajor int         `json:"response_proto_major"`
	ResponseProtoMinor int         `json:"response_proto_minor"`
	ResponseHeaders    http.Header `json:"response_headers"`
	StartTime          int64       `json:"start_time"`
	EndTime            int64       `json:"end_time"`
	Subdomains         []string    `json:"subdomains"`
	Error              string      `json:"error"`
}

Crawl crawls one single domain, returns all subdomains matched in the response and write the task log to logs channel.

func NewResult

func NewResult(task Task) Result

func Processer

func Processer(task Task, suffix string) (result Result)

func (Result) ToJSON

func (r Result) ToJSON() []byte

type Task

type Task struct {
	Rank   int    `json:"rank"`
	URL    string `json:"url"`
	Domain string `json:"domain"`
}

func NewTask

func NewTask(rank int, url, domain string) Task