crawler

Documentation

Index

• func CleanSubdomain(s string) string
• func Crawl(config *Config)
• func DecodeChars(s string) string
• func FilterNewLines(s string) string
• func FixUrl(mainSite *url.URL, nextLoc string) string
• func GetDomain(site *url.URL) string
• func GetExtType(rawUrl string) string
• func GetRawCookie(cookies []*http.Cookie) string
• func InScope(u *url.URL, regexps []*regexp.Regexp) bool
• func LinkFinder(source string) []string
• func LoadCookies(rawCookie string) []*http.Cookie
• func NormalizePath(path string) string
• func ReadingLines(filename string) []string
• func Unique(intSlice []string) []string
• type Config
• type Stats
  • func (s *Stats) String() string
• type WordMap
  • func NewWordMap() *WordMap
  • func (w *WordMap) Add(word string)
  • func (w *WordMap) Sort()

Functions

func CleanSubdomain

func CleanSubdomain(s string) string

func Crawl

func Crawl(config *Config)

func DecodeChars

func DecodeChars(s string) string

func FilterNewLines

func FilterNewLines(s string) string

func FixUrl

func FixUrl(mainSite *url.URL, nextLoc string) string

func GetDomain

func GetDomain(site *url.URL) string

func GetExtType

func GetExtType(rawUrl string) string

func GetRawCookie

func GetRawCookie(cookies []*http.Cookie) string

func InScope

func InScope(u *url.URL, regexps []*regexp.Regexp) bool

func LinkFinder

func LinkFinder(source string) []string

func LoadCookies

func LoadCookies(rawCookie string) []*http.Cookie

func NormalizePath

func NormalizePath(path string) string

NormalizePath the path

func ReadingLines

func ReadingLines(filename string) []string

ReadingLines Reading file and return content as []string

func Unique

func Unique(intSlice []string) []string

Types

type Config

type Config struct {
	URL               string
	List              bool
	Sub               bool
	Js                bool
	Depth             int
	SkipSSL           bool
	Threads           int
	Append            string
	Prepend           string
	Proxy             string
	Cookies           string
	UserAgent         string
	OutputPath        string
	OutputFilename    string
	MinimumWordCount  int
	MininumWordLength int
	MaximumWordLength int
	Verbose           bool
	Domains           []string
	Permutations      []string
	Headers           []string
	Offsite           bool
	ShowCount         bool
}

type Stats

type Stats struct {
	RequestCount  uint64
	ResponseCount uint64
	ErrorCount    uint64
	TotalWords    uint64
	TotalTime     float64
}

func (*Stats) String

func (s *Stats) String() string

type WordMap

type WordMap struct {
	// contains filtered or unexported fields
}

func NewWordMap

func NewWordMap() *WordMap

func (*WordMap) Add

func (w *WordMap) Add(word string)

func (*WordMap) Sort

func (w *WordMap) Sort()