core

Documentation

Index

• Variables
• type Cache
  • func (c *Cache) AddVisited(url string)
  • func (c *Cache) Flush()
  • func (c *Cache) IsVisited(url string) bool
• type Crawler
  • func NewCrawler(url string, level int, liveMode bool, exportFile string, ...) *Crawler
  • func (c *Crawler) AddMatches(page webtree.Page)
  • func (c *Crawler) Crawl()
  • func (c *Crawler) CrawlNodeBlock(w *webtree.Node)
  • func (c *Crawler) CrawlNodeLive(w *webtree.Node)
  • func (c *Crawler) Export(tree webtree.Node, format string, filename string) error
  • func (c *Crawler) ExportJSON(root webtree.Node, filename string) error
  • func (c *Crawler) ExportTXT(root webtree.Node, filename string) error
  • func (c *Crawler) ExportXML(tree webtree.Node, filename string) error
  • func (c *Crawler) ExtractLinks(page *webtree.Page) (links []string)
  • func (c *Crawler) Fetch(page *webtree.Page)
  • func (c *Crawler) IsSkipablePage(page webtree.Page) bool
  • func (c *Crawler) SaveResults(root webtree.Node)

Variables

var (
	GeneralRegex = `((?:https?)://[\w\-]+(?:\.[\w\-]+)+[\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])`
)

Types

type Cache

type Cache struct {
	Visited map[string]bool
	Lock    sync.Mutex
}

func (*Cache) AddVisited

func (c *Cache) AddVisited(url string)

func (*Cache) Flush

func (c *Cache) Flush()

func (*Cache) IsVisited

func (c *Cache) IsVisited(url string) bool

type Crawler

type Crawler struct {
	RootURL        string
	Level          int
	LiveMode       bool
	ExportFile     string
	RegexMap       map[string]string
	ExcludedStatus []int
	IncludedUrls   []string
	Client         *http.Client
	Cache          Cache
}

func NewCrawler

func NewCrawler(url string, level int, liveMode bool, exportFile string, regexMap map[string]string, statusResponses []int, includes []string) *Crawler

func (*Crawler) AddMatches

func (c *Crawler) AddMatches(page webtree.Page)

func (*Crawler) Crawl

func (c *Crawler) Crawl()

func (*Crawler) CrawlNodeBlock

func (c *Crawler) CrawlNodeBlock(w *webtree.Node)

func (*Crawler) CrawlNodeLive

func (c *Crawler) CrawlNodeLive(w *webtree.Node)

func (*Crawler) Export

func (c *Crawler) Export(tree webtree.Node, format string, filename string) error

func (*Crawler) ExportJSON

func (c *Crawler) ExportJSON(root webtree.Node, filename string) error

func (*Crawler) ExportTXT

func (c *Crawler) ExportTXT(root webtree.Node, filename string) error

func (*Crawler) ExportXML

func (c *Crawler) ExportXML(tree webtree.Node, filename string) error

func (*Crawler) ExtractLinks

func (c *Crawler) ExtractLinks(page *webtree.Page) (links []string)

func (*Crawler) Fetch

func (c *Crawler) Fetch(page *webtree.Page)

func (*Crawler) IsSkipablePage

func (c *Crawler) IsSkipablePage(page webtree.Page) bool

func (*Crawler) SaveResults

func (c *Crawler) SaveResults(root webtree.Node)