web

Documentation

Index

• type CrawlOptions
• type CrawlResult
  • func ProcessWebURL(urlStr string, options CrawlOptions, excludePatterns []string) (*CrawlResult, error)
• type Crawler
  • func NewCrawler(options CrawlOptions, startURL string) *Crawler
  • func (c *Crawler) Crawl(startURL string) ([]*WebPage, error)
  • func (c *Crawler) SetExcludePatterns(patterns []string)
• type WebPage

Types

type CrawlOptions

type CrawlOptions struct {
	MaxDepth       int
	AllowedDomains []string
	Timeout        int
	ConcurrentJobs int
}

type CrawlResult

type CrawlResult struct {
	TreeString string
	Files      []filesystem.FileInfo
}

func ProcessWebURL

func ProcessWebURL(urlStr string, options CrawlOptions, excludePatterns []string) (*CrawlResult, error)

type Crawler

type Crawler struct {
	// contains filtered or unexported fields
}

func NewCrawler

func NewCrawler(options CrawlOptions, startURL string) *Crawler

func (*Crawler) Crawl

func (c *Crawler) Crawl(startURL string) ([]*WebPage, error)

func (*Crawler) SetExcludePatterns

func (c *Crawler) SetExcludePatterns(patterns []string)

type WebPage

type WebPage struct {
	URL         string
	Content     string
	Title       string
	Links       []string
	Depth       int
	StatusCode  int
	ContentType string
}