web

Oct 28, 2024

Changes in this version

+ type CrawlOptions struct

+ AllowedDomains []string

+ ConcurrentJobs int

+ MaxDepth int

+ Timeout int

+ type CrawlResult struct

+ Files []filesystem.FileInfo

+ TreeString string

+ func ProcessWebURL(urlStr string, options CrawlOptions, excludePatterns []string) (*CrawlResult, error)

+ type Crawler struct

+ func NewCrawler(options CrawlOptions) *Crawler

+ func (c *Crawler) Crawl(startURL string) ([]*WebPage, error)

+ func (c *Crawler) SetExcludePatterns(patterns []string)

+ type WebPage struct

+ Content string

+ ContentType string

+ Depth int

+ Links []string

+ StatusCode int

+ Title string

+ URL string