scraper

Documentation

Index

• Variables
• type ContentProvider
  • func GetContentProvider(rawurl string) ContentProvider
• type WebPage
• type WebScraper
  • func NewExternalWebScraper(httpClient *http.Client, uri string) (WebScraper, error)
  • func NewInternalWebScraper(httpClient *http.Client) WebScraper
  • func NewWebScraper(httpClient *http.Client, uri string) (WebScraper, error)

Variables

var ContentProviders = []ContentProvider{}

ContentProviders is the registry of all supported content provider

Types

type ContentProvider

type ContentProvider interface {
	Get(ctx context.Context, rawurl string) (*WebPage, error)
	Match(url string) bool
}

ContentProvider is a content provider interface

func GetContentProvider

func GetContentProvider(rawurl string) ContentProvider

GetContentProvider return content provider that match the given URL

type WebPage

type WebPage struct {
	URL      string `json:"url,omitempty"`
	Title    string `json:"title,omitempty"`
	HTML     string `json:"html,omitempty"`
	Text     string `json:"text,omitempty"`
	Length   int    `json:"length,omitempty"`
	Excerpt  string `json:"excerpt,omitempty"`
	SiteName string `json:"sitename,omitempty"`
	Image    string `json:"image,omitempty"`
	Favicon  string `json:"favicon,omitempty"`
}

WebPage is the result of a web scraping

type WebScraper

type WebScraper interface {
	Scrap(ctx context.Context, rawurl string) (*WebPage, error)
}

WebScraper is an interface with Web Scrapping provider

func NewExternalWebScraper

func NewExternalWebScraper(httpClient *http.Client, uri string) (WebScraper, error)

NewExternalWebScraper create an external web scrapping service

func NewInternalWebScraper

func NewInternalWebScraper(httpClient *http.Client) WebScraper

NewInternalWebScraper create an internal web scrapping service

func NewWebScraper

func NewWebScraper(httpClient *http.Client, uri string) (WebScraper, error)

NewWebScraper create new Web Scraping service