scraper

Documentation

Index

• Variables
• func DeleteScrapeCache() error
• func RegisterScraper(id string, name string, f ScraperFunc)
• func Scrape(wg *sync.WaitGroup, out chan<- *ScrapedEpisode) error
• type Episode
• type ScrapedEpisode
  • func (s *ScrapedEpisode) Log() error
  • func (s *ScrapedEpisode) ToJSON() ([]byte, error)
• type Scraper
  • func GetScrapers() []Scraper
• type ScraperFunc

Variables

var UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36"

Functions

func DeleteScrapeCache

func DeleteScrapeCache() error

func RegisterScraper

func RegisterScraper(id string, name string, f ScraperFunc)

func Scrape

func Scrape(wg *sync.WaitGroup, out chan<- *ScrapedEpisode) error

Types

type Episode

type Episode = sqlc.Episode

type ScrapedEpisode

type ScrapedEpisode struct {
	ID            int32     `json:"id"`
	Name          string    `json:"name"`
	NumberSeries  int32     `json:"number_series"`
	NumberOverall int32     `json:"number_overall"`
	ReleaseDate   time.Time `json:"release_date"`
	Description   string    `json:"description"`
	Body          string    `json:"body"`
	TranscriptUrl string    `json:"transcript_url"`
	PodcastID     int32     `json:"podcast_id"`
	SeriesID      int32     `json:"series_id"`
	LastUpdated   time.Time `json:"last_updated"`
	Headline      string    `json:"headline"`
}

func (*ScrapedEpisode) Log

func (s *ScrapedEpisode) Log() error

func (*ScrapedEpisode) ToJSON

func (s *ScrapedEpisode) ToJSON() ([]byte, error)

type Scraper

type Scraper struct {
	ID     string
	Name   string
	Scrape ScraperFunc
}

func GetScrapers

func GetScrapers() []Scraper

type ScraperFunc

type ScraperFunc func(*sync.WaitGroup, chan<- *ScrapedEpisode) error