scraper

Documentation

Index

• func GetClient(rawURL string) (*http.Client, error)
• func ParsePage(pageNumber int, resp *http.Response) ([]string, error)
• type Album
• type Config
• type Page
• type Review
  • func ParseReview(reviewID string, resp *http.Response) (*Review, error)
  • func (review *Review) PrintInfo()
• type Scraper
  • func (scraper *Scraper) ScrapePage(pageNumber int) (string, *http.Response, error)
  • func (scraper *Scraper) ScrapeReview(reviewID string) (string, *http.Response, error)

Functions

func GetClient

func GetClient(rawURL string) (*http.Client, error)

GetClient builds a HTTP client

func ParsePage

func ParsePage(pageNumber int, resp *http.Response) ([]string, error)

ParsePage ...

Types

type Album

type Album struct {
	Artists []string `json:"artists"`
	Title   string   `json:"title"`
	Score   string   `json:"score"`
	Labels  []string `json:"labels"`
	Year    string   `json:"year"`
	// contains filtered or unexported fields
}

Album ...

type Config

type Config struct {
	ProxyRawURL     string `json:"proxyRawURL"`
	OutputDirectory string `json:"outputDirectory"`
}

Config ...

type Page

type Page struct {
	PageNumber int
	// contains filtered or unexported fields
}

Page ...

type Review

type Review struct {
	ReviewID string   `json:"id"`
	Albums   []*Album `json:"albums"`
	Authors  []string `json:"authors"`
	Genres   []string `json:"genres"`
	Article  string   `json:"article"`
	// contains filtered or unexported fields
}

Review ...

func ParseReview

func ParseReview(reviewID string, resp *http.Response) (*Review, error)

ParseReview ...

func (*Review) PrintInfo

func (review *Review) PrintInfo()

PrintInfo ...

type Scraper

type Scraper struct {
	Client          *http.Client
	OutputDirectory string
}

Scraper ...

func (*Scraper) ScrapePage

func (scraper *Scraper) ScrapePage(pageNumber int) (string, *http.Response, error)

ScrapePage ...

func (*Scraper) ScrapeReview

func (scraper *Scraper) ScrapeReview(reviewID string) (string, *http.Response, error)

ScrapeReview ...