lib

Documentation

Index

• type App
  • func (a *App) Scrape(h *Host) (*ScrapeResult, error)
  • func (a *App) Worker(id int, q WorkerQueues)
• type Book
• type BooksQueryResult
• type CalibreBook
• type DownloadBookRequest
• type DownloadBookResponse
• type GetBooksRequest
• type GetBooksResponse
• type GetIDSRequest
• type GetIDSResponse
• type Host
  • func (h *Host) Print(verbose bool)
  • func (h *Host) Scrape(workers, stepSize int, userAgent, outputDir string, semaphore chan bool) (*ScrapeResult, error)
  • func (h *Host) Stats(n int) (fails, downloads int)
• type ScrapeResult
  • func (s *ScrapeResult) Print()
• type SearchResult
• type WorkerCounter
• type WorkerQueues

Types

type App

type App struct {
	UserAgent       string
	Timeout         time.Duration
	DownloadTimeout time.Duration
	WorkerInterval  time.Duration
	StepSize        int
	OutputDir       string
	Queues          WorkerQueues
}

App holds all the config for the V2 demeter type

func (*App) Scrape

func (a *App) Scrape(h *Host) (*ScrapeResult, error)

Scrape performs the actual scrape

func (*App) Worker

func (a *App) Worker(id int, q WorkerQueues)

Worker is the only unit that actually makes requests

type Book

type Book struct {
	ID       int `storm:"id,increment"`
	Added    time.Time
	Hash     string `storm:"unique"`
	SourceID int
	Author   string
	Title    string
}

Book is a oversimplified representation of a book

type BooksQueryResult

type BooksQueryResult map[string]CalibreBook

BooksQueryResult is

type CalibreBook

type CalibreBook struct {
	UUID          string            `json:"uuid"`
	Title         string            `json:"title"`
	ApplicationID int               `json:"application_id"`
	TitleSort     string            `json:"title_sort"`
	Cover         string            `json:"cover"`
	Pubdate       string            `json:"pubdate"`
	MainFormat    map[string]string `json:"main_format"`
	AuthorSort    string            `json:"author_sort"`
	Authors       []string          `json:"authors"`
	Languages     []string          `json:"languages"`
	LastModified  time.Time         `json:"last_modified"`
	Thumbnail     string            `json:"thumbnail"`
	Formats       []string          `json:"formats"`
}

CalibreBook is a book from calibre

type DownloadBookRequest

type DownloadBookRequest struct {
	URL  string
	Path string
	Resp chan DownloadBookResponse
}

DownloadBookRequest holds the request to DL a book

type DownloadBookResponse

type DownloadBookResponse struct {
	Err error
}

DownloadBookResponse holds the result of a book dl

type GetBooksRequest

type GetBooksRequest struct {
	IDs  []int
	U    url.URL
	Resp chan GetBooksResponse
}

GetBooksRequest converts IDs into books

type GetBooksResponse

type GetBooksResponse struct {
	Books BooksQueryResult
	Err   error
}

GetBooksResponse is the response

type GetIDSRequest

type GetIDSRequest struct {
	Num    int
	Offset int
	U      url.URL
	Resp   chan GetIDSResponse
}

GetIDSRequest holds the information to retrieve the book ids from a calibre host

type GetIDSResponse

type GetIDSResponse struct {
	IDs []int
	Err error
}

GetIDSResponse is the response for IDS request

type Host

type Host struct {
	ID            int    `storm:"id,increment"`
	URL           string `storm:"unique"`
	Downloads     int
	Scrapes       int
	LastScrape    time.Time
	LastDownload  time.Time
	Added         time.Time
	ScrapeResults []ScrapeResult
	Active        bool
}

Host describes all attributes related to a host

func (*Host) Print

func (h *Host) Print(verbose bool)

Print prints a host in a nicely formatted way

func (*Host) Scrape

func (h *Host) Scrape(workers, stepSize int, userAgent, outputDir string, semaphore chan bool) (*ScrapeResult, error)

Scrape performs the actual scrape

func (*Host) Stats

func (h *Host) Stats(n int) (fails, downloads int)

Stats returns usefull stats about the last n scrape runs of a host

type ScrapeResult

type ScrapeResult struct {
	Start     time.Time
	End       time.Time
	Success   bool
	Results   int
	Downloads int
}

ScrapeResult is the result of a single scrape attempt

func (*ScrapeResult) Print

func (s *ScrapeResult) Print()

Print prints a scrapeResult in a nicely formatted way

type SearchResult

type SearchResult struct {
	TotalNum  int    `json:"total_num"`
	Offset    int    `json:"offset"`
	Sort      string `json:"sort"`
	LibraryID string `json:"library_id"`
	SortOrder string `json:"sort_order"`
	Vl        string `json:"vl"`
	BaseURL   string `json:"base_url"`
	Num       int    `json:"num"`
	BookIds   []int  `json:"book_ids"`
	Query     string `json:"query"`
}

SearchResult contains the results from a calibre query

type WorkerCounter

type WorkerCounter struct {
	GetIDS   int
	GetBooks int
	DlBook   int
	ID       int
}

WorkerCounter counts all the work a worker did

type WorkerQueues

type WorkerQueues struct {
	IDS    chan GetIDSRequest
	Books  chan GetBooksRequest
	DlBook chan DownloadBookRequest
}

WorkerQueues holds all the queues that the worker can act upon