caching_downloader

Documentation

Index

• func GetCacheFileNameFromURL(uri string) string
• func Sha1OfURL(uri string) string
• type Cache
• type DirectoryCache
  • func NewDirectoryCache(dir string) (*DirectoryCache, error)
  • func (c *DirectoryCache) GetPageIDs() ([]string, error)
  • func (c *DirectoryCache) ReadFile(name string) ([]byte, error)
  • func (c *DirectoryCache) Remove(name string)
  • func (c *DirectoryCache) WriteFile(name string, data []byte) error
• type Downloader
  • func New(cache Cache, client *notionapi.Client) *Downloader
  • func (d *Downloader) DownloadFile(uri string, blockID string) (*notionapi.DownloadFileResponse, error)
  • func (d *Downloader) DownloadPage(pageID string) (*notionapi.Page, error)
  • func (d *Downloader) DownloadPagesRecursively(startPageID string, afterDownload func(*notionapi.Page) error) ([]*notionapi.Page, error)
  • func (d *Downloader) GetClientCopy() *notionapi.Client
  • func (d *Downloader) NameForPageID(pageID string) string
  • func (d *Downloader) ReadPageFromCache(pageID string) (*notionapi.Page, error)
• type EventDidDownload
• type EventDidReadFromCache
• type EventError
• type EventGotVersions

Functions

func GetCacheFileNameFromURL

func GetCacheFileNameFromURL(uri string) string

GetCacheFileNameFromURL returns name of the file in cache for this URL. The name is files/${sha1OfURL}.${ext} It's a consistent, one-way transform

func Sha1OfURL

func Sha1OfURL(uri string) string

Sha1OfURL returns sha1 of url

Types

type Cache

type Cache interface {
	// ReadFile reads a file with a given name from cache
	ReadFile(string) ([]byte, error)
	// WriteFile writes a file with a given name to cache
	WriteFile(string, []byte) error
	// GetPageIDs returns ids of pages in the cache
	GetPageIDs() ([]string, error)
	// Remove removes a file with a given name from cache
	Remove(string)
}

Cache describes a caching interface

type DirectoryCache

type DirectoryCache struct {
	Dir string
	// contains filtered or unexported fields
}

DirectoryCache implements disk-based Cache interface

func NewDirectoryCache

func NewDirectoryCache(dir string) (*DirectoryCache, error)

NewDirectoryCache returns a new DirectoryCache which caches files in a directory

func (*DirectoryCache) GetPageIDs

func (c *DirectoryCache) GetPageIDs() ([]string, error)

GetPageIDs returns ids of pages in the cache

func (*DirectoryCache) ReadFile

func (c *DirectoryCache) ReadFile(name string) ([]byte, error)

ReadFile reads a file with a given name from cache

func (*DirectoryCache) Remove

func (c *DirectoryCache) Remove(name string)

Remove removes a file with a given name from cache

func (*DirectoryCache) WriteFile

func (c *DirectoryCache) WriteFile(name string, data []byte) error

WriteFile writes a file with a given name to cache

type Downloader

type Downloader struct {
	Client *notionapi.Client
	// cached pages are stored in Cache as ${pageID}.txt files
	Cache Cache
	// NoReadCache disables reading from cache i.e. downloaded pages
	// will be written to cache but not read from it
	NoReadCache bool
	// if true, we'll re-download a page if a newer version is
	// on the server
	RedownloadNewerVersions bool
	// maps id of the page (in the no-dash format) to a cached Page
	IdToPage map[string]*notionapi.Page
	// maps id of the page (in the no-dash format) to latest version
	// of the page available on the server.
	// if doesn't exist, we haven't yet queried the server for the
	// version
	IdToPageLatestVersion map[string]int64

	// for diagnostics, number of downloaded pages
	DownloadedCount int
	// number of pages we got from cache
	FromCacheCount int

	// for diagnostics, number of downloaded files
	DownloadedFilesCount int
	// number of files we got from cache
	FilesFromCacheCount int

	EventObserver func(interface{})
	// contains filtered or unexported fields
}

Downloader implements optimized (cached) downloading of pages from the server. Cache of pages is stored in CacheDir. We return pages from cache. If RedownloadNewerVersions is true, we'll re-download latest version of the page (as opposed to returning possibly outdated version from cache). We do it more efficiently than just blindly re-downloading.

func New

func New(cache Cache, client *notionapi.Client) *Downloader

New returns a new Downloader which caches page loads on disk and can return pages from that cache

func (*Downloader) DownloadFile

func (d *Downloader) DownloadFile(uri string, blockID string) (*notionapi.DownloadFileResponse, error)

DownloadFile downloads a file, caching in the cache

func (*Downloader) DownloadPage

func (d *Downloader) DownloadPage(pageID string) (*notionapi.Page, error)

func (*Downloader) DownloadPagesRecursively

func (d *Downloader) DownloadPagesRecursively(startPageID string, afterDownload func(*notionapi.Page) error) ([]*notionapi.Page, error)

func (*Downloader) GetClientCopy

func (d *Downloader) GetClientCopy() *notionapi.Client

GetClientCopy returns a copy of client

func (*Downloader) NameForPageID

func (d *Downloader) NameForPageID(pageID string) string

NameForPageID returns name of the file used for storing HTTP cache for a given page

func (*Downloader) ReadPageFromCache

func (d *Downloader) ReadPageFromCache(pageID string) (*notionapi.Page, error)

ReadPageFromCache returns a page read from the cache

type EventDidDownload

type EventDidDownload struct {
	// if page, PageID is set
	PageID string
	// if file, URL is set
	FileURL string
	// how long did it take to download
	Duration time.Duration
}

EventDidDownload is for logging. Emitted when page or file is downloaded

type EventDidReadFromCache

type EventDidReadFromCache struct {
	// if page, PageID is set
	PageID string
	// if file, URL is set
	FileURL string
	// how long did it take to download
	Duration time.Duration
}

EventDidReadFromCache is for logging. Emitted when page or file is read from cache.

type EventError

type EventError struct {
	Error string
}

EventError is for logging. Emitted when there's error to log

type EventGotVersions

type EventGotVersions struct {
	Count    int
	Duration time.Duration
}

EventGotVersions is for logging. Emitted