cacheddownloader

README

CachedDownloader

CachedDownloader is responsible for downloading and caching files and maintaining reference counts for each cache entry. Entries in the cache with no active references are ejected from the cache when new space is needed.

[!NOTE]

This repository should be imported as code.cloudfoundry.org/cacheddownloader.

Contributing

See the Contributing.md for more information on how to contribute.

Working Group Charter

This repository is maintained by App Runtime Platform under Diego area.

[!IMPORTANT]

Content in this file is managed by the CI task sync-readme and is generated by CI following a convention.

Documentation

Index

• Constants
• Variables
• func HexValue(algorithm, content string) (string, error)
• func HexValueForByteArray(algorithm string, content []byte) (string, error)
• func New(downloader *Downloader, cache *FileCache, transformer CacheTransformer) (*cachedDownloader, error)
• func NewChecksumFailedError(msg, expected, received string) error
• func NewDownloadCancelledError(source string, duration time.Duration, written int64, additionalError error) error
• func NewHashValidator(algorithm string) (*hashValidator, error)
• func NoopTransform(source, destination string) (int64, error)
• func TarTransform(source string, destination string) (int64, error)
• type CacheTransformer
• type CachedDownloader
• type CachedFile
  • func NewFileCloser(file *os.File, onClose func(string)) *CachedFile
  • func (fw *CachedFile) Close() error
• type CachingInfoType
  • func (c CachingInfoType) Equal(other CachingInfoType) bool
• type ChecksumFailedError
  • func (e *ChecksumFailedError) Error() string
• type ChecksumInfoType
• type DownloadCancelledError
  • func (e *DownloadCancelledError) Error() string
• type Downloader
  • func NewDownloader(requestTimeout time.Duration, maxConcurrentDownloads int, ...) *Downloader
  • func NewDownloaderWithIdleTimeout(requestTimeout time.Duration, idleTimeout time.Duration, ...) *Downloader
  • func (downloader *Downloader) Download(logger lager.Logger, url *url.URL, createDestination func() (*os.File, error), ...) (path string, cachingInfoOut CachingInfoType, err error)
• type FileCache
  • func NewCache(dir string, maxSizeInBytes int64) *FileCache
  • func (c *FileCache) Add(logger lager.Logger, cacheKey, sourcePath string, size int64, ...) (*CachedFile, error)
  • func (c *FileCache) AddDirectory(logger lager.Logger, cacheKey, sourcePath string, size int64, ...) (string, error)
  • func (c *FileCache) CloseDirectory(logger lager.Logger, cacheKey, dirPath string) error
  • func (c *FileCache) Get(logger lager.Logger, cacheKey string) (*CachedFile, CachingInfoType, error)
  • func (c *FileCache) GetDirectory(logger lager.Logger, cacheKey string) (string, CachingInfoType, error)
  • func (c *FileCache) Remove(logger lager.Logger, cacheKey string)
• type FileCacheEntry

Constants

const (
	MAX_DOWNLOAD_ATTEMPTS = 4
	IDLE_TIMEOUT          = 10 * time.Second
	RETRY_WAIT_MIN        = 500 * time.Millisecond
	RETRY_WAIT_MAX        = 5 * time.Second
	MAX_JITTER            = 200 * time.Millisecond
	NoBytesReceived       = -1
)

Variables

var (
	EntryNotFound          = errors.New("Entry Not Found")
	AlreadyClosed          = errors.New("Already closed directory")
	MissingCacheKeyErr     = errors.New("Not cacheable directory: cache key is missing")
	MissingCacheHeadersErr = errors.New("Not cacheable directory: ETag and Last-Modified were missing from response")
)

var ErrUnknownArchiveFormat = errors.New("unknown archive format")

Functions

func HexValue

func HexValue(algorithm, content string) (string, error)

func HexValueForByteArray

func HexValueForByteArray(algorithm string, content []byte) (string, error)

func New

func New(
	downloader *Downloader,
	cache *FileCache,
	transformer CacheTransformer,
) (*cachedDownloader, error)

A transformer function can be used to do post-download processing on the file before it is stored in the cache.

func NewChecksumFailedError

func NewChecksumFailedError(msg, expected, received string) error

func NewDownloadCancelledError

func NewDownloadCancelledError(source string, duration time.Duration, written int64, additionalError error) error

func NewHashValidator

func NewHashValidator(algorithm string) (*hashValidator, error)

func NoopTransform

func NoopTransform(source, destination string) (int64, error)

func TarTransform

func TarTransform(source string, destination string) (int64, error)

Types

type CacheTransformer

type CacheTransformer func(source, destination string) (newSize int64, err error)

called after a new object has entered the cache. it is assumed that `path` will be removed, if a new path is returned. a noop transformer returns the given path and its detected size.

type CachedDownloader

type CachedDownloader interface {
	// Fetch downloads the file at the given URL and stores it in the cache with the given cacheKey.
	// If cacheKey is empty, the file will not be saved in the cache.
	//
	// Fetch returns a stream that can be used to read the contents of the downloaded file. While this stream is active (i.e., not yet closed),
	// the associated cache entry will be considered in use and will not be ejected from the cache.
	Fetch(logger lager.Logger, urlToFetch *url.URL, cacheKey string, checksum ChecksumInfoType, cancelChan <-chan struct{}) (stream io.ReadCloser, size int64, err error)

	// FetchAsDirectory downloads the tarfile pointed to by the given URL, expands the tarfile into a directory, and returns the path of that directory as well as the total number of bytes downloaded.
	FetchAsDirectory(logger lager.Logger, urlToFetch *url.URL, cacheKey string, checksum ChecksumInfoType, cancelChan <-chan struct{}) (dirPath string, size int64, err error)

	// CloseDirectory decrements the usage counter for the given cacheKey/directoryPath pair.
	// It should be called when the directory returned by FetchAsDirectory is no longer in use.
	// In this way, FetchAsDirectory and CloseDirectory should be treated as a pair of operations,
	// and a process that calls FetchAsDirectory should make sure a corresponding CloseDirectory is eventually called.
	CloseDirectory(logger lager.Logger, cacheKey, directoryPath string) error

	// SaveState writes the current state of the cache metadata to a file so that it can be recovered
	// later. This should be called on process shutdown.
	SaveState(logger lager.Logger) error

	// RecoverState checks to see if a state file exists (from a previous SaveState call), and restores
	// the cache state from that information if such a file exists. This should be called on startup.
	RecoverState(logger lager.Logger) error
}

CachedDownloader is responsible for downloading and caching files and maintaining reference counts for each cache entry. Entries in the cache with no active references are ejected from the cache when new space is needed.

type CachedFile

type CachedFile struct {
	*os.File
	// contains filtered or unexported fields
}

func NewFileCloser

func NewFileCloser(file *os.File, onClose func(string)) *CachedFile

func (*CachedFile) Close

func (fw *CachedFile) Close() error

type CachingInfoType

type CachingInfoType struct {
	ETag         string
	LastModified string
}

func (CachingInfoType) Equal

func (c CachingInfoType) Equal(other CachingInfoType) bool

type ChecksumFailedError

type ChecksumFailedError struct {
	// contains filtered or unexported fields
}

func (*ChecksumFailedError) Error

func (e *ChecksumFailedError) Error() string

type ChecksumInfoType

type ChecksumInfoType struct {
	Algorithm string
	Value     string
}

type DownloadCancelledError

type DownloadCancelledError struct {
	// contains filtered or unexported fields
}

func (*DownloadCancelledError) Error

func (e *DownloadCancelledError) Error() string

type Downloader

type Downloader struct {
	// contains filtered or unexported fields
}

func NewDownloader

func NewDownloader(requestTimeout time.Duration, maxConcurrentDownloads int, tlsConfig *tls.Config) *Downloader

func NewDownloaderWithIdleTimeout

func NewDownloaderWithIdleTimeout(requestTimeout time.Duration, idleTimeout time.Duration, maxConcurrentDownloads int, tlsConfig *tls.Config) *Downloader

func (*Downloader) Download

func (downloader *Downloader) Download(
	logger lager.Logger,
	url *url.URL,
	createDestination func() (*os.File, error),
	cachingInfoIn CachingInfoType,
	checksum ChecksumInfoType,
	cancelChan <-chan struct{},
) (path string, cachingInfoOut CachingInfoType, err error)

type FileCache

type FileCache struct {
	CachedPath string

	Entries    map[string]*FileCacheEntry
	OldEntries map[string]*FileCacheEntry
	Seq        uint64
	// contains filtered or unexported fields
}

func NewCache

func NewCache(dir string, maxSizeInBytes int64) *FileCache

func (*FileCache) Add

func (c *FileCache) Add(logger lager.Logger, cacheKey, sourcePath string, size int64, cachingInfo CachingInfoType) (*CachedFile, error)

func (*FileCache) AddDirectory

func (c *FileCache) AddDirectory(logger lager.Logger, cacheKey, sourcePath string, size int64, cachingInfo CachingInfoType) (string, error)

func (*FileCache) CloseDirectory

func (c *FileCache) CloseDirectory(logger lager.Logger, cacheKey, dirPath string) error

func (*FileCache) Get

func (c *FileCache) Get(logger lager.Logger, cacheKey string) (*CachedFile, CachingInfoType, error)

func (*FileCache) GetDirectory

func (c *FileCache) GetDirectory(logger lager.Logger, cacheKey string) (string, CachingInfoType, error)

func (*FileCache) Remove

func (c *FileCache) Remove(logger lager.Logger, cacheKey string)

type FileCacheEntry

type FileCacheEntry struct {
	Size                  int64
	Access                time.Time
	CachingInfo           CachingInfoType
	FilePath              string
	ExpandedDirectoryPath string
	// contains filtered or unexported fields
}