cache

package
v0.0.0-...-1b1593a Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 13, 2017 License: MIT Imports: 24 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// DefCacheChunkSize is the default value for chunk size
	DefCacheChunkSize = "5M"
	// DefCacheTotalChunkSize is the default value for the maximum size of stored chunks
	DefCacheTotalChunkSize = "10G"
	// DefCacheChunkCleanInterval is the interval at which chunks are cleaned
	DefCacheChunkCleanInterval = "1m"
	// DefCacheInfoAge is the default value for object info age
	DefCacheInfoAge = "6h"
	// DefCacheReadRetries is the default value for read retries
	DefCacheReadRetries = 10
	// DefCacheTotalWorkers is how many workers run in parallel to download chunks
	DefCacheTotalWorkers = 4
	// DefCacheChunkNoMemory will enable or disable in-memory storage for chunks
	DefCacheChunkNoMemory = false
	// DefCacheRps limits the number of requests per second to the source FS
	DefCacheRps = -1
	// DefCacheWrites will cache file data on writes through the cache
	DefCacheWrites = false
)
View Source 
const (
	RootBucket   = "root"
	RootTsBucket = "rootTs"
	DataTsBucket = "dataTs"
)

Constants

Variables

This section is empty.

Functions

func NewFs 

func NewFs(name, rpath string) (fs.Fs, error)

NewFs contstructs an Fs from the path, container:path

Types

type ChunkStorage 

type ChunkStorage interface {
	// will check if the chunk is in storage. should be fast and not read the chunk itself if possible
	HasChunk(cachedObject *Object, offset int64) bool

	// returns the chunk in storage. return an error if it's not
	GetChunk(cachedObject *Object, offset int64) ([]byte, error)

	// add a new chunk
	AddChunk(fp string, data []byte, offset int64) error

	// if the storage can cleanup on a cron basis
	// otherwise it can do a noop operation
	CleanChunksByAge(chunkAge time.Duration)

	// if the storage can cleanup chunks after we no longer need them
	// otherwise it can do a noop operation
	CleanChunksByNeed(offset int64)

	// if the storage can cleanup chunks after the total size passes a certain point
	// otherwise it can do a noop operation
	CleanChunksBySize(maxSize int64)
}

ChunkStorage is a storage type that supports only chunk operations (i.e in RAM)

type Directory 

type Directory struct {
	fs.Directory `json:"-"`

	CacheFs      *Fs    `json:"-"`       // cache fs
	Name         string `json:"name"`    // name of the directory
	Dir          string `json:"dir"`     // abs path of the directory
	CacheModTime int64  `json:"modTime"` // modification or creation time - IsZero for unknown
	CacheSize    int64  `json:"size"`    // size of directory and contents or -1 if unknown

	CacheItems int64  `json:"items"`     // number of objects or -1 for unknown
	CacheType  string `json:"cacheType"` // object type
}

Directory is a generic dir that stores basic information about it

func DirectoryFromOriginal 

func DirectoryFromOriginal(f *Fs, d fs.Directory) *Directory

DirectoryFromOriginal builds one from a generic fs.Directory

func NewDirectory 

func NewDirectory(f *Fs, remote string) *Directory

NewDirectory builds an empty dir which will be used to unmarshal data in it

func (*Directory) Fs 

func (d *Directory) Fs() fs.Info

Fs returns its FS info

func (*Directory) Items 

func (d *Directory) Items() int64

Items returns the cached Items

func (*Directory) ModTime 

func (d *Directory) ModTime() time.Time

ModTime returns the cached ModTime

func (*Directory) Remote 

func (d *Directory) Remote() string

Remote returns the remote path

func (*Directory) Size 

func (d *Directory) Size() int64

Size returns the cached Size

func (*Directory) String 

func (d *Directory) String() string

String returns a human friendly name for this object

type Features 

type Features struct {
	PurgeDb bool // purge the db before starting
}

Features flags for this storage type

type Fs 

type Fs struct {
	fs.Fs
	// contains filtered or unexported fields
}

Fs represents a wrapped fs.Fs

func (*Fs) ChunkSize 

func (f *Fs) ChunkSize() int64

ChunkSize returns the configured chunk size

func (*Fs) CleanUp 

func (f *Fs) CleanUp() error

CleanUp the trash in the Fs

func (*Fs) CleanUpCache 

func (f *Fs) CleanUpCache(ignoreLastTs bool)

CleanUpCache will cleanup only the cache data that is expired

func (*Fs) Copy 

func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error)

Copy src to this remote using server side copy operations.

func (*Fs) DirCacheFlush 

func (f *Fs) DirCacheFlush()

DirCacheFlush flushes the dir cache

func (*Fs) DirMove 

func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error

DirMove moves src, srcRemote to this remote at dstRemote using server side move operations.

func (*Fs) Features 

func (f *Fs) Features() *fs.Features

Features returns the optional features of this Fs

func (*Fs) Hashes 

func (f *Fs) Hashes() fs.HashSet

Hashes returns the supported hash sets.

func (*Fs) List 

func (f *Fs) List(dir string) (entries fs.DirEntries, err error)

List the objects and directories in dir into entries

func (*Fs) ListR 

func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error)

ListR lists the objects and directories of the Fs starting from dir recursively into out.

func (*Fs) Mkdir 

func (f *Fs) Mkdir(dir string) error

Mkdir makes the directory (container, bucket)

func (*Fs) Move 

func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error)

Move src to this remote using server side move operations.

func (*Fs) Name 

func (f *Fs) Name() string

Name of the remote (as passed into NewFs)

func (*Fs) NewObject 

func (f *Fs) NewObject(remote string) (fs.Object, error)

NewObject finds the Object at remote.

func (*Fs) OpenRateLimited 

func (f *Fs) OpenRateLimited(fn func() (io.ReadCloser, error)) (io.ReadCloser, error)

OpenRateLimited will execute a closure under a rate limiter watch

func (*Fs) Purge 

func (f *Fs) Purge() error

Purge all files in the root and the root directory

func (*Fs) Put 

func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)

Put in to the remote path with the modTime given of the given size

func (*Fs) PutStream 

func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)

PutStream uploads the object

func (*Fs) PutUnchecked 

func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)

PutUnchecked uploads the object

func (*Fs) Rmdir 

func (f *Fs) Rmdir(dir string) error

Rmdir removes the directory (container, bucket) if empty

func (*Fs) Root 

func (f *Fs) Root() string

Root of the remote (as passed into NewFs)

func (*Fs) SetWrapper 

func (f *Fs) SetWrapper(wrapper fs.Fs)

SetWrapper sets the Fs that is wrapping this Fs

func (*Fs) Stats 

func (f *Fs) Stats() (map[string]map[string]interface{}, error)

Stats returns stats about the cache storage

func (*Fs) String 

func (f *Fs) String() string

String returns a description of the FS

func (*Fs) UnWrap 

func (f *Fs) UnWrap() fs.Fs

UnWrap returns the Fs that this Fs is wrapping

func (*Fs) WrapFs 

func (f *Fs) WrapFs() fs.Fs

WrapFs returns the Fs that is wrapping this Fs

type Handle 

type Handle struct {
	UseMemory bool
	// contains filtered or unexported fields
}

Handle is managing the read/write/seek operations on an open handle

func NewObjectHandle 

func NewObjectHandle(o *Object) *Handle

NewObjectHandle returns a new Handle for an existing Object

func (*Handle) Close 

func (r *Handle) Close() error

Close will tell the workers to stop

func (*Handle) Read 

func (r *Handle) Read(p []byte) (n int, err error)

Read a chunk from storage or len(p)

func (*Handle) Seek 

func (r *Handle) Seek(offset int64, whence int) (int64, error)

Seek will move the current offset based on whence and instruct the workers to move there too

func (*Handle) String 

func (r *Handle) String() string

String representation of this reader

type Memory 

type Memory struct {
	ChunkStorage
	// contains filtered or unexported fields
}

Memory is a wrapper of transient storage for a go-cache store

func NewMemory 

func NewMemory(defaultExpiration time.Duration) *Memory

NewMemory builds this cache storage defaultExpiration will set the expiry time of chunks in this storage

func (*Memory) AddChunk 

func (m *Memory) AddChunk(fp string, data []byte, offset int64) error

AddChunk adds a new chunk of a cached object

func (*Memory) AddChunkAhead 

func (m *Memory) AddChunkAhead(fp string, data []byte, offset int64, t time.Duration) error

AddChunkAhead adds a new chunk of a cached object

func (*Memory) CleanChunksByAge 

func (m *Memory) CleanChunksByAge(chunkAge time.Duration)

CleanChunksByAge will cleanup on a cron basis

func (*Memory) CleanChunksByNeed 

func (m *Memory) CleanChunksByNeed(offset int64)

CleanChunksByNeed will cleanup chunks after the FS passes a specific chunk

func (*Memory) CleanChunksBySize 

func (m *Memory) CleanChunksBySize(maxSize int64)

CleanChunksBySize will cleanup chunks after the total size passes a certain point

func (*Memory) Connect 

func (m *Memory) Connect(defaultExpiration time.Duration) error

Connect will create a connection for the storage

func (*Memory) GetChunk 

func (m *Memory) GetChunk(cachedObject *Object, offset int64) ([]byte, error)

GetChunk will retrieve a single chunk which belongs to a cached object or an error if it doesn't find it

func (*Memory) HasChunk 

func (m *Memory) HasChunk(cachedObject *Object, offset int64) bool

HasChunk confirms the existence of a single chunk of an object

type Object 

type Object struct {
	fs.Object `json:"-"`

	CacheFs       *Fs    `json:"-"`        // cache fs
	Name          string `json:"name"`     // name of the directory
	Dir           string `json:"dir"`      // abs path of the object
	CacheModTime  int64  `json:"modTime"`  // modification or creation time - IsZero for unknown
	CacheSize     int64  `json:"size"`     // size of directory and contents or -1 if unknown
	CacheStorable bool   `json:"storable"` // says whether this object can be stored
	CacheType     string `json:"cacheType"`
	// contains filtered or unexported fields
}

Object is a generic file like object that stores basic information about it

func NewObject 

func NewObject(f *Fs, remote string) *Object

NewObject builds one from a generic fs.Object

func ObjectFromOriginal 

func ObjectFromOriginal(f *Fs, o fs.Object) *Object

ObjectFromOriginal builds one from a generic fs.Object

func (*Object) Fs 

func (o *Object) Fs() fs.Info

Fs returns its FS info

func (*Object) Hash 

func (o *Object) Hash(ht fs.HashType) (string, error)

Hash requests a hash of the object and stores in the cache since it might or might not be called, this is lazy loaded

func (*Object) MarshalJSON 

func (o *Object) MarshalJSON() ([]byte, error)

MarshalJSON is needed to override the hashes map (needed to support older versions of Go)

func (*Object) ModTime 

func (o *Object) ModTime() time.Time

ModTime returns the cached ModTime

func (*Object) Open 

func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error)

Open is used to request a specific part of the file using fs.RangeOption

func (*Object) Remote 

func (o *Object) Remote() string

Remote returns the remote path

func (*Object) Remove 

func (o *Object) Remove() error

Remove deletes the object from both the cache and the source

func (*Object) SetModTime 

func (o *Object) SetModTime(t time.Time) error

SetModTime sets the ModTime of this object

func (*Object) Size 

func (o *Object) Size() int64

Size returns the cached Size

func (*Object) Storable 

func (o *Object) Storable() bool

Storable returns the cached Storable

func (*Object) String 

func (o *Object) String() string

String returns a human friendly name for this object

func (*Object) UnmarshalJSON 

func (o *Object) UnmarshalJSON(b []byte) error

UnmarshalJSON is needed to override the CacheHashes map (needed to support older versions of Go)

func (*Object) Update 

func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error

Update will change the object data

type Persistent 

type Persistent struct {
	Storage
	// contains filtered or unexported fields
}

Persistent is a wrapper of persistent storage for a bolt.DB file

func GetPersistent 

func GetPersistent(dbPath string, f *Features) (*Persistent, error)

GetPersistent returns a single instance for the specific store

func (*Persistent) AddChunk 

func (b *Persistent) AddChunk(fp string, data []byte, offset int64) error

AddChunk adds a new chunk of a cached object

func (*Persistent) AddDir 

func (b *Persistent) AddDir(cachedDir *Directory) error

AddDir will update a CachedDirectory metadata and all its entries

func (*Persistent) AddObject 

func (b *Persistent) AddObject(cachedObject *Object) error

AddObject will create a cached object in its parent directory

func (*Persistent) CleanChunksByAge 

func (b *Persistent) CleanChunksByAge(chunkAge time.Duration)

CleanChunksByAge will cleanup on a cron basis

func (*Persistent) CleanChunksByNeed 

func (b *Persistent) CleanChunksByNeed(offset int64)

CleanChunksByNeed is a noop for this implementation

func (*Persistent) CleanChunksBySize 

func (b *Persistent) CleanChunksBySize(maxSize int64)

CleanChunksBySize will cleanup chunks after the total size passes a certain point

func (*Persistent) CleanEntriesByAge 

func (b *Persistent) CleanEntriesByAge(entryAge time.Duration)

CleanEntriesByAge will cleanup on a cron basis

func (*Persistent) Close 

func (b *Persistent) Close()

Close should be called when the program ends gracefully

func (*Persistent) Connect 

func (b *Persistent) Connect() error

Connect creates a connection to the configured file refreshDb will delete the file before to create an empty DB if it's set to true

func (*Persistent) ExpireDir 

func (b *Persistent) ExpireDir(fp string) error

ExpireDir will flush a CachedDirectory and all its objects from the objects chunks will remain as they are

func (*Persistent) GetChunk 

func (b *Persistent) GetChunk(cachedObject *Object, offset int64) ([]byte, error)

GetChunk will retrieve a single chunk which belongs to a cached object or an error if it doesn't find it

func (*Persistent) GetChunkTs 

func (b *Persistent) GetChunkTs(path string, offset int64) (time.Time, error)

GetChunkTs retrieves the current timestamp of this chunk

func (*Persistent) GetDirEntries 

func (b *Persistent) GetDirEntries(cachedDir *Directory) (fs.DirEntries, error)

GetDirEntries will return a CachedDirectory, its list of dir entries and/or an error if it encountered issues

func (*Persistent) GetObject 

func (b *Persistent) GetObject(cachedObject *Object) (err error)

GetObject will return a CachedObject from its parent directory or an error if it doesn't find it

func (*Persistent) GetRootTs 

func (b *Persistent) GetRootTs(path string) (time.Time, error)

GetRootTs retrieves the current timestamp of an object or dir

func (*Persistent) HasChunk 

func (b *Persistent) HasChunk(cachedObject *Object, offset int64) bool

HasChunk confirms the existence of a single chunk of an object

func (*Persistent) HasEntry 

func (b *Persistent) HasEntry(remote string) bool

HasEntry confirms the existence of a single entry (dir or object)

func (*Persistent) Purge 

func (b *Persistent) Purge()

Purge will flush the entire cache

func (*Persistent) RemoveDir 

func (b *Persistent) RemoveDir(fp string) error

RemoveDir will delete a CachedDirectory, all its objects and all the chunks stored for it

func (*Persistent) RemoveObject 

func (b *Persistent) RemoveObject(fp string) error

RemoveObject will delete a single cached object and all the chunks which belong to it

func (*Persistent) Stats 

func (b *Persistent) Stats() (map[string]map[string]interface{}, error)

Stats returns a go map with the stats key values

func (*Persistent) String 

func (b *Persistent) String() string

String will return a human friendly string for this DB (currently the dbPath)

type Storage 

type Storage interface {
	ChunkStorage

	// will update/create a directory or an error if it's not found
	AddDir(cachedDir *Directory) error

	// will return a directory with all the entries in it or an error if it's not found
	GetDirEntries(cachedDir *Directory) (fs.DirEntries, error)

	// remove a directory and all the objects and chunks in it
	RemoveDir(fp string) error

	// remove a directory and all the objects and chunks in it
	ExpireDir(fp string) error

	// will return an object (file) or error if it doesn't find it
	GetObject(cachedObject *Object) (err error)

	// add a new object to its parent directory
	// the directory structure (all the parents of this object) is created if its not found
	AddObject(cachedObject *Object) error

	// remove an object and all its chunks
	RemoveObject(fp string) error

	// Stats returns stats about the cache storage
	Stats() (map[string]map[string]interface{}, error)

	// if the storage can cleanup on a cron basis
	// otherwise it can do a noop operation
	CleanEntriesByAge(entryAge time.Duration)

	// Purge will flush the entire cache
	Purge()

	// Close should be called when the program ends gracefully
	Close()
}

Storage is a storage type (Bolt) which needs to support both chunk and file based operations

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.