lazycache

README

go-lazycache

Todo

[ ] Add integration test for extraction of .png, .jpg and no extension...

Benchmarking using curl

repeat 10 { curl -s -o /dev/null -w "%{time_total}\n" -H "Range: bytes=2615776240-2616368015" https://rawdata.oceanobservatories.org/files//RS03ASHS/PN03B/06-CAMHDA301/2017/09/21/CAMHDA301-20170921T211500.mov }

Documentation

Index

• Constants
• Variables
• func AddMirror(serverAddr string)
• func ConfigureFromViper()
• func ConfigureImageStoreFromViper()
• func InfoHandler(w http.ResponseWriter, req *http.Request)
• func MakeRootNode(fs FileSystem, root string)
• func MungeHostname(hostname string) []string
• func RecursiveMkdir(dir string)
• func RegisterDefaultHandlers()
• func RootHandler(w http.ResponseWriter, req *http.Request)
• func ViperConfiguration()
• type DirListing
• type DirMapStore
• type FileOverlayFS
  • func OpenFileOverlayFS(fs FileSystem, path string) (*FileOverlayFS, error)
  • func (fs *FileOverlayFS) FileSource(p string) (lazyfs.FileSource, error)
  • func (fs *FileOverlayFS) OriginalPath(p string) string
  • func (fs *FileOverlayFS) PathType(path string) int
  • func (fs *FileOverlayFS) ReadDir(p string) (*DirListing, error)
• type FileSystem
• type GoogleImageStore
  • func CreateGoogleStore(bucket string) *GoogleImageStore
  • func (store GoogleImageStore) Has(key string) bool
  • func (store GoogleImageStore) Retrieve(key string) (io.Reader, error)
  • func (store GoogleImageStore) Statistics() interface{}
  • func (store GoogleImageStore) Store(key string, data io.Reader)
  • func (store *GoogleImageStore) Url(key string) (string, bool)
• type HttpFS
  • func OpenHttpFS(uri url.URL) (*HttpFS, error)
  • func (fs *HttpFS) FileSource(p string) (lazyfs.FileSource, error)
  • func (fs *HttpFS) OriginalPath(p string) string
  • func (fs *HttpFS) OriginalUri(p string) url.URL
  • func (fs *HttpFS) PathType(path string) int
  • func (fs *HttpFS) ReadDir(p string) (*DirListing, error)
• type ImageStore
• type LocalFS
  • func OpenLocalFS(path string) (*LocalFS, error)
  • func (fs *LocalFS) LazyFile(p string) (lazyfs.FileSource, error)
  • func (fs *LocalFS) OriginalPath(p string) string
  • func (fs *LocalFS) PathType(path string) int
  • func (fs *LocalFS) ReadDir(p string) (*DirListing, error)
• type LocalImageStore
  • func CreateLocalStore(localRoot string, addr string) *LocalImageStore
  • func (store *LocalImageStore) Has(key string) bool
  • func (store LocalImageStore) Retrieve(key string) (io.Reader, error)
  • func (store LocalImageStore) ServeHTTP(w http.ResponseWriter, r *http.Request)
  • func (store LocalImageStore) Statistics() interface{}
  • func (store *LocalImageStore) Store(key string, data io.Reader)
  • func (store *LocalImageStore) Url(key string) (string, bool)
• type MoovHandlerTiming
• type Node
  • func CacheHandler(node *Node, path []string, w http.ResponseWriter, req *http.Request) *Node
  • func HandleDirectory(node *Node, path []string, w http.ResponseWriter, req *http.Request) *Node
  • func MoovHandler(node *Node, path []string, w http.ResponseWriter, req *http.Request) *Node
  • func RedirectHandler(node *Node, path []string, w http.ResponseWriter, req *http.Request) *Node
  • func (node *Node) BootstrapDirectory(listing DirListing)
  • func (parent *Node) MakeNode(path string) *Node
• type NullImageStore
  • func (store NullImageStore) Has(key string) bool
  • func (store NullImageStore) Retrieve(key string) (io.Reader, error)
  • func (store NullImageStore) Store(key string, data io.Reader)
  • func (store NullImageStore) Url(key string) (string, bool)
• type QTEntry
• type QTStore
• type RootNode
  • func (root *RootNode) Handle(node *Node, path []string, w http.ResponseWriter, req *http.Request)
  • func (root RootNode) ServeHTTP(w http.ResponseWriter, req *http.Request)

Constants

const (
	Directory = iota
	File      = iota
)

const ApiVersion = "v1"

const CachedExpiration = 5 * time.Minute

a Duration, measured in nanoseconds

const Version = "v0.1.0"

Variables

var (
	PromCacheRequests = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "cache_requests_total",
			Help: "Number of cache requests.",
		},
		[]string{"store"},
	)

	PromCacheMisses = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "cache_misses_total",
			Help: "Number of cache misses.",
		},
		[]string{"store"},
	)

	PromCacheSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "cache_size",
		Help: "Number of elements in cache",
	},
		[]string{"store"},
	)
)

var Logger kitlog.Logger

var RootMap map[string]*RootNode

Functions

func AddMirror

func AddMirror(serverAddr string)

func ConfigureFromViper

func ConfigureFromViper()

func ConfigureImageStoreFromViper

func ConfigureImageStoreFromViper()

func InfoHandler

func InfoHandler(w http.ResponseWriter, req *http.Request)

func MakeRootNode

func MakeRootNode(fs FileSystem, root string)

func MungeHostname

func MungeHostname(hostname string) []string

func RecursiveMkdir

func RecursiveMkdir(dir string)

func RegisterDefaultHandlers

func RegisterDefaultHandlers()

func RootHandler

func RootHandler(w http.ResponseWriter, req *http.Request)

RootHandler is the default HTTP handler, registered at "/" It returns a JSON structure giving the relative path to each of the registered mirrors.

e.g.

{
  "https://rawdata.oceanobservatories.org/files/": {
    "APIPath": {
      "V1": "/v1/org/oceanobservatories/rawdata/files/"
    }
  }
}

func ViperConfiguration

func ViperConfiguration()

Types

type DirListing

type DirListing struct {
	Path        string
	Files       []string
	Directories []string
	// contains filtered or unexported fields
}

type DirMapStore

type DirMapStore struct {
	Cache map[string](*DirListing)
	Mutex sync.Mutex
}

var DirCache DirMapStore

type FileOverlayFS

type FileOverlayFS struct {
	Flatten bool
	// contains filtered or unexported fields
}

func OpenFileOverlayFS

func OpenFileOverlayFS(fs FileSystem, path string) (*FileOverlayFS, error)

func (*FileOverlayFS) FileSource

func (fs *FileOverlayFS) FileSource(p string) (lazyfs.FileSource, error)

func (*FileOverlayFS) OriginalPath

func (fs *FileOverlayFS) OriginalPath(p string) string

func (*FileOverlayFS) PathType

func (fs *FileOverlayFS) PathType(path string) int

func (*FileOverlayFS) ReadDir

func (fs *FileOverlayFS) ReadDir(p string) (*DirListing, error)

type FileSystem

type FileSystem interface {
	PathType(path string) int
	ReadDir(p string) (*DirListing, error)
	OriginalPath(p string) string
	FileSource(p string) (lazyfs.FileSource, error)
}

type GoogleImageStore

type GoogleImageStore struct {
	Stats struct {
		// contains filtered or unexported fields
	}
	// contains filtered or unexported fields
}

func CreateGoogleStore

func CreateGoogleStore(bucket string) *GoogleImageStore

func (GoogleImageStore) Has

func (store GoogleImageStore) Has(key string) bool

func (GoogleImageStore) Retrieve

func (store GoogleImageStore) Retrieve(key string) (io.Reader, error)

func (GoogleImageStore) Statistics

func (store GoogleImageStore) Statistics() interface{}

func (GoogleImageStore) Store

func (store GoogleImageStore) Store(key string, data io.Reader)

func (*GoogleImageStore) Url

func (store *GoogleImageStore) Url(key string) (string, bool)

type HttpFS

type HttpFS struct {
	Uri url.URL
	// contains filtered or unexported fields
}

func OpenHttpFS

func OpenHttpFS(uri url.URL) (*HttpFS, error)

func (*HttpFS) FileSource

func (fs *HttpFS) FileSource(p string) (lazyfs.FileSource, error)

func (*HttpFS) OriginalPath

func (fs *HttpFS) OriginalPath(p string) string

func (*HttpFS) OriginalUri

func (fs *HttpFS) OriginalUri(p string) url.URL

func (*HttpFS) PathType

func (fs *HttpFS) PathType(path string) int

func (*HttpFS) ReadDir

func (fs *HttpFS) ReadDir(p string) (*DirListing, error)

type ImageStore

type ImageStore interface {
	Has(key string) bool
	Url(key string) (string, bool)
	Store(key string, data io.Reader)
	Retrieve(key string) (io.Reader, error)
}

var ImageCache ImageStore = NullImageStore{}

type LocalFS

type LocalFS struct {
	Path string
}

func OpenLocalFS

func OpenLocalFS(path string) (*LocalFS, error)

func (*LocalFS) LazyFile

func (fs *LocalFS) LazyFile(p string) (lazyfs.FileSource, error)

func (*LocalFS) OriginalPath

func (fs *LocalFS) OriginalPath(p string) string

func (*LocalFS) PathType

func (fs *LocalFS) PathType(path string) int

func (*LocalFS) ReadDir

func (fs *LocalFS) ReadDir(p string) (*DirListing, error)

type LocalImageStore

type LocalImageStore struct {
	LocalRoot string
	UrlRoot   string

	Stats struct {
		// contains filtered or unexported fields
	}
	// contains filtered or unexported fields
}

func CreateLocalStore

func CreateLocalStore(localRoot string, addr string) *LocalImageStore

func (*LocalImageStore) Has

func (store *LocalImageStore) Has(key string) bool

func (LocalImageStore) Retrieve

func (store LocalImageStore) Retrieve(key string) (io.Reader, error)

func (LocalImageStore) ServeHTTP

func (store LocalImageStore) ServeHTTP(w http.ResponseWriter, r *http.Request)

func (LocalImageStore) Statistics

func (store LocalImageStore) Statistics() interface{}

func (*LocalImageStore) Store

func (store *LocalImageStore) Store(key string, data io.Reader)

func (*LocalImageStore) Url

func (store *LocalImageStore) Url(key string) (string, bool)

type MoovHandlerTiming

type MoovHandlerTiming struct {
	Handler, Metadata, Extraction, Encode time.Duration
	// contains filtered or unexported fields
}

MoovHandlerTiming

type Node

type Node struct {
	Path     string
	Children map[string]*Node

	Fs FileSystem
	// contains filtered or unexported fields
}

func CacheHandler

func CacheHandler(node *Node, path []string, w http.ResponseWriter, req *http.Request) *Node

Doesn't actually cache yet, just redirects

func HandleDirectory

func HandleDirectory(node *Node, path []string, w http.ResponseWriter, req *http.Request) *Node

func MoovHandler

func MoovHandler(node *Node, path []string, w http.ResponseWriter, req *http.Request) *Node

func RedirectHandler

func RedirectHandler(node *Node, path []string, w http.ResponseWriter, req *http.Request) *Node

func (*Node) BootstrapDirectory

func (node *Node) BootstrapDirectory(listing DirListing)

func (*Node) MakeNode

func (parent *Node) MakeNode(path string) *Node

type NullImageStore

type NullImageStore struct {
}

func (NullImageStore) Has

func (store NullImageStore) Has(key string) bool

func (NullImageStore) Retrieve

func (store NullImageStore) Retrieve(key string) (io.Reader, error)

func (NullImageStore) Store

func (store NullImageStore) Store(key string, data io.Reader)

func (NullImageStore) Url

func (store NullImageStore) Url(key string) (string, bool)

type QTEntry

type QTEntry struct {
	// contains filtered or unexported fields
}

type QTStore

type QTStore struct {
	Cache map[string](*QTEntry)
	Mutex sync.Mutex

	Stats struct {
		Requests, Misses int64
	}
}

type RootNode

type RootNode struct {
	// contains filtered or unexported fields
}

func (*RootNode) Handle

func (root *RootNode) Handle(node *Node, path []string, w http.ResponseWriter, req *http.Request)

func (RootNode) ServeHTTP

func (root RootNode) ServeHTTP(w http.ResponseWriter, req *http.Request)