filestore

package
v0.8.8-beta.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 1, 2024 License: Apache-2.0 Imports: 19 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// ijson meta keys
	IJsonNumCommands      = "ijson:numcmds"
	IJsonIncrementalBytes = "ijson:incbytes"
)
View Source 
const (
	IJsonHighCommands = 100
	IJsonHighRatio    = 3
	IJsonLowRatio     = 1
	IJsonLowCommands  = 10
)
View Source 
const DefaultFlushTime = 5 * time.Second
View Source 
const DefaultPartDataSize = 64 * 1024
View Source 
const FilestoreDBName = "filestore.db"
View Source 
const NoPartIdx = -1

Variables

This section is empty.

Functions

func GetDBName 

func GetDBName() string

func InitFilestore 

func InitFilestore() error

func MakeDB 

func MakeDB(ctx context.Context) (*sqlx.DB, error)

func WithTx 

func WithTx(ctx context.Context, fn func(tx *TxWrap) error) error

func WithTxRtn 

func WithTxRtn[RT any](ctx context.Context, fn func(tx *TxWrap) (RT, error)) (RT, error)

Types

type CacheEntry 

type CacheEntry struct {
	PinCount int // this is synchronzed with the FileStore lock (not the entry lock)

	Lock        *sync.Mutex
	ZoneId      string
	Name        string
	File        *WaveFile
	DataEntries map[int]*DataCacheEntry
	FlushErrors int
}

if File or DataEntries are not nil then they are dirty (need to be flushed to disk)

type DataCacheEntry 

type DataCacheEntry struct {
	PartIdx int
	Data    []byte // capacity is always ZoneDataPartSize
}

type FileData 

type FileData struct {
	ZoneId  string `json:"zoneid"`
	Name    string `json:"name"`
	PartIdx int    `json:"partidx"`
	Data    []byte `json:"data"`
}

func (FileData) UseDBMap 

func (FileData) UseDBMap()

type FileMeta 

type FileMeta = map[string]any

type FileOptsType 

type FileOptsType struct {
	MaxSize     int64 `json:"maxsize,omitempty"`
	Circular    bool  `json:"circular,omitempty"`
	IJson       bool  `json:"ijson,omitempty"`
	IJsonBudget int   `json:"ijsonbudget,omitempty"`
}

type FileStore 

type FileStore struct {
	Lock       *sync.Mutex
	Cache      map[cacheKey]*CacheEntry
	IsFlushing bool
}
var WFS *FileStore = &FileStore{
	Lock:  &sync.Mutex{},
	Cache: make(map[cacheKey]*CacheEntry),
}

func (*FileStore) AppendData 

func (s *FileStore) AppendData(ctx context.Context, zoneId string, name string, data []byte) error

func (*FileStore) AppendIJson 

func (s *FileStore) AppendIJson(ctx context.Context, zoneId string, name string, command map[string]any) error

func (*FileStore) CompactIJson 

func (s *FileStore) CompactIJson(ctx context.Context, zoneId string, name string) error

func (*FileStore) DeleteFile 

func (s *FileStore) DeleteFile(ctx context.Context, zoneId string, name string) error

func (*FileStore) DeleteZone 

func (s *FileStore) DeleteZone(ctx context.Context, zoneId string) error

func (*FileStore) FlushCache 

func (s *FileStore) FlushCache(ctx context.Context) (stats FlushStats, rtnErr error)

func (*FileStore) GetAllZoneIds 

func (s *FileStore) GetAllZoneIds(ctx context.Context) ([]string, error)

func (*FileStore) ListFiles 

func (s *FileStore) ListFiles(ctx context.Context, zoneId string) ([]*WaveFile, error)

func (*FileStore) MakeFile 

func (s *FileStore) MakeFile(ctx context.Context, zoneId string, name string, meta FileMeta, opts FileOptsType) error

synchronous (does not interact with the cache)

func (*FileStore) ReadAt 

func (s *FileStore) ReadAt(ctx context.Context, zoneId string, name string, offset int64, size int64) (rtnOffset int64, rtnData []byte, rtnErr error)

returns (offset, data, error) we return the offset because the offset may have been adjusted if the size was too big (for circular files)

func (*FileStore) ReadFile 

func (s *FileStore) ReadFile(ctx context.Context, zoneId string, name string) (rtnOffset int64, rtnData []byte, rtnErr error)

returns (offset, data, error)

func (*FileStore) Stat 

func (s *FileStore) Stat(ctx context.Context, zoneId string, name string) (*WaveFile, error)

if file doesn't exsit, returns fs.ErrNotExist

func (*FileStore) WriteAt 

func (s *FileStore) WriteAt(ctx context.Context, zoneId string, name string, offset int64, data []byte) error

func (*FileStore) WriteFile 

func (s *FileStore) WriteFile(ctx context.Context, zoneId string, name string, data []byte) error

func (*FileStore) WriteMeta 

func (s *FileStore) WriteMeta(ctx context.Context, zoneId string, name string, meta FileMeta, merge bool) error

type FlushStats 

type FlushStats struct {
	FlushDuration   time.Duration
	NumDirtyEntries int
	NumCommitted    int
}

type TxWrap 

type TxWrap = txwrap.TxWrap

type WaveFile 

type WaveFile struct {
	// these fields are static (not updated)
	ZoneId    string       `json:"zoneid"`
	Name      string       `json:"name"`
	Opts      FileOptsType `json:"opts"`
	CreatedTs int64        `json:"createdts"`

	//  these fields are mutable
	Size  int64    `json:"size"`
	ModTs int64    `json:"modts"`
	Meta  FileMeta `json:"meta"` // only top-level keys can be updated (lower levels are immutable)
}

func (WaveFile) DataLength 

func (f WaveFile) DataLength() int64

for regular files this is just Size for circular files this is min(Size, MaxSize)

func (WaveFile) DataStartIdx 

func (f WaveFile) DataStartIdx() int64

for regular files this is just 0 for circular files this is the index of the first byte of data we have

func (*WaveFile) DeepCopy 

func (f *WaveFile) DeepCopy() *WaveFile

func (WaveFile) UseDBMap 

func (WaveFile) UseDBMap()

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.