Documentation
¶
Index ¶
- Constants
- Variables
- func BucketCapacity(blockSize int) int
- func BucketIndex(h uint64, buckets int, modulus uint64) int
- func BucketSize(capacity int) int
- func CreateDataFile(path string, appnum, uid uint64) error
- func CreateKeyFile(path string, uid uint64, appnum uint64, salt uint64, blockSize int, ...) error
- func CreateStore(datPath, keyPath, logPath string, appnum, uid, salt uint64, blockSize int, ...) error
- func DecodeUint16(b []byte) uint16
- func DecodeUint24(b []byte) uint32
- func DecodeUint32(b []byte) uint32
- func DecodeUint48(b []byte) uint64
- func DecodeUint64(b []byte) uint64
- func EncodeUint16(b []byte, v uint16)
- func EncodeUint24(b []byte, v uint32)
- func EncodeUint32(b []byte, v uint32)
- func EncodeUint48(b []byte, v uint64)
- func EncodeUint64(b []byte, v uint64)
- func Fadvise(fd int, offset int64, length int64, advice int) error
- func NewSalt() uint64
- func NewUID() uint64
- func Version() string
- type Bucket
- func (b *Bucket) ActualSize() int
- func (b *Bucket) BlockSize() int
- func (b *Bucket) Capacity() int
- func (b *Bucket) CopyInto(b2 *Bucket)
- func (b *Bucket) Count() int
- func (b *Bucket) Entry(idx int) Entry
- func (b *Bucket) Has(h uint64) bool
- func (b *Bucket) HighestHash() uint64
- func (b *Bucket) IsEmpty() bool
- func (b *Bucket) IsFull() bool
- func (b *Bucket) LoadFrom(spill int64, s Spiller) error
- func (b *Bucket) Lock()
- func (b *Bucket) LowestHash() uint64
- func (b *Bucket) SetSpill(v int64)
- func (b *Bucket) Spill() int64
- func (b *Bucket) Unlock()
- func (b *Bucket) WriteTo(w io.Writer) (int64, error)
- type BucketCache
- func (c *BucketCache) BucketCount() int
- func (c *BucketCache) EntryCount() int
- func (c *BucketCache) Exists(hash uint64, key string, df *DataFile) (bool, error)
- func (c *BucketCache) Fetch(hash uint64, key string, df *DataFile) (io.Reader, error)
- func (c *BucketCache) FetchHeader(hash uint64, key string, df *DataFile) (*DataRecordHeader, error)
- func (c *BucketCache) Get(idx int) *Bucket
- func (c *BucketCache) Has(hash uint64, sp Spiller) (bool, error)
- func (c *BucketCache) Insert(offset int64, size int64, hash uint64, df Spiller) error
- func (c *BucketCache) WriteDirty(lf *LogFile, kf *KeyFile) (int64, error)
- type BucketRecord
- type BucketScanner
- type Cache
- type CacheData
- type CountWriter
- type DatFileHeader
- type DataFile
- func (d *DataFile) AppendBucketSpill(blob []byte) (int64, error)
- func (d *DataFile) AppendRecord(dr *DataRecord) (int64, error)
- func (d *DataFile) Close() error
- func (d *DataFile) Flush() error
- func (d *DataFile) LoadBucketSpill(offset int64, blob []byte) error
- func (d *DataFile) LoadRecordHeader(offset int64) (*DataRecordHeader, error)
- func (d *DataFile) Offset() int64
- func (d *DataFile) RecordDataReader(offset int64, key string) (io.Reader, error)
- func (d *DataFile) RecordScanner() *RecordScanner
- func (d *DataFile) Size() (int64, error)
- func (d *DataFile) Sync() error
- type DataRecord
- type DataRecordHeader
- type Entry
- type Hasher
- type KeyFile
- func (k *KeyFile) BlockSize() uint16
- func (k *KeyFile) BucketScanner(df *DataFile) *BucketScanner
- func (k *KeyFile) Close() error
- func (k *KeyFile) Hash(key []byte) uint64
- func (k *KeyFile) HashString(key string) uint64
- func (k *KeyFile) LoadBucket(idx int) (*Bucket, error)
- func (k *KeyFile) PutBucket(idx int, b *Bucket) error
- func (k *KeyFile) Size() (int64, error)
- func (k *KeyFile) Sync() error
- type KeyFileHeader
- type LogFile
- type LogFileHeader
- type Pool
- func (p *Pool) Clear()
- func (p *Pool) Count() int
- func (p *Pool) DataSize() int
- func (p *Pool) Find(key string) ([]byte, bool)
- func (p *Pool) Has(key string) bool
- func (p *Pool) Insert(hash uint64, key string, value []byte)
- func (p *Pool) IsEmpty() bool
- func (p *Pool) WithRecords(fn func([]DataRecord) error) error
- func (p *Pool) WriteRecords(df *DataFile) (int64, error)
- type RecordScanner
- func (s *RecordScanner) Close() error
- func (s *RecordScanner) Err() error
- func (s *RecordScanner) IsData() bool
- func (s *RecordScanner) IsSpill() bool
- func (s *RecordScanner) Key() string
- func (s *RecordScanner) Next() bool
- func (s *RecordScanner) Reader() io.Reader
- func (s *RecordScanner) RecordSize() int64
- func (s *RecordScanner) Size() int64
- type SectionWriter
- type Spiller
- type Store
- func (s *Store) Close() error
- func (s *Store) DataFile() *DataFile
- func (s *Store) DataSize(key string) (int64, error)
- func (s *Store) Err() error
- func (s *Store) Exists(key string) (bool, error)
- func (s *Store) FetchReader(key string) (io.Reader, error)
- func (s *Store) Flush()
- func (s *Store) Insert(key string, data []byte) error
- func (s *Store) KeyFile() *KeyFile
- func (s *Store) LogFile() *LogFile
- func (s *Store) Rate() float64
- func (s *Store) RecordCount() int
- type VerifyResult
- type WriterFlusher
Constants ¶
const ( // Bucket header BucketHeaderSize = SizeUint16 + SizeUint48 // Spill // Bucket item BucketEntrySize = SizeUint48 + SizeUint48 + SizeUint64 // Hash )
const ( MaxBlockSize = MaxUint16 // maximum length of a keyfile block in bytes (must not be larger than MaxKeySize due to on-disk representation) MaxKeySize = MaxUint16 // maximum length of a data record's key in bytes MaxDataSize = MaxUint48 // maximum length of a data record's value in bytes )
const ( MaxUint16 = math.MaxUint16 MaxUint24 = 0xffffff MaxUint32 = math.MaxUint32 MaxUint48 = 0x0000ffffffffffff MaxUint64 = math.MaxUint64 MaxInt16 = math.MaxInt16 )
const ( SizeUint16 = 2 SizeUint24 = 3 SizeUint32 = 4 SizeUint48 = 6 SizeUint64 = 8 )
const ( DatFileHeaderSize = SizeUint64 + SizeUint16 + SizeUint64 + SizeUint64 + SizeUint16 + 64 // (Reserved) KeyFileHeaderSize = 8 + SizeUint16 + SizeUint64 + SizeUint64 + SizeUint16 + SizeUint64 + SizeUint64 + SizeUint16 + SizeUint16 + 56 // (Reserved) LogFileHeaderSize = 8 + SizeUint16 + SizeUint64 + SizeUint64 + SizeUint16 + SizeUint64 + SizeUint64 + SizeUint16 + SizeUint64 + SizeUint64 // DataFileSize SpillHeaderSize = SizeUint48 + SizeUint16 // size )
const ( FADV_NORMAL = 0x0 FADV_RANDOM = 0x1 FADV_SEQUENTIAL = 0x2 FADV_WILLNEED = 0x3 )
Variables ¶
var ( ErrAppNumMismatch = errors.New("appnum mismatch") ErrDataMissing = errors.New("data missing") ErrDataTooLarge = errors.New("data too large") ErrDifferentVersion = errors.New("different version") ErrHashMismatch = errors.New("hash mismatch") ErrInvalidBlockSize = errors.New("invalid block size") ErrInvalidBucketCount = errors.New("invalid bucket count") ErrInvalidCapacity = errors.New("invalid capacity") ErrInvalidDataRecord = errors.New("not a data record: contains spill marker") ErrInvalidKeySize = errors.New("invalid key size") ErrInvalidLoadFactor = errors.New("invalid load factor") ErrInvalidRecordSize = errors.New("invalid record size") ErrInvalidSpill = errors.New("not a spill record: missing spill marker") ErrKeyExists = errors.New("key exists") ErrKeyMismatch = errors.New("key mismatch") ErrKeyMissing = errors.New("key missing") ErrKeyNotFound = errors.New("key not found") ErrKeySizeMismatch = errors.New("key size mismatch") ErrKeyTooLarge = errors.New("key too large") ErrKeyWrongSize = errors.New("key wrong size") // deprecated: use ErrKeyMissing and ErrKeyTooLarge instead ErrNotDataFile = errors.New("not a data file") ErrNotKeyFile = errors.New("not a key file") ErrNotLogFile = errors.New("not a log file") ErrShortKeyFile = errors.New("short key file") ErrUIDMismatch = errors.New("uid mismatch") )
var ( DatFileHeaderType = []byte("gonudbdt") KeyFileHeaderType = []byte("gonudbky") LogFileHeaderType = []byte("gonudblg") )
var GitVersion string = "unknown"
Functions ¶
func BucketCapacity ¶
BucketCapacity returns the number of entries that fit in a bucket
func BucketSize ¶
BucketSize returns the actual size of a bucket. This can be smaller than the block size.
func CreateDataFile ¶
func CreateKeyFile ¶
func CreateStore ¶
func DecodeUint16 ¶
func DecodeUint24 ¶
func DecodeUint32 ¶
func DecodeUint48 ¶
func DecodeUint64 ¶
func EncodeUint16 ¶
func EncodeUint24 ¶
func EncodeUint32 ¶
func EncodeUint48 ¶
func EncodeUint64 ¶
func NewSalt ¶
func NewSalt() uint64
NewSalt returns a random salt or panics if the system source of entropy cannot be read
Types ¶
type Bucket ¶
type Bucket struct {
// contains filtered or unexported fields
}
TODO: evaluate tradeoffs of using a slice of Entry instead of blob
func (*Bucket) ActualSize ¶
ActualSize returns the serialized bucket size, excluding empty space
func (*Bucket) HighestHash ¶
func (*Bucket) LoadFrom ¶
LoadFrom reads data containing entries from r, padding the rest of the bucket with zero bytes.
func (*Bucket) LowestHash ¶
type BucketCache ¶
type BucketCache struct {
// contains filtered or unexported fields
}
func (*BucketCache) BucketCount ¶
func (c *BucketCache) BucketCount() int
BucketCount returns the number of buckets in the cache
func (*BucketCache) EntryCount ¶
func (c *BucketCache) EntryCount() int
EntryCount returns the number of entries in the cache
func (*BucketCache) Exists ¶
Exists reports whether a record with the given hash and key exists in the data file
func (*BucketCache) Fetch ¶
Fetch returns a reader that can be used to read the data record associated with the key
func (*BucketCache) FetchHeader ¶ added in v0.2.1
func (c *BucketCache) FetchHeader(hash uint64, key string, df *DataFile) (*DataRecordHeader, error)
FetchHeader returns a record header for the data record associated with the key
func (*BucketCache) Get ¶
func (c *BucketCache) Get(idx int) *Bucket
Get retrieves a copy of the bucket at index idx
func (*BucketCache) WriteDirty ¶
func (c *BucketCache) WriteDirty(lf *LogFile, kf *KeyFile) (int64, error)
type BucketRecord ¶
type BucketRecord struct {
// contains filtered or unexported fields
}
type BucketScanner ¶
type BucketScanner struct {
// contains filtered or unexported fields
}
BucketScanner implements a sequential scan through a key file. Successive calls to the Next method will step through the buckets in the file, including spilled buckets in the data file.
func (*BucketScanner) Bucket ¶
func (b *BucketScanner) Bucket() *Bucket
Bucket returns the current bucket. Should not be called until Next has been called. The bucket is backed by data that may be overwritten with a call to Next so should not be retained.
func (*BucketScanner) Close ¶
func (b *BucketScanner) Close() error
func (*BucketScanner) Err ¶
func (b *BucketScanner) Err() error
Err returns the first non-EOF error that was encountered by the BucketScanner.
func (*BucketScanner) Index ¶
func (b *BucketScanner) Index() int
Index returns the index of the current bucket. Should not be called until Next has been called. Spill buckets share an index with their parent.
func (*BucketScanner) IsSpill ¶
func (b *BucketScanner) IsSpill() bool
IsSpill reports whether the current bucket was read from a data store spill.
func (*BucketScanner) Next ¶
func (b *BucketScanner) Next() bool
Next reads the next bucket in sequence, including spills to the data store. It returns false if it encounters an error or there are no more buckets to read.
type Cache ¶
type Cache struct {
// contains filtered or unexported fields
}
Cache is an in memory buffer of buckets. It is not safe for concurrent use.
func (*Cache) WithBuckets ¶
func (c *Cache) WithBuckets(fn func(bs []BucketRecord) error) error
type CacheData ¶
type CacheData struct {
// contains filtered or unexported fields
}
CacheData is a read only view of a bucket cache. It is safe for concurrent use.
func (*CacheData) WithBuckets ¶
func (c *CacheData) WithBuckets(fn func(bs []BucketRecord) error) error
type CountWriter ¶
type CountWriter interface { WriterFlusher // Offset returns the position in the file at which the next write will be made Offset() int64 // Count returns the number of bytes written Count() int64 }
type DatFileHeader ¶
func (*DatFileHeader) DecodeFrom ¶
func (d *DatFileHeader) DecodeFrom(r io.Reader) error
DecodeFrom reads d from a reader
func (*DatFileHeader) EncodeTo ¶
func (d *DatFileHeader) EncodeTo(w io.Writer) error
EncodeTo writes d to a writer
func (*DatFileHeader) Size ¶
func (*DatFileHeader) Size() int
func (*DatFileHeader) Verify ¶
func (d *DatFileHeader) Verify() error
Verify contents of data file header
func (*DatFileHeader) VerifyMatchingKey ¶
func (d *DatFileHeader) VerifyMatchingKey(k *KeyFileHeader) error
VerifyMatchingKey makes sure key file and data file headers match
type DataFile ¶
type DataFile struct { Path string Header DatFileHeader // contains filtered or unexported fields }
DataFile assumes it has exclusive write access to the file
func OpenDataFile ¶
OpenDataFile opens a data file for appending and random reads
func (*DataFile) AppendBucketSpill ¶
func (*DataFile) AppendRecord ¶
func (d *DataFile) AppendRecord(dr *DataRecord) (int64, error)
AppendRecord writes a record to the data file. It returns the position at which the record was written.
func (*DataFile) LoadBucketSpill ¶
func (*DataFile) LoadRecordHeader ¶
func (d *DataFile) LoadRecordHeader(offset int64) (*DataRecordHeader, error)
func (*DataFile) RecordDataReader ¶
func (*DataFile) RecordScanner ¶
func (d *DataFile) RecordScanner() *RecordScanner
RecordScanner returns a RecordScanner that may be used to iterate over the records in the data file.
type DataRecord ¶
type DataRecord struct {
// contains filtered or unexported fields
}
type DataRecordHeader ¶
DataRecordHeader is prepended to each record written to the data file. Layout is:
6 bytes DataSize 2 bytes KeySize n bytes Key
func (*DataRecordHeader) IsData ¶
func (d *DataRecordHeader) IsData() bool
IsData reports whether the data record contains data
func (*DataRecordHeader) IsSpill ¶
func (d *DataRecordHeader) IsSpill() bool
IsSpill reports whether the data record is a bucket spill
func (*DataRecordHeader) Size ¶ added in v0.2.0
func (d *DataRecordHeader) Size() int64
Size returns the size of the header in bytes
type KeyFile ¶
type KeyFile struct { Path string Header KeyFileHeader // contains filtered or unexported fields }
KeyFile assumes it has exclusive write access to the file
func OpenKeyFile ¶
OpenKeyFile opens a key file for random reads and writes
func (*KeyFile) BucketScanner ¶
func (k *KeyFile) BucketScanner(df *DataFile) *BucketScanner
BucketScanner returns a BucketScanner that may be used to iterate over the buckets in the key file.
func (*KeyFile) HashString ¶
type KeyFileHeader ¶
type KeyFileHeader struct { Type [8]byte Version uint16 UID uint64 AppNum uint64 Salt uint64 Pepper uint64 BlockSize uint16 LoadFactor uint16 // Computed values Capacity int // Entries per bucket Buckets int // Number of buckets Modulus uint64 // pow(2,ceil(log2(buckets))) }
func (*KeyFileHeader) DecodeFrom ¶
func (k *KeyFileHeader) DecodeFrom(r io.Reader, fileSize int64) error
func (*KeyFileHeader) Size ¶
func (k *KeyFileHeader) Size() int
func (*KeyFileHeader) Verify ¶
func (k *KeyFileHeader) Verify() error
Verify contents of key file header
type LogFile ¶
type LogFile struct { Path string Header LogFileHeader // contains filtered or unexported fields }
func OpenLogFile ¶
OpenLogFile opens a log file for appending, creating it if necessary.
type LogFileHeader ¶
type LogFileHeader struct { Type [8]byte Version uint16 UID uint64 AppNum uint64 Salt uint64 Pepper uint64 BlockSize uint16 KeyFileSize int64 DatFileSize int64 }
func (*LogFileHeader) DecodeFrom ¶
func (l *LogFileHeader) DecodeFrom(r io.Reader) error
func (*LogFileHeader) Size ¶
func (l *LogFileHeader) Size() int
type Pool ¶
type Pool struct {
// contains filtered or unexported fields
}
Buffers data records in a map
func (*Pool) WithRecords ¶
func (p *Pool) WithRecords(fn func([]DataRecord) error) error
type RecordScanner ¶
type RecordScanner struct {
// contains filtered or unexported fields
}
RecordScanner implements a sequential scan through a data file. Successive calls to the Next method will step through the records in the file.
func (*RecordScanner) Close ¶
func (s *RecordScanner) Close() error
func (*RecordScanner) Err ¶
func (s *RecordScanner) Err() error
Err returns the first non-EOF error that was encountered by the RecordScanner.
func (*RecordScanner) IsData ¶
func (s *RecordScanner) IsData() bool
IsData reports whether the current record is a data record
func (*RecordScanner) IsSpill ¶
func (s *RecordScanner) IsSpill() bool
IsSpill reports whether the current record is a bucket spill
func (*RecordScanner) Key ¶
func (s *RecordScanner) Key() string
Key returns the key of the current record
func (*RecordScanner) Next ¶
func (s *RecordScanner) Next() bool
Next reads the next bucket in sequence, including spills to the data store. It returns false if it encounters an error or there are no more buckets to read.
func (*RecordScanner) Reader ¶
func (s *RecordScanner) Reader() io.Reader
Reader returns an io.Reader that may be used to read the data from the record. Should not be called until Next has been called. The Reader is only valid for use until the next call to Next().
func (*RecordScanner) RecordSize ¶
func (s *RecordScanner) RecordSize() int64
RecordSize returns the number of bytes occupied by the current record including its header
func (*RecordScanner) Size ¶
func (s *RecordScanner) Size() int64
Size returns the size of the current record's data in bytes
type SectionWriter ¶
type SectionWriter struct {
// contains filtered or unexported fields
}
SectionWriter implements Write on a section of an underlying WriterAt
func NewSectionWriter ¶
func NewSectionWriter(w io.WriterAt, offset int64, size int64) *SectionWriter
type Store ¶
type Store struct {
// contains filtered or unexported fields
}
func (*Store) RecordCount ¶ added in v0.2.0
type VerifyResult ¶
type VerifyResult struct { DatPath string // The path to the data file KeyPath string // The path to the key file Version uint16 // The API version used to create the database UID uint64 // The unique identifier AppNum uint64 // The application-defined constant Salt uint64 // The salt used in the key file Pepper uint64 // The salt fingerprint BlockSize uint16 // The block size used in the key file LoadFactor float64 // The target load factor used in the key file KeyFileSize int64 // The size of the key file in bytes DatFileSize int64 // The size of the data file in bytes Capacity int // The maximum number of keys each bucket can hold Buckets int // The number of buckets in the key file BucketSize int64 // The size of a bucket in bytes Modulus uint64 KeyCount int64 // The number of keys found ValueCountInUse int64 // The number of values found that are referenced by a key ValueCountTotal int64 // The number of values found ValueBytesInUse int64 // The total number of bytes occupied by values that are referenced by a key ValueBytesTotal int64 // The total number of bytes occupied by values RecordBytesInUse int64 // The total number of bytes occupied by records (header + value) that are referenced by a key RecordBytesTotal int64 // The total number of bytes occupied by records (header + value) SpillCountInUse int64 // The number of spill records in use SpillCountTotal int64 // The total number of spill records SpillBytesInUse int64 // The number of bytes occupied by spill records in use SpillBytesTotal int64 // The number of bytes occupied by all spill records AverageFetch float64 // Average number of key file reads per fetch Waste float64 // The fraction of the data file that is wasted Overhead float64 // The data amplification ratio (size of data files compared to the size of the underlying data and keys) ActualLoad float64 // The measured bucket load fraction (number of keys as a fraction of the total capacity) }
func VerifyStore ¶
func VerifyStore(datPath, keyPath string, logger logr.Logger) (*VerifyResult, error)
VerifyStore verifies consistency of the data and key files.