velocitylog

Documentation

Index

• Constants
• func Add(bloomFilter *pb.BloomFilter, key []byte)
• func Contains(bloomFilter *pb.BloomFilter, key []byte) bool
• func MarshalEntry(pb proto.Message) []byte
• func UnmarshalEntry(bytes []byte, entry protoreflect.ProtoMessage)
• type EntrySize
  • func WriteToSSTable(filename string, bloomFilterData []byte, indexData []byte, ...) (EntrySize, error)
• type KVPair
• type LSMTree
  • func Open(directory string, maxMemtableSize int64, recoveryFromWAL bool) (*LSMTree, error)
  • func (l *LSMTree) Close() error
  • func (l *LSMTree) Delete(key string) error
  • func (l *LSMTree) Get(key string) ([]byte, error)
  • func (l *LSMTree) Put(key string, value []byte) error
  • func (l *LSMTree) RangeScan(startKey string, endKey string) ([]KVPair, error)
• type Memtable
  • func NewMemtable() *Memtable
  • func (m *Memtable) Clear()
  • func (m *Memtable) Delete(key string)
  • func (m *Memtable) GenerateEntries() []*pb.LSMEntry
  • func (m *Memtable) Get(key string) *pb.LSMEntry
  • func (m *Memtable) Len() int
  • func (m *Memtable) Put(key string, value []byte)
  • func (m *Memtable) RangeScan(startKey, endKey string) []*pb.LSMEntry
  • func (m *Memtable) SizeInBytes() int64
• type SSTable
  • func OpenSSTable(filename string) (*SSTable, error)
  • func SerializeToSSTable(messages []*pb.LSMEntry, filename string) (*SSTable, error)
  • func (s *SSTable) Close() error
  • func (s *SSTable) Front() *SSTableIterator
  • func (s *SSTable) Get(key string) (*pb.LSMEntry, error)
  • func (s *SSTable) GetEntries() ([]*pb.LSMEntry, error)
  • func (s *SSTable) RangeScan(startKey, endKey string) ([]*pb.LSMEntry, error)
• type SSTableIterator
  • func (it *SSTableIterator) Close() error
  • func (it *SSTableIterator) Next() *SSTableIterator

Constants

const (
	BloomFilterSize    = 1000000    // 1 million bits.
	InitialOffset      = 0          // Initial offset for the data entries.
	SSTableFilePrefix  = "sstable_" // Prefix for SSTable files.
	WALDirectorySuffix = "_wal"     // Suffix for WAL directory.
	MaxLevels          = 6          // Maximum number of levels in the LSM tree.
	WALMaxFileSize     = 128000     // 128 KB
	WALMaxSegments     = 1000       // Maximum number of WAL segments.
)

Functions

func Add

func Add(bloomFilter *pb.BloomFilter, key []byte)

Add key to the BloomFilter

func Contains

func Contains(bloomFilter *pb.BloomFilter, key []byte) bool

Check if key is in the BloomFilter

func MarshalEntry

func MarshalEntry(pb proto.Message) []byte

MarshalEntry marshals any proto message into a byte slice and panics if there is an error.

func UnmarshalEntry

func UnmarshalEntry(bytes []byte, entry protoreflect.ProtoMessage)

UnmarshalEntry unmarshals the data into the entry and panics if there is an error.

Types

type EntrySize

type EntrySize int64

EntrySize is the size of an entry in the log.

func WriteToSSTable

func WriteToSSTable(filename string, bloomFilterData []byte, indexData []byte, entriesData *bytes.Buffer) (EntrySize, error)

type KVPair

type KVPair struct {
	Key   string
	Value []byte
}

KVPair struct represents a key-value pair for upstream applications.

type LSMTree

type LSMTree struct {
	// contains filtered or unexported fields
}

func Open

func Open(directory string, maxMemtableSize int64, recoveryFromWAL bool) (*LSMTree, error)

func (*LSMTree) Close

func (l *LSMTree) Close() error

func (*LSMTree) Delete

func (l *LSMTree) Delete(key string) error

Delete, Delete a key from the LSM tree.

func (*LSMTree) Get

func (l *LSMTree) Get(key string) ([]byte, error)

func (*LSMTree) Put

func (l *LSMTree) Put(key string, value []byte) error

Put, Insert a key-value pair into the LSM tree.

func (*LSMTree) RangeScan

func (l *LSMTree) RangeScan(startKey string, endKey string) ([]KVPair, error)

type Memtable

type Memtable struct {
	// contains filtered or unexported fields
}

Memtable is a memory table that supports fast writes, reads, deletes, and range scans. It uses a skip list as the underlying data structure to store key-value pairs.

func NewMemtable

func NewMemtable() *Memtable

NewMemtable creates a new memtable.

func (*Memtable) Clear

func (m *Memtable) Clear()

Clear resets the memtable to an empty state. Not Thread-Safe Implementation.

func (*Memtable) Delete

func (m *Memtable) Delete(key string)

Delete deletes a key from the memtable, Not Thread-Safe Implementation.

func (*Memtable) GenerateEntries

func (m *Memtable) GenerateEntries() []*pb.LSMEntry

GenerateEntries returns a serializable list of memtable entries in sorted order. Not Thread-Safe Implementation.

func (*Memtable) Get

func (m *Memtable) Get(key string) *pb.LSMEntry

Get retrieves a value for a given key from the memtable, Not Thread-Safe Implementation.

func (*Memtable) Len

func (m *Memtable) Len() int

func (*Memtable) Put

func (m *Memtable) Put(key string, value []byte)

Put inserts a key-value pair into the memtable, Not Thread-Safe Implementation.

func (*Memtable) RangeScan

func (m *Memtable) RangeScan(startKey, endKey string) []*pb.LSMEntry

RangeScan returns all key-value pairs in the memtable within the given key range, Not Thread-Safe Implementation. The startKey is inclusive, and the endKey is inclusive.

func (*Memtable) SizeInBytes

func (m *Memtable) SizeInBytes() int64

SizeInBytes returns the size of the memtable in bytes. Not Thread-Safe Implementation.

type SSTable

type SSTable struct {
	// contains filtered or unexported fields
}

func OpenSSTable

func OpenSSTable(filename string) (*SSTable, error)

OpenSSTable opens an SSTable file and returns an SSTable object for reading.

func SerializeToSSTable

func SerializeToSSTable(messages []*pb.LSMEntry, filename string) (*SSTable, error)

Writes a list of MemtableKeyValue entires to file in SSTable format. The format of the SSTable file is as follows: 1. Bloom filter size (OffsetSize) 2. Bloom filter data (BloomFilter ProtoBuf) 3. Index size (OffsetSize) 4. Index data (Index ProtoBuf) 5. Data entries

The data entries are written in the following format: 1. Size of the entry (OffsetSize) 2. Entry data (LSMEntry ProtoBuf)

func (*SSTable) Close

func (s *SSTable) Close() error

Close closes the SSTable file.

func (*SSTable) Front

func (s *SSTable) Front() *SSTableIterator

Front returns an Iterator for the SSTable. The iterator is positioned at the first entry in the SSTable.

func (*SSTable) Get

func (s *SSTable) Get(key string) (*pb.LSMEntry, error)

Get returns the value/ for the given key from the SSTable. Returns nil if the key is not found.

func (*SSTable) GetEntries

func (s *SSTable) GetEntries() ([]*pb.LSMEntry, error)

GetEntries returns all the entries in the SSTable.

func (*SSTable) RangeScan

func (s *SSTable) RangeScan(startKey, endKey string) ([]*pb.LSMEntry, error)

rangeScan returns all the entries in the SSTable that have keys in the range [startKey, endKey) inclusive.

type SSTableIterator

type SSTableIterator struct {
	Value *pb.LSMEntry // current entry.
	// contains filtered or unexported fields
}

SSTableIterator is an iterator for SSTable

func (*SSTableIterator) Close

func (it *SSTableIterator) Close() error

Closes the iterator.

func (*SSTableIterator) Next

func (it *SSTableIterator) Next() *SSTableIterator

Next returns the next entry in the SSTable. Returns nil if there are no more entries.