lsm

package
v1.0.2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 16, 2025 License: MIT Imports: 28 Imported by: 0

Documentation

Index

Constants

View Source
const KvWriteChCapacity = 1000
View Source
const MemTableName string = ".memtable"
View Source
const SSTableName string = ".sst"
View Source
const (
	WalHeaderSize int = 17
)

Variables

This section is empty.

Functions

func CheckLSMOpt

func CheckLSMOpt(opt *Options) (func() error, error)

func EstimateWalEncodeSize

func EstimateWalEncodeSize(e *model.Entry) int

EstimateWalEncodeSize WalEncode | header(klen,vlen,meta,expir) | key | value | crc32 |

func GetSSTablePathFromId

func GetSSTablePathFromId(dir string, id uint64) string

func IsDeletedOrExpired

func IsDeletedOrExpired(e model.Entry) bool

func NewMergeIterator

func NewMergeIterator(iters []model.Iterator, reverse bool) model.Iterator

Types

type ConcatIterator

type ConcatIterator struct {
	// contains filtered or unexported fields
}

func NewConcatIterator

func NewConcatIterator(tables []*table, opt *model.Options) *ConcatIterator

func (*ConcatIterator) Close

func (conIter *ConcatIterator) Close() error

func (*ConcatIterator) Item

func (conIter *ConcatIterator) Item() model.Item

func (*ConcatIterator) Key

func (conIter *ConcatIterator) Key() []byte

func (*ConcatIterator) Name

func (s *ConcatIterator) Name() string

func (*ConcatIterator) Next

func (conIter *ConcatIterator) Next()

func (*ConcatIterator) Rewind

func (conIter *ConcatIterator) Rewind()

func (*ConcatIterator) Seek

func (conIter *ConcatIterator) Seek(key []byte)

func (*ConcatIterator) Valid

func (conIter *ConcatIterator) Valid() bool

func (*ConcatIterator) Value

func (conIter *ConcatIterator) Value() []byte

type LSM

type LSM struct {
	sync.RWMutex

	ExpiredValPtrChan chan model.ValuePtr // compact`MergeIterator`fix() to lsm;
	ExpiredValNum     int
	ExpiredValSize    int64
	// contains filtered or unexported fields
}

func NewLSM

func NewLSM(opt *Options, closer *utils.Closer) *LSM

func (*LSM) Close

func (lsm *LSM) Close() error

func (*LSM) CloseFlushIMemChan

func (lsm *LSM) CloseFlushIMemChan()

func (*LSM) Get

func (lsm *LSM) Get(keyTs []byte) (model.Entry, error)

func (*LSM) GetSkipListFromMemTable

func (lsm *LSM) GetSkipListFromMemTable() *skl.SkipList

func (*LSM) InitLevelManger

func (lsm *LSM) InitLevelManger(opt *Options) *levelsManger

func (*LSM) MemSize

func (lsm *LSM) MemSize() int64

func (*LSM) MemTableIsNil

func (lsm *LSM) MemTableIsNil() bool

func (*LSM) NewLsmIterator

func (lsm *LSM) NewLsmIterator(opt *model.Options) []model.Iterator

func (*LSM) NewMemoryTable

func (lsm *LSM) NewMemoryTable() *memoryTable

func (*LSM) Put

func (lsm *LSM) Put(entry model.Entry) (err error)

func (*LSM) Rotate

func (lsm *LSM) Rotate()

func (*LSM) StartCompacter

func (lsm *LSM) StartCompacter(closer *utils.Closer)

func (*LSM) StartFlushMemTable

func (lsm *LSM) StartFlushMemTable(closer *utils.Closer)

type LevelManifest

type LevelManifest struct {
	TableIds map[uint64]struct{}
}

type LevelsCache

type LevelsCache struct {
	// contains filtered or unexported fields
}

type Manifest

type Manifest struct {
	Levels    []LevelManifest
	Tables    map[uint64]TableManifest
	Creations int
	Deletions int
}

func NewManifest

func NewManifest() *Manifest

func ReplyManifestFile

func ReplyManifestFile(file *os.File) (m *Manifest, truncOffset int64, err error)

type ManifestFile

type ManifestFile struct {
	// contains filtered or unexported fields
}

func OpenManifestFile

func OpenManifestFile(opt *utils.FileOptions) (*ManifestFile, error)

func (*ManifestFile) AddChanges

func (mf *ManifestFile) AddChanges(changes []*pb.ManifestChange) error

func (*ManifestFile) AddTableMeta

func (mf *ManifestFile) AddTableMeta(levelNum int, t *TableMeta) (err error)

func (*ManifestFile) Close

func (mf *ManifestFile) Close() error

func (*ManifestFile) GetManifest

func (mf *ManifestFile) GetManifest() *Manifest

type MergeIterator

type MergeIterator struct {
	// contains filtered or unexported fields
}

func (*MergeIterator) Close

func (m *MergeIterator) Close() error

func (*MergeIterator) Item

func (m *MergeIterator) Item() model.Item

func (*MergeIterator) Key

func (m *MergeIterator) Key() []byte

func (*MergeIterator) Name

func (iter *MergeIterator) Name() string

func (*MergeIterator) Next

func (m *MergeIterator) Next()

func (*MergeIterator) Rewind

func (m *MergeIterator) Rewind()

func (*MergeIterator) Seek

func (m *MergeIterator) Seek(key []byte)

func (*MergeIterator) Valid

func (m *MergeIterator) Valid() bool

func (*MergeIterator) Value

func (m *MergeIterator) Value() []byte

type Options

type Options struct {
	WorkDir           string // 工作数据目录;
	MemTableSize      int64  // 内存表最大限制;
	SSTableMaxSz      int64  // SSSTable 最大限制,同上;
	NumFlushMemtables int    // 刷盘队列大小;
	BlockSize         uint32 // 数据块持久化时的大小;

	BloomFalsePositive float64 // 布隆过滤器的容错率;

	CacheNums int // 缓存元素个数, 缺省值默认 1024*10个;

	ValueThreshold      int   // 进入vlog的value阈值;
	ValueLogMaxEntries  int32 // vlogFile文件保存的entry最大数量;
	ValueLogFileSize    int32 // vlogFile的文件大小;
	VerifyValueChecksum bool  // 是否开启vlogFile的crc检查;

	MaxBatchCount int64 // 批处理entry数量;
	MaxBatchSize  int64 // 批处理entry总量大小;

	// compact 合并相关
	NumCompactors       int   // 合并协程数量;默认2;
	BaseLevelSize       int64 // 基层中 所期望的文件大小;
	LevelSizeMultiplier int   // 决定 level 之间期望 总体文件 size 比例, 默认是 10倍;
	TableSizeMultiplier int   // 决定每层 文件 递增倍数;
	BaseTableSize       int64 // 基层中 文件所期望的文件大小;
	NumLevelZeroTables  int   // 第 0 层中,允许的表数量;
	MaxLevelNum         int   // 最大层数,默认是 7 层;

	DiscardStatsCh *chan map[uint32]int64 //  用于 compact 组件向 vlog 组件传递信息使用,在合并过程中,知道哪些文件是失效的,让vlog组件知道,方便其GC;
}

func GetLSMDefaultOpt

func GetLSMDefaultOpt(dirPath string) *Options

type SSTable

type SSTable struct {
	// contains filtered or unexported fields
}

func OpenSStable

func OpenSStable(opt *utils.FileOptions) *SSTable

func (*SSTable) Bytes

func (sst *SSTable) Bytes(off, sz int) ([]byte, error)

func (*SSTable) Close

func (sst *SSTable) Close() error

func (*SSTable) Detele

func (sst *SSTable) Detele() error

func (*SSTable) FID

func (sst *SSTable) FID() uint64

func (*SSTable) GetCreatedAt

func (sst *SSTable) GetCreatedAt() *time.Time

func (*SSTable) HasBloomFilter

func (sst *SSTable) HasBloomFilter() bool

func (*SSTable) Indexs

func (sst *SSTable) Indexs() *pb.TableIndex

func (*SSTable) Init

func (sst *SSTable) Init() error

func (*SSTable) MaxKey

func (sst *SSTable) MaxKey() []byte

func (*SSTable) MinKey

func (sst *SSTable) MinKey() []byte

func (*SSTable) SetCreatedAt

func (sst *SSTable) SetCreatedAt(t *time.Time)

func (*SSTable) SetMaxKey

func (sst *SSTable) SetMaxKey(maxKey []byte)

SetMaxKey max 需要使用table的迭代器,来获取最后一个block的最后一个key;

func (*SSTable) Size

func (sst *SSTable) Size() int64

Size 返回底层文件的尺寸;

func (*SSTable) Truncature

func (sst *SSTable) Truncature(size int64) error

type TableManifest

type TableManifest struct {
	ID       uint64
	LevelID  uint8
	CheckSum []byte
}

type TableMeta

type TableMeta struct {
	ID       uint64
	Checksum []byte
}

type WAL

type WAL struct {
	// contains filtered or unexported fields
}

func OpenWalFile

func OpenWalFile(opt *utils.FileOptions) *WAL

func (*WAL) Close

func (w *WAL) Close() error

func (*WAL) CloseAndRemove

func (w *WAL) CloseAndRemove() error

func (*WAL) Fid

func (w *WAL) Fid() uint64

func (*WAL) Name

func (w *WAL) Name() string

func (*WAL) Read

func (w *WAL) Read(reader io.Reader) (model.Entry, uint32)

func (*WAL) SetSize

func (w *WAL) SetSize(offset uint32)

func (*WAL) Size

func (w *WAL) Size() uint32

func (*WAL) WalDecode

func (w *WAL) WalDecode(reader io.Reader) (model.Entry, error)

func (*WAL) WalEncode

func (w *WAL) WalEncode(e model.Entry) ([]byte, int)

WalEncode | header(klen,vlen,meta,expir) | key | value | crc32 |

func (*WAL) Write

func (w *WAL) Write(e model.Entry) error

type WalHeader

type WalHeader struct {
	Meta      byte   // 1B
	ExpiredAt uint64 // 8B
	// contains filtered or unexported fields
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL