lsm

package
v1.0.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 16, 2025 License: MIT Imports: 28 Imported by: 0

Documentation

Index

Constants

View Source 
const KvWriteChCapacity = 1000
View Source 
const MemTableName string = ".memtable"
View Source 
const SSTableName string = ".sst"
View Source 
const (
	WalHeaderSize int = 17
)

Variables

This section is empty.

Functions

func CheckLSMOpt 

func CheckLSMOpt(opt *Options) (func() error, error)

func EstimateWalEncodeSize 

func EstimateWalEncodeSize(e *model.Entry) int

EstimateWalEncodeSize WalEncode | header(klen,vlen,meta,expir) | key | value | crc32 |

func GetSSTablePathFromId 

func GetSSTablePathFromId(dir string, id uint64) string

func IsDeletedOrExpired 

func IsDeletedOrExpired(e model.Entry) bool

func NewMergeIterator 

func NewMergeIterator(iters []model.Iterator, reverse bool) model.Iterator

Types

type ConcatIterator 

type ConcatIterator struct {
	// contains filtered or unexported fields
}

func NewConcatIterator 

func NewConcatIterator(tables []*table, opt *model.Options) *ConcatIterator

func (*ConcatIterator) Close 

func (conIter *ConcatIterator) Close() error

func (*ConcatIterator) Item 

func (conIter *ConcatIterator) Item() model.Item

func (*ConcatIterator) Key 

func (conIter *ConcatIterator) Key() []byte

func (*ConcatIterator) Name 

func (s *ConcatIterator) Name() string

func (*ConcatIterator) Next 

func (conIter *ConcatIterator) Next()

func (*ConcatIterator) Rewind 

func (conIter *ConcatIterator) Rewind()

func (*ConcatIterator) Seek 

func (conIter *ConcatIterator) Seek(key []byte)

func (*ConcatIterator) Valid 

func (conIter *ConcatIterator) Valid() bool

func (*ConcatIterator) Value 

func (conIter *ConcatIterator) Value() []byte

type LSM 

type LSM struct {
	sync.RWMutex

	ExpiredValPtrChan chan model.ValuePtr // compact`MergeIterator`fix() to lsm;
	ExpiredValNum     int
	ExpiredValSize    int64
	// contains filtered or unexported fields
}

func NewLSM 

func NewLSM(opt *Options, closer *utils.Closer) *LSM

func (*LSM) Close 

func (lsm *LSM) Close() error

func (*LSM) CloseFlushIMemChan 

func (lsm *LSM) CloseFlushIMemChan()

func (*LSM) Get 

func (lsm *LSM) Get(keyTs []byte) (model.Entry, error)

func (*LSM) GetSkipListFromMemTable 

func (lsm *LSM) GetSkipListFromMemTable() *skl.SkipList

func (*LSM) InitLevelManger 

func (lsm *LSM) InitLevelManger(opt *Options) *levelsManger

func (*LSM) MemSize 

func (lsm *LSM) MemSize() int64

func (*LSM) MemTableIsNil 

func (lsm *LSM) MemTableIsNil() bool

func (*LSM) NewLsmIterator 

func (lsm *LSM) NewLsmIterator(opt *model.Options) []model.Iterator

func (*LSM) NewMemoryTable 

func (lsm *LSM) NewMemoryTable() *memoryTable

func (*LSM) Put 

func (lsm *LSM) Put(entry model.Entry) (err error)

func (*LSM) Rotate 

func (lsm *LSM) Rotate()

func (*LSM) StartCompacter 

func (lsm *LSM) StartCompacter(closer *utils.Closer)

func (*LSM) StartFlushMemTable 

func (lsm *LSM) StartFlushMemTable(closer *utils.Closer)

type LevelManifest 

type LevelManifest struct {
	TableIds map[uint64]struct{}
}

type LevelsCache 

type LevelsCache struct {
	// contains filtered or unexported fields
}

type Manifest 

type Manifest struct {
	Levels    []LevelManifest
	Tables    map[uint64]TableManifest
	Creations int
	Deletions int
}

func NewManifest 

func NewManifest() *Manifest

func ReplyManifestFile 

func ReplyManifestFile(file *os.File) (m *Manifest, truncOffset int64, err error)

type ManifestFile 

type ManifestFile struct {
	// contains filtered or unexported fields
}

func OpenManifestFile 

func OpenManifestFile(opt *utils.FileOptions) (*ManifestFile, error)

func (*ManifestFile) AddChanges 

func (mf *ManifestFile) AddChanges(changes []*pb.ManifestChange) error

func (*ManifestFile) AddTableMeta 

func (mf *ManifestFile) AddTableMeta(levelNum int, t *TableMeta) (err error)

func (*ManifestFile) Close 

func (mf *ManifestFile) Close() error

func (*ManifestFile) GetManifest 

func (mf *ManifestFile) GetManifest() *Manifest

type MergeIterator 

type MergeIterator struct {
	// contains filtered or unexported fields
}

func (*MergeIterator) Close 

func (m *MergeIterator) Close() error

func (*MergeIterator) Item 

func (m *MergeIterator) Item() model.Item

func (*MergeIterator) Key 

func (m *MergeIterator) Key() []byte

func (*MergeIterator) Name 

func (iter *MergeIterator) Name() string

func (*MergeIterator) Next 

func (m *MergeIterator) Next()

func (*MergeIterator) Rewind 

func (m *MergeIterator) Rewind()

func (*MergeIterator) Seek 

func (m *MergeIterator) Seek(key []byte)

func (*MergeIterator) Valid 

func (m *MergeIterator) Valid() bool

func (*MergeIterator) Value 

func (m *MergeIterator) Value() []byte

type Options 

type Options struct {
	WorkDir           string // 工作数据目录;
	MemTableSize      int64  // 内存表最大限制;
	SSTableMaxSz      int64  // SSSTable 最大限制,同上;
	NumFlushMemtables int    // 刷盘队列大小;
	BlockSize         uint32 // 数据块持久化时的大小;

	BloomFalsePositive float64 // 布隆过滤器的容错率;

	CacheNums int // 缓存元素个数, 缺省值默认 1024*10个;

	ValueThreshold      int   // 进入vlog的value阈值;
	ValueLogMaxEntries  int32 // vlogFile文件保存的entry最大数量;
	ValueLogFileSize    int32 // vlogFile的文件大小;
	VerifyValueChecksum bool  // 是否开启vlogFile的crc检查;

	MaxBatchCount int64 // 批处理entry数量;
	MaxBatchSize  int64 // 批处理entry总量大小;

	// compact 合并相关
	NumCompactors       int   // 合并协程数量;默认2;
	BaseLevelSize       int64 // 基层中 所期望的文件大小;
	LevelSizeMultiplier int   // 决定 level 之间期望 总体文件 size 比例, 默认是 10倍;
	TableSizeMultiplier int   // 决定每层 文件 递增倍数;
	BaseTableSize       int64 // 基层中 文件所期望的文件大小;
	NumLevelZeroTables  int   // 第 0 层中,允许的表数量;
	MaxLevelNum         int   // 最大层数,默认是 7 层;

	DiscardStatsCh *chan map[uint32]int64 //  用于 compact 组件向 vlog 组件传递信息使用,在合并过程中,知道哪些文件是失效的,让vlog组件知道,方便其GC;
}

func GetLSMDefaultOpt 

func GetLSMDefaultOpt(dirPath string) *Options

type SSTable 

type SSTable struct {
	// contains filtered or unexported fields
}

func OpenSStable 

func OpenSStable(opt *utils.FileOptions) *SSTable

func (*SSTable) Bytes 

func (sst *SSTable) Bytes(off, sz int) ([]byte, error)

func (*SSTable) Close 

func (sst *SSTable) Close() error

func (*SSTable) Detele 

func (sst *SSTable) Detele() error

func (*SSTable) FID 

func (sst *SSTable) FID() uint64

func (*SSTable) GetCreatedAt 

func (sst *SSTable) GetCreatedAt() *time.Time

func (*SSTable) HasBloomFilter 

func (sst *SSTable) HasBloomFilter() bool

func (*SSTable) Indexs 

func (sst *SSTable) Indexs() *pb.TableIndex

func (*SSTable) Init 

func (sst *SSTable) Init() error

func (*SSTable) MaxKey 

func (sst *SSTable) MaxKey() []byte

func (*SSTable) MinKey 

func (sst *SSTable) MinKey() []byte

func (*SSTable) SetCreatedAt 

func (sst *SSTable) SetCreatedAt(t *time.Time)

func (*SSTable) SetMaxKey 

func (sst *SSTable) SetMaxKey(maxKey []byte)

SetMaxKey max 需要使用table的迭代器,来获取最后一个block的最后一个key;

func (*SSTable) Size 

func (sst *SSTable) Size() int64

Size 返回底层文件的尺寸;

func (*SSTable) Truncature 

func (sst *SSTable) Truncature(size int64) error

type TableManifest 

type TableManifest struct {
	ID       uint64
	LevelID  uint8
	CheckSum []byte
}

type TableMeta 

type TableMeta struct {
	ID       uint64
	Checksum []byte
}

type WAL 

type WAL struct {
	// contains filtered or unexported fields
}

func OpenWalFile 

func OpenWalFile(opt *utils.FileOptions) *WAL

func (*WAL) Close 

func (w *WAL) Close() error

func (*WAL) CloseAndRemove 

func (w *WAL) CloseAndRemove() error

func (*WAL) Fid 

func (w *WAL) Fid() uint64

func (*WAL) Name 

func (w *WAL) Name() string

func (*WAL) Read 

func (w *WAL) Read(reader io.Reader) (model.Entry, uint32)

func (*WAL) SetSize 

func (w *WAL) SetSize(offset uint32)

func (*WAL) Size 

func (w *WAL) Size() uint32

func (*WAL) WalDecode 

func (w *WAL) WalDecode(reader io.Reader) (model.Entry, error)

func (*WAL) WalEncode 

func (w *WAL) WalEncode(e model.Entry) ([]byte, int)

WalEncode | header(klen,vlen,meta,expir) | key | value | crc32 |

func (*WAL) Write 

func (w *WAL) Write(e model.Entry) error

type WalHeader 

type WalHeader struct {
	Meta      byte   // 1B
	ExpiredAt uint64 // 8B
	// contains filtered or unexported fields
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.