storage

Documentation

Index

• Variables
• func EncodeTSO(ts int64) uint64
• func InitMetircs(registry *prometheus.Registry)
• type Append
  • func NewAppend(dir string, options *Options) (append *Append, err error)
  • func NewAppendWithResolver(dir string, options *Options, tiStore kv.Storage, ...) (append *Append, err error)
  • func (a *Append) AllMatched() bool
  • func (a *Append) Close() error
  • func (a *Append) GC(ts int64)
  • func (a *Append) GetBinlog(ts int64) (*pb.Binlog, error)
  • func (a *Append) GetGCTS() int64
  • func (a *Append) MaxCommitTS() int64
  • func (a *Append) PullCommitBinlog(ctx context.Context, last int64) <-chan *pb.Entity
  • func (a *Append) WriteBinlog(binlog *pb.Binlog) error
• type Config
  • func (c *Config) GetKVChanCapacity() int
  • func (c *Config) GetSlowWriteThreshold() float64
  • func (c *Config) GetStopWriteAtAvailableSpace() uint64
  • func (c *Config) GetSyncLog() bool
• type Helper
  • func (h *Helper) GetMvccByEncodedKey(encodedKey kv.Key) (*kvrpcpb.MvccGetByKeyResponse, error)
• type HumanizeBytes
  • func (b HumanizeBytes) Uint64() uint64
  • func (b *HumanizeBytes) UnmarshalText(text []byte) error
• type KVConfig
• type Options
  • func DefaultOptions() *Options
  • func (o *Options) WithKVChanCapacity(capacity int) *Options
  • func (o *Options) WithKVConfig(kvConfig *KVConfig) *Options
  • func (o *Options) WithSlowWriteThreshold(threshold float64) *Options
  • func (o *Options) WithStopWriteAtAvailableSpace(bytes uint64) *Options
  • func (o *Options) WithSync(sync bool) *Options
  • func (o *Options) WithValueLogFileSize(size int64) *Options
• type Record
• type Storage

Variables

var (
	// ErrWrongMagic means the magic number mismatch
	ErrWrongMagic = errors.New("wrong magic")
)

Functions

func EncodeTSO

func EncodeTSO(ts int64) uint64

EncodeTSO encodes a millisecond into tso. TODO: Use the function defined in github.com/tikv/client-go/v2.

func InitMetircs

func InitMetircs(registry *prometheus.Registry)

InitMetircs register the metrics to registry

Types

type Append

type Append struct {
	// contains filtered or unexported fields
}

Append implement the Storage interface

func NewAppend

func NewAppend(dir string, options *Options) (append *Append, err error)

NewAppend returns a instance of Append

func NewAppendWithResolver

func NewAppendWithResolver(dir string, options *Options, tiStore kv.Storage, tiLockResolver *txnlock.LockResolver) (append *Append, err error)

NewAppendWithResolver returns a instance of Append if tiStore and tiLockResolver is not nil, we will try to query tikv to know whether a txn is committed

func (*Append) AllMatched

func (a *Append) AllMatched() bool

AllMatched implement Storage.AllMatched

func (*Append) Close

func (a *Append) Close() error

Close release resource of Append

func (*Append) GC

func (a *Append) GC(ts int64)

GC implement Storage.GC

func (*Append) GetBinlog

func (a *Append) GetBinlog(ts int64) (*pb.Binlog, error)

GetBinlog gets binlog by ts

func (*Append) GetGCTS

func (a *Append) GetGCTS() int64

GetGCTS implement Storage.GetGCTS

func (*Append) MaxCommitTS

func (a *Append) MaxCommitTS() int64

MaxCommitTS implement Storage.MaxCommitTS

func (*Append) PullCommitBinlog

func (a *Append) PullCommitBinlog(ctx context.Context, last int64) <-chan *pb.Entity

PullCommitBinlog return commit binlog > last

func (*Append) WriteBinlog

func (a *Append) WriteBinlog(binlog *pb.Binlog) error

WriteBinlog implement Storage.WriteBinlog

type Config

type Config struct {
	SyncLog *bool `toml:"sync-log" json:"sync-log"`
	// the channel to buffer binlog meta, pump will block write binlog request if the channel is full
	KVChanCapacity            int            `toml:"kv_chan_cap" json:"kv_chan_cap"`
	SlowWriteThreshold        float64        `toml:"slow_write_threshold" json:"slow_write_threshold"`
	KV                        *KVConfig      `toml:"kv" json:"kv"`
	StopWriteAtAvailableSpace *HumanizeBytes `toml:"stop-write-at-available-space" json:"stop-write-at-available-space"`
}

Config holds the configuration of storage

func (*Config) GetKVChanCapacity

func (c *Config) GetKVChanCapacity() int

GetKVChanCapacity return kv_chan_cap config option

func (*Config) GetSlowWriteThreshold

func (c *Config) GetSlowWriteThreshold() float64

GetSlowWriteThreshold return slow write threshold

func (*Config) GetStopWriteAtAvailableSpace

func (c *Config) GetStopWriteAtAvailableSpace() uint64

GetStopWriteAtAvailableSpace return stop write available space

func (*Config) GetSyncLog

func (c *Config) GetSyncLog() bool

GetSyncLog return sync-log config option

type Helper

type Helper struct {
	Store       tikv.Storage
	RegionCache *tikv.RegionCache
}

Helper is a middleware to get some information from tikv/pd.

func (*Helper) GetMvccByEncodedKey

func (h *Helper) GetMvccByEncodedKey(encodedKey kv.Key) (*kvrpcpb.MvccGetByKeyResponse, error)

GetMvccByEncodedKey get the MVCC value by the specific encoded key.

type HumanizeBytes

type HumanizeBytes uint64

HumanizeBytes is used for humanize configure

func (HumanizeBytes) Uint64

func (b HumanizeBytes) Uint64() uint64

Uint64 return bytes

func (*HumanizeBytes) UnmarshalText

func (b *HumanizeBytes) UnmarshalText(text []byte) error

UnmarshalText implements UnmarshalText

type KVConfig

type KVConfig struct {
	BlockCacheCapacity            int     `toml:"block-cache-capacity" json:"block-cache-capacity"`
	BlockRestartInterval          int     `toml:"block-restart-interval" json:"block-restart-interval"`
	BlockSize                     int     `toml:"block-size" json:"block-size"`
	CompactionL0Trigger           int     `toml:"compaction-L0-trigger" json:"compaction-L0-trigger"`
	CompactionTableSize           int     `toml:"compaction-table-size" json:"compaction-table-size"`
	CompactionTotalSize           int     `toml:"compaction-total-size" json:"compaction-total-size"`
	CompactionTotalSizeMultiplier float64 `toml:"compaction-total-size-multiplier" json:"compaction-total-size-multiplier"`
	WriteBuffer                   int     `toml:"write-buffer" json:"write-buffer"`
	WriteL0PauseTrigger           int     `toml:"write-L0-pause-trigger" json:"write-L0-pause-trigger"`
	WriteL0SlowdownTrigger        int     `toml:"write-L0-slowdown-trigger" json:"write-L0-slowdown-trigger"`
}

KVConfig if the configuration of goleveldb

type Options

type Options struct {
	ValueLogFileSize          int64
	Sync                      bool
	KVChanCapacity            int
	SlowWriteThreshold        float64
	StopWriteAtAvailableSpace uint64

	KVConfig *KVConfig
}

Options is the config options of Append and vlog

func DefaultOptions

func DefaultOptions() *Options

DefaultOptions return the default options

func (*Options) WithKVChanCapacity

func (o *Options) WithKVChanCapacity(capacity int) *Options

WithKVChanCapacity set the ChanCapacity

func (*Options) WithKVConfig

func (o *Options) WithKVConfig(kvConfig *KVConfig) *Options

WithKVConfig set the Config

func (*Options) WithSlowWriteThreshold

func (o *Options) WithSlowWriteThreshold(threshold float64) *Options

WithSlowWriteThreshold set the Config

func (*Options) WithStopWriteAtAvailableSpace

func (o *Options) WithStopWriteAtAvailableSpace(bytes uint64) *Options

WithStopWriteAtAvailableSpace set the Config

func (*Options) WithSync

func (o *Options) WithSync(sync bool) *Options

WithSync set the Sync

func (*Options) WithValueLogFileSize

func (o *Options) WithValueLogFileSize(size int64) *Options

WithValueLogFileSize set the ValueLogFileSize

type Record

type Record struct {
	// contains filtered or unexported fields
}

Record is the format in the log file

type Storage

type Storage interface {
	WriteBinlog(binlog *pb.Binlog) error

	// delete <= ts
	GC(ts int64)

	GetGCTS() int64

	// AllMatched return if all the P-binlog have the matching C-binlog
	AllMatched() bool

	MaxCommitTS() int64

	// GetBinlog return the binlog of ts
	GetBinlog(ts int64) (binlog *pb.Binlog, err error)

	// PullCommitBinlog return the chan to consume the binlog
	PullCommitBinlog(ctx context.Context, last int64) <-chan *pb.Entity

	Close() error
}

Storage is the interface to handle binlog storage