kv

package
v2.0.10-binlog+incompa... Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 26, 2018 License: Apache-2.0 Imports: 20 Imported by: 6,388

Documentation

Index

Constants

View Source 
const (
	PriorityNormal = iota
	PriorityLow
	PriorityHigh
)

Priority value for transaction priority.

View Source 
const (
	ReqTypeSelect   = 101
	ReqTypeIndex    = 102
	ReqTypeDAG      = 103
	ReqTypeAnalyze  = 104
	ReqTypeChecksum = 105

	ReqSubTypeBasic      = 0
	ReqSubTypeDesc       = 10000
	ReqSubTypeGroupBy    = 10001
	ReqSubTypeTopN       = 10002
	ReqSubTypeSignature  = 10003
	ReqSubTypeAnalyzeIdx = 10004
	ReqSubTypeAnalyzeCol = 10005
)

ReqTypes.

Variables

View Source 
var (
	// DefaultTxnMembufCap is the default transaction membuf capability.
	DefaultTxnMembufCap = 4 * 1024
	// ImportingTxnMembufCap is the capability of tidb importing data situation.
	ImportingTxnMembufCap = 32 * 1024
	// TempTxnMemBufCap is the capability of temporary membuf.
	TempTxnMemBufCap = 64
)
View Source 
var (
	// ErrClosed is used when close an already closed txn.
	ErrClosed = terror.ClassKV.New(codeClosed, "Error: Transaction already closed")
	// ErrNotExist is used when try to get an entry with an unexist key from KV store.
	ErrNotExist = terror.ClassKV.New(codeNotExist, "Error: key not exist")
	// ErrConditionNotMatch is used when condition is not met.
	ErrConditionNotMatch = terror.ClassKV.New(codeConditionNotMatch, "Error: Condition not match")
	// ErrLockConflict is used when try to lock an already locked key.
	ErrLockConflict = terror.ClassKV.New(codeLockConflict, "Error: Lock conflict")
	// ErrLazyConditionPairsNotMatch is used when value in store differs from expect pairs.
	ErrLazyConditionPairsNotMatch = terror.ClassKV.New(codeLazyConditionPairsNotMatch, "Error: Lazy condition pairs not match")
	// ErrRetryable is used when KV store occurs RPC error or some other
	// errors which SQL layer can safely retry.
	ErrRetryable = terror.ClassKV.New(codeRetryable, "Error: KV error safe to retry")
	// ErrCannotSetNilValue is the error when sets an empty value.
	ErrCannotSetNilValue = terror.ClassKV.New(codeCantSetNilValue, "can not set nil value")
	// ErrInvalidTxn is the error when commits or rollbacks in an invalid transaction.
	ErrInvalidTxn = terror.ClassKV.New(codeInvalidTxn, "invalid transaction")
	// ErrTxnTooLarge is the error when transaction is too large, lock time reached the maximum value.
	ErrTxnTooLarge = terror.ClassKV.New(codeTxnTooLarge, "transaction is too large")
	// ErrEntryTooLarge is the error when a key value entry is too large.
	ErrEntryTooLarge = terror.ClassKV.New(codeEntryTooLarge, "entry is too large")

	// ErrNotCommitted is the error returned by CommitVersion when this
	// transaction is not committed.
	ErrNotCommitted = terror.ClassKV.New(codeNotCommitted, "this transaction has not committed")

	// ErrKeyExists returns when key is already exist.
	ErrKeyExists = terror.ClassKV.New(codeKeyExists, "key already exist")
	// ErrNotImplemented returns when a function is not implemented yet.
	ErrNotImplemented = terror.ClassKV.New(codeNotImplemented, "not implemented")
)
View Source 
var (
	// TxnEntrySizeLimit is limit of single entry size (len(key) + len(value)).
	TxnEntrySizeLimit = 6 * 1024 * 1024
	// TxnEntryCountLimit  is limit of number of entries in the MemBuffer.
	TxnEntryCountLimit uint64 = 300 * 1000
	// TxnTotalSizeLimit is limit of the sum of all entry size.
	TxnTotalSizeLimit = 100 * 1024 * 1024
)

Those limits is enforced to make sure the transaction can be well handled by TiKV.

View Source 
var (
	// MaxVersion is the maximum version, notice that it's not a valid version.
	MaxVersion = Version{Ver: math.MaxUint64}
	// MinVersion is the minimum version, it's not a valid version, too.
	MinVersion = Version{Ver: 0}
)

Functions

func BackOff 

func BackOff(attempts uint) int

BackOff Implements exponential backoff with full jitter. Returns real back off time in microsecond. See http://www.awsarchitectureblog.com/2015/03/backoff.html.

func BatchGetValues 

func BatchGetValues(txn Transaction, keys []Key) (map[string][]byte, error)

BatchGetValues gets values in batch. The values from buffer in transaction and the values from the storage node are merged together.

func GetInt64 

func GetInt64(r Retriever, k Key) (int64, error)

GetInt64 get int64 value which created by IncInt64 method.

func IncInt64 

func IncInt64(rm RetrieverMutator, k Key, step int64) (int64, error)

IncInt64 increases the value for key k in kv store by step.

func IsErrNotFound 

func IsErrNotFound(err error) bool

IsErrNotFound checks if err is a kind of NotFound error.

func IsRetryableError 

func IsRetryableError(err error) bool

IsRetryableError checks if the err is a fatal error and the under going operation is worth to retry.

func NextUntil 

func NextUntil(it Iterator, fn FnKeyCmp) error

NextUntil applies FnKeyCmp to each entry of the iterator until meets some condition. It will stop when fn returns true, or iterator is invalid or an error occurs.

func RunInNewTxn 

func RunInNewTxn(store Storage, retryable bool, f func(txn Transaction) error) error

RunInNewTxn will run the f in a new transaction environment.

func WalkMemBuffer 

func WalkMemBuffer(memBuf MemBuffer, f func(k Key, v []byte) error) error

WalkMemBuffer iterates all buffered kv pairs in memBuf

Types

type BufferStore 

type BufferStore struct {
	MemBuffer
	// contains filtered or unexported fields
}

BufferStore wraps a Retriever for read and a MemBuffer for buffered write. Common usage pattern: 

bs := NewBufferStore(r) // use BufferStore to wrap a Retriever
// ...
// read/write on bs
// ...
bs.SaveTo(m)	        // save above operations to a Mutator

func NewBufferStore 

func NewBufferStore(r Retriever, cap int) *BufferStore

NewBufferStore creates a BufferStore using r for read.

func (*BufferStore) Get 

func (s *BufferStore) Get(k Key) ([]byte, error)

Get implements the Retriever interface.

func (*BufferStore) Iter 

func (s *BufferStore) Iter(k Key, upperBound Key) (Iterator, error)

Iter implements the Retriever interface.

func (*BufferStore) IterReverse 

func (s *BufferStore) IterReverse(k Key) (Iterator, error)

IterReverse implements the Retriever interface.

func (*BufferStore) Reset 

func (s *BufferStore) Reset()

Reset resets s.MemBuffer.

func (*BufferStore) SaveTo 

func (s *BufferStore) SaveTo(m Mutator) error

SaveTo saves all buffered kv pairs into a Mutator.

func (*BufferStore) SetCap 

func (s *BufferStore) SetCap(cap int)

SetCap sets the MemBuffer capability.

func (*BufferStore) WalkBuffer 

func (s *BufferStore) WalkBuffer(f func(k Key, v []byte) error) error

WalkBuffer iterates all buffered kv pairs.

type Client 

type Client interface {
	// Send sends request to KV layer, returns a Response.
	Send(ctx context.Context, req *Request) Response

	// IsRequestTypeSupported checks if reqType and subType is supported.
	IsRequestTypeSupported(reqType, subType int64) bool
}

Client is used to send request to KV layer.

type Driver 

type Driver interface {
	// Open returns a new Storage.
	// The path is the string for storage specific format.
	Open(path string) (Storage, error)
}

Driver is the interface that must be implemented by a KV storage.

type EncodedKey 

type EncodedKey []byte

EncodedKey represents encoded key in low-level storage engine.

func (EncodedKey) Cmp 

func (k EncodedKey) Cmp(another EncodedKey) int

Cmp returns the comparison result of two key. The result will be 0 if a==b, -1 if a < b, and +1 if a > b.

func (EncodedKey) Next 

func (k EncodedKey) Next() EncodedKey

Next returns the next key in byte-order.

type FnKeyCmp 

type FnKeyCmp func(key Key) bool

FnKeyCmp is the function for iterator the keys

type InjectedSnapshot 

type InjectedSnapshot struct {
	Snapshot
	// contains filtered or unexported fields
}

InjectedSnapshot wraps a Snapshot with injections.

func (*InjectedSnapshot) BatchGet added in v1.0.8 

func (t *InjectedSnapshot) BatchGet(keys []Key) (map[string][]byte, error)

BatchGet returns an error if cfg.getError is set.

func (*InjectedSnapshot) Get 

func (t *InjectedSnapshot) Get(k Key) ([]byte, error)

Get returns an error if cfg.getError is set.

type InjectedStore 

type InjectedStore struct {
	Storage
	// contains filtered or unexported fields
}

InjectedStore wraps a Storage with injections.

func (*InjectedStore) Begin 

func (s *InjectedStore) Begin() (Transaction, error)

Begin creates an injected Transaction.

func (*InjectedStore) BeginWithStartTS 

func (s *InjectedStore) BeginWithStartTS(startTS uint64) (Transaction, error)

BeginWithStartTS creates an injected Transaction with startTS.

func (*InjectedStore) GetSnapshot 

func (s *InjectedStore) GetSnapshot(ver Version) (Snapshot, error)

GetSnapshot creates an injected Snapshot.

type InjectedTransaction 

type InjectedTransaction struct {
	Transaction
	// contains filtered or unexported fields
}

InjectedTransaction wraps a Transaction with injections.

func (*InjectedTransaction) Commit 

func (t *InjectedTransaction) Commit(ctx context.Context) error

Commit returns an error if cfg.commitError is set.

func (*InjectedTransaction) Get 

func (t *InjectedTransaction) Get(k Key) ([]byte, error)

Get returns an error if cfg.getError is set.

func (*InjectedTransaction) GetSnapshot added in v1.0.8 

func (t *InjectedTransaction) GetSnapshot() Snapshot

GetSnapshot implements Transaction GetSnapshot method.

type InjectionConfig 

type InjectionConfig struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

InjectionConfig is used for fault injections for KV components.

func (*InjectionConfig) SetCommitError 

func (c *InjectionConfig) SetCommitError(err error)

SetCommitError injects an error for all Transaction.Commit() methods.

func (*InjectionConfig) SetGetError 

func (c *InjectionConfig) SetGetError(err error)

SetGetError injects an error for all kv.Get() methods.

type IsoLevel 

type IsoLevel int

IsoLevel is the transaction's isolation level. 

const (
	// SI stands for 'snapshot isolation'.
	SI IsoLevel = iota
	// RC stands for 'read committed'.
	RC
)

type Iterator 

type Iterator interface {
	Valid() bool
	Key() Key
	Value() []byte
	Next() error
	Close()
}

Iterator is the interface for a iterator on KV store.

type Key 

type Key []byte

Key represents high-level Key type.

func (Key) Clone 

func (k Key) Clone() Key

Clone returns a copy of the Key.

func (Key) Cmp 

func (k Key) Cmp(another Key) int

Cmp returns the comparison result of two key. The result will be 0 if a==b, -1 if a < b, and +1 if a > b.

func (Key) HasPrefix 

func (k Key) HasPrefix(prefix Key) bool

HasPrefix tests whether the Key begins with prefix.

func (Key) Next 

func (k Key) Next() Key

Next returns the next key in byte-order.

func (Key) PrefixNext 

func (k Key) PrefixNext() Key

PrefixNext returns the next prefix key.

Assume there are keys like: 

rowkey1
rowkey1_column1
rowkey1_column2
rowKey2

If we seek 'rowkey1' Next, we will get 'rowkey1_column1'. If we seek 'rowkey1' PrefixNext, we will get 'rowkey2'.

type KeyRange 

type KeyRange struct {
	StartKey Key
	EndKey   Key
}

KeyRange represents a range where StartKey <= key < EndKey.

func (*KeyRange) IsPoint 

func (r *KeyRange) IsPoint() bool

IsPoint checks if the key range represents a point.

type MemBuffer 

type MemBuffer interface {
	RetrieverMutator
	// Size returns sum of keys and values length.
	Size() int
	// Len returns the number of entries in the DB.
	Len() int
	// Reset cleanup the MemBuffer
	Reset()
	// SetCap sets the MemBuffer capability, to reduce memory allocations.
	// Please call it before you use the MemBuffer, otherwise it will not works.
	SetCap(cap int)
}

MemBuffer is an in-memory kv collection, can be used to buffer write operations.

func NewMemDbBuffer 

func NewMemDbBuffer(cap int) MemBuffer

NewMemDbBuffer creates a new memDbBuffer.

type MockTxn 

type MockTxn interface {
	Transaction
	GetOption(opt Option) interface{}
}

MockTxn is used for test cases that need more interfaces than Transaction.

type Mutator 

type Mutator interface {
	// Set sets the value for key k as v into kv store.
	// v must NOT be nil or empty, otherwise it returns ErrCannotSetNilValue.
	Set(k Key, v []byte) error
	// Delete removes the entry for key k from kv store.
	Delete(k Key) error
}

Mutator is the interface wraps the basic Set and Delete methods.

type Option 

type Option int

Option is used for customizing kv store's behaviors during a transaction. 

const (
	// PresumeKeyNotExists indicates that when dealing with a Get operation but failing to read data from cache,
	// we presume that the key does not exist in Store. The actual existence will be checked before the
	// transaction's commit.
	// This option is an optimization for frequent checks during a transaction, e.g. batch inserts.
	PresumeKeyNotExists Option = iota + 1
	// PresumeKeyNotExistsError is the option key for error.
	// When PresumeKeyNotExists is set and condition is not match, should throw the error.
	PresumeKeyNotExistsError
	// BinlogInfo contains the binlog data and client.
	BinlogInfo
	// Skip existing check when "prewrite".
	SkipCheckForWrite
	// SchemaLeaseChecker is used for schema lease check.
	SchemaLeaseChecker
	// IsolationLevel sets isolation level for current transaction. The default level is SI.
	IsolationLevel
	// Priority marks the priority of this transaction.
	Priority
	// NotFillCache makes this request do not touch the LRU cache of the underlying storage.
	NotFillCache
	// SyncLog decides whether the WAL(write-ahead log) of this request should be synchronized.
	SyncLog
)

Transaction options

type Options 

type Options interface {
	// Get gets an option value.
	Get(opt Option) (v interface{}, ok bool)
}

Options is an interface of a set of options. Each option is associated with a value.

type Request 

type Request struct {
	// Tp is the request type.
	Tp        int64
	StartTs   uint64
	Data      []byte
	KeyRanges []KeyRange
	// KeepOrder is true, if the response should be returned in order.
	KeepOrder bool
	// Desc is true, if the request is sent in descending order.
	Desc bool
	// Concurrency is 1, if it only sends the request to a single storage unit when
	// ResponseIterator.Next is called. If concurrency is greater than 1, the request will be
	// sent to multiple storage units concurrently.
	Concurrency int
	// IsolationLevel is the isolation level, default is SI.
	IsolationLevel IsoLevel
	// Priority is the priority of this KV request, its value may be PriorityNormal/PriorityLow/PriorityHigh.
	Priority int
	// NotFillCache makes this request do not touch the LRU cache of the underlying storage.
	NotFillCache bool
	// SyncLog decides whether the WAL(write-ahead log) of this request should be synchronized.
	SyncLog bool
	// Streaming indicates using streaming API for this request, result in that one Next()
	// call would not corresponds to a whole region result.
	Streaming bool
}

Request represents a kv request.

type Response 

type Response interface {
	// Next returns a resultSubset from a single storage unit.
	// When full result set is returned, nil is returned.
	Next(ctx context.Context) (resultSubset ResultSubset, err error)
	// Close response.
	Close() error
}

Response represents the response returned from KV layer.

type ResultSubset 

type ResultSubset interface {
	// GetData gets the data.
	GetData() []byte
	// GetStartKey gets the start key.
	GetStartKey() Key
	// GetExecDetails gets the detail information.
	GetExecDetails() *execdetails.ExecDetails
}

ResultSubset represents a result subset from a single storage unit. TODO: Find a better interface for ResultSubset that can reuse bytes.

type Retriever 

type Retriever interface {
	// Get gets the value for key k from kv store.
	// If corresponding kv pair does not exist, it returns nil and ErrNotExist.
	Get(k Key) ([]byte, error)
	// Iter creates an Iterator positioned on the first entry that k <= entry's key.
	// If such entry is not found, it returns an invalid Iterator with no error.
	// It yields only keys that < upperBound. If upperBound is nil, it means the upperBound is unbounded.
	// The Iterator must be Closed after use.
	Iter(k Key, upperBound Key) (Iterator, error)

	// IterReverse creates a reversed Iterator positioned on the first entry which key is less than k.
	// The returned iterator will iterate from greater key to smaller key.
	// If k is nil, the returned iterator will be positioned at the last key.
	// TODO: Add lower bound limit
	IterReverse(k Key) (Iterator, error)
}

Retriever is the interface wraps the basic Get and Seek methods.

type RetrieverMutator 

type RetrieverMutator interface {
	Retriever
	Mutator
}

RetrieverMutator is the interface that groups Retriever and Mutator interfaces.

type Snapshot 

type Snapshot interface {
	Retriever
	// BatchGet gets a batch of values from snapshot.
	BatchGet(keys []Key) (map[string][]byte, error)
	// SetPriority snapshot set the priority
	SetPriority(priority int)
}

Snapshot defines the interface for the snapshot fetched from KV store.

type Storage 

type Storage interface {
	// Begin transaction
	Begin() (Transaction, error)
	// BeginWithStartTS begins transaction with startTS.
	BeginWithStartTS(startTS uint64) (Transaction, error)
	// GetSnapshot gets a snapshot that is able to read any data which data is <= ver.
	// if ver is MaxVersion or > current max committed version, we will use current version for this snapshot.
	GetSnapshot(ver Version) (Snapshot, error)
	// GetClient gets a client instance.
	GetClient() Client
	// Close store
	Close() error
	// UUID return a unique ID which represents a Storage.
	UUID() string
	// CurrentVersion returns current max committed version.
	CurrentVersion() (Version, error)
	// GetOracle gets a timestamp oracle client.
	GetOracle() oracle.Oracle
	// SupportDeleteRange gets the storage support delete range or not.
	SupportDeleteRange() (supported bool)
}

Storage defines the interface for storage. Isolation should be at least SI(SNAPSHOT ISOLATION)

func NewInjectedStore 

func NewInjectedStore(store Storage, cfg *InjectionConfig) Storage

NewInjectedStore creates a InjectedStore with config.

func NewMockStorage 

func NewMockStorage() Storage

NewMockStorage creates a new mockStorage.

type Transaction 

type Transaction interface {
	MemBuffer
	// Commit commits the transaction operations to KV store.
	Commit(context.Context) error
	// Rollback undoes the transaction operations to KV store.
	Rollback() error
	// String implements fmt.Stringer interface.
	String() string
	// LockKeys tries to lock the entries with the keys in KV store.
	LockKeys(keys ...Key) error
	// SetOption sets an option with a value, when val is nil, uses the default
	// value of this option.
	SetOption(opt Option, val interface{})
	// DelOption deletes an option.
	DelOption(opt Option)
	// IsReadOnly checks if the transaction has only performed read operations.
	IsReadOnly() bool
	// StartTS returns the transaction start timestamp.
	StartTS() uint64
	// Valid returns if the transaction is valid.
	// A transaction become invalid after commit or rollback.
	Valid() bool
	// GetMemBuffer return the MemBuffer binding to this transaction.
	GetMemBuffer() MemBuffer
	// GetSnapshot returns the snapshot of this transaction.
	GetSnapshot() Snapshot
}

Transaction defines the interface for operations inside a Transaction. This is not thread safe.

func NewMockTxn 

func NewMockTxn() Transaction

NewMockTxn new a mockTxn.

type UnionIter 

type UnionIter struct {
	// contains filtered or unexported fields
}

UnionIter is the iterator on an UnionStore.

func NewUnionIter 

func NewUnionIter(dirtyIt Iterator, snapshotIt Iterator, reverse bool) (*UnionIter, error)

NewUnionIter returns a union iterator for BufferStore.

func (*UnionIter) Close 

func (iter *UnionIter) Close()

Close implements the Iterator Close interface.

func (*UnionIter) Key 

func (iter *UnionIter) Key() Key

Key implements the Iterator Key interface.

func (*UnionIter) Next 

func (iter *UnionIter) Next() error

Next implements the Iterator Next interface.

func (*UnionIter) Valid 

func (iter *UnionIter) Valid() bool

Valid implements the Iterator Valid interface.

func (*UnionIter) Value 

func (iter *UnionIter) Value() []byte

Value implements the Iterator Value interface. Multi columns

type UnionStore 

type UnionStore interface {
	MemBuffer
	// CheckLazyConditionPairs loads all lazy values from store then checks if all values are matched.
	// Lazy condition pairs should be checked before transaction commit.
	CheckLazyConditionPairs() error
	// WalkBuffer iterates all buffered kv pairs.
	WalkBuffer(f func(k Key, v []byte) error) error
	// SetOption sets an option with a value, when val is nil, uses the default
	// value of this option.
	SetOption(opt Option, val interface{})
	// DelOption deletes an option.
	DelOption(opt Option)
	// GetOption gets an option.
	GetOption(opt Option) interface{}
	// GetMemBuffer return the MemBuffer binding to this UnionStore.
	GetMemBuffer() MemBuffer
}

UnionStore is a store that wraps a snapshot for read and a BufferStore for buffered write. Also, it provides some transaction related utilities.

func NewUnionStore 

func NewUnionStore(snapshot Snapshot) UnionStore

NewUnionStore builds a new UnionStore.

type Version 

type Version struct {
	Ver uint64
}

Version is the wrapper of KV's version.

func NewVersion 

func NewVersion(v uint64) Version

NewVersion creates a new Version struct.

func (Version) Cmp 

func (v Version) Cmp(another Version) int

Cmp returns the comparison result of two versions. The result will be 0 if a==b, -1 if a < b, and +1 if a > b.

type VersionProvider 

type VersionProvider interface {
	CurrentVersion() (Version, error)
}

VersionProvider provides increasing IDs.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.