kv

package
v0.94.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 26, 2023 License: Apache-2.0 Imports: 22 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	InitialMigrateVersion = 1
	PathDelimiter         = "/"
	MetadataPartitionKey  = "kv-internal-metadata"
)

Variables

View Source 
var (
	ErrClosedEntries       = errors.New("closed entries")
	ErrConnectFailed       = errors.New("connect failed")
	ErrDriverConfiguration = errors.New("driver configuration")
	ErrMissingPartitionKey = errors.New("missing partition key")
	ErrMissingKey          = errors.New("missing key")
	ErrMissingValue        = errors.New("missing value")
	ErrNotFound            = errors.New("not found")
	ErrPredicateFailed     = errors.New("predicate failed")
	ErrSetupFailed         = errors.New("setup failed")
	ErrUnknownDriver       = errors.New("unknown driver")
	ErrTableNotActive      = errors.New("table not active")
)
View Source 
var ErrInvalidFormat = errors.New("invalid format")
View Source 
var ErrMigrationRequired = errors.New("wrong kv version")
View Source 
var ErrPatternAlreadyRegistered = errors.New("pattern already registered")
View Source 
var File_secondary_index_proto protoreflect.FileDescriptor
View Source 
var PrecondConditionalExists = Precond("ConditionalExists")

PrecondConditionalExists Conditional for SetIf which performs Set only if key already exists in store

Functions

func Drivers 

func Drivers() []string

Drivers returns a list of registered drive names

func FindMessageTypeRecord added in v0.87.0 

func FindMessageTypeRecord(partition, path string) protoreflect.MessageType

FindMessageTypeRecord lookup proto message type based on the partition and path. 

Can return nil in case the value is not matched

func FormatPath added in v0.65.0 

func FormatPath(p ...string) string

func GetDBSchemaVersion added in v0.66.0 

func GetDBSchemaVersion(ctx context.Context, store Store) (int, error)

GetDBSchemaVersion returns the current KV DB schema version

func Import added in v0.66.0 

func Import(ctx context.Context, reader io.Reader, store Store) error

func MustRegisterType added in v0.87.0 

func MustRegisterType(partitionPattern, pathPattern string, mt protoreflect.MessageType)

func Register 

func Register(name string, driver Driver)

Register 'driver' implementation under 'name'. Panic in case of empty name, nil driver or name already registered.

func RegisterType added in v0.87.0 

func RegisterType(partitionPattern, pathPattern string, mt protoreflect.MessageType) error

RegisterType - Register a pb message type to parse the data, according to a path regex All objects which match the path regex will be parsed as that type A nil type parses the value as a plain string

func SetDBSchemaVersion added in v0.70.1 

func SetDBSchemaVersion(ctx context.Context, store Store, version uint) error

SetDBSchemaVersion sets KV DB schema version

func SetMsg added in v0.92.0 

func SetMsg(ctx context.Context, s Store, partitionKey string, key []byte, msg protoreflect.ProtoMessage) error

func SetMsgIf added in v0.92.0 

func SetMsgIf(ctx context.Context, s Store, partitionKey string, key []byte, msg protoreflect.ProtoMessage, predicate Predicate) error

func UnregisterAllDrivers 

func UnregisterAllDrivers()

UnregisterAllDrivers remove all loaded drivers, used for test code.

func ValidateSchemaVersion added in v0.70.1 

func ValidateSchemaVersion(ctx context.Context, store Store) (int, error)

Types

type DatabaseMigrator added in v0.70.1 

type DatabaseMigrator struct {
	// contains filtered or unexported fields
}

func NewDatabaseMigrator added in v0.70.1 

func NewDatabaseMigrator(params kvparams.Config) *DatabaseMigrator

func (*DatabaseMigrator) Migrate added in v0.70.1 

func (d *DatabaseMigrator) Migrate(ctx context.Context) error

type Driver 

type Driver interface {
	// Open opens access to the database store. Implementations give access to the same storage based on the dsn.
	// Implementation can return the same Storage instance based on dsn or new one as long as it provides access to
	// the same storage.
	Open(ctx context.Context, params kvparams.Config) (Store, error)
}

Driver is the interface to access a kv database as a Store. Each kv provider implements a Driver.

type EntriesIterator 

type EntriesIterator interface {
	// Next should be called first before access Entry.
	// it will process the next entry and return true if it was successful, and false when none or error.
	Next() bool

	// Entry current entry read after calling Next, set to nil in case of an error or no more entries.
	Entry() *Entry

	// Err set to last error by reading or parse the next entry.
	Err() error

	// Close should be called at the end of processing entries, required to release resources used to scan entries.
	// After calling 'Close' the instance should not be used as the behaviour will not be defined.
	Close()
}

EntriesIterator used to enumerate over Scan results

func NewSkipIterator added in v0.68.0 

func NewSkipIterator(it EntriesIterator, after []byte) EntriesIterator

func ScanPrefix 

func ScanPrefix(ctx context.Context, store Store, partitionKey, prefix, after []byte) (EntriesIterator, error)

ScanPrefix returns an iterator on store that scan the set of keys that start with prefix after is the full key name for which to start the scan from

type Entry 

type Entry struct {
	PartitionKey []byte
	Key          []byte
	Value        []byte
}

Entry holds a pair of key/value

func (*Entry) String 

func (e *Entry) String() string
type Header struct {
	LakeFSVersion   string
	PackageName     string
	DBSchemaVersion int
	CreatedAt       time.Time
}

Header contains metadata information for import / export file

type IteratorOptions added in v0.69.1 

type IteratorOptions interface {
	// Start returns the starting point of the iterator
	Start() []byte

	// IncludeStart determines whether to include Start() value in the iterator
	IncludeStart() bool
}

IteratorOptions are the starting point options for PrimaryIterator

func IteratorOptionsAfter added in v0.69.1 

func IteratorOptionsAfter(start []byte) IteratorOptions

IteratorOptionsAfter - returns iterator options from that exclude the start key.

func IteratorOptionsFrom added in v0.69.1 

func IteratorOptionsFrom(start []byte) IteratorOptions

IteratorOptionsFrom - returns iterator options from that includes the start key, if exists.

type MatchRecord added in v0.87.0 

type MatchRecord struct {
	PartitionPattern string
	PathPattern      string
	MessageType      protoreflect.MessageType
}

type MessageEntry added in v0.66.0 

type MessageEntry struct {
	Key   []byte
	Value protoreflect.ProtoMessage
}

type MessageIterator added in v0.66.0 

type MessageIterator interface {
	Next() bool
	Entry() *MessageEntry
	Err() error
	Close()
}

type Migrator added in v0.70.1 

type Migrator interface {
	Migrate(ctx context.Context) error
}

type PartitionIterator added in v0.69.0 

type PartitionIterator struct {
	// contains filtered or unexported fields
}

PartitionIterator Used to scan through a whole partition

func NewPartitionIterator added in v0.69.0 

func NewPartitionIterator(ctx context.Context, store Store, msgType protoreflect.MessageType, partitionKey string, batchSize int) *PartitionIterator

func (*PartitionIterator) Close added in v0.69.0 

func (p *PartitionIterator) Close()

func (*PartitionIterator) Entry added in v0.69.0 

func (p *PartitionIterator) Entry() *MessageEntry

func (*PartitionIterator) Err added in v0.69.0 

func (p *PartitionIterator) Err() error

func (*PartitionIterator) Next added in v0.69.0 

func (p *PartitionIterator) Next() bool

func (*PartitionIterator) SeekGE added in v0.69.0 

func (p *PartitionIterator) SeekGE(key []byte)

type Precond added in v0.89.0 

type Precond string

Precond Type for special conditionals provided as predicates for the SetIf method

type Predicate added in v0.66.0 

type Predicate interface{}

Predicate value used to update a key base on a previous fetched value. 

Store's Get used to pull the key's value with the associated predicate.
Store's SetIf used to set the key's value based on the predicate.

func GetMsg added in v0.92.0 

func GetMsg(ctx context.Context, s Store, partitionKey string, key []byte, msg protoreflect.ProtoMessage) (Predicate, error)

type PrefixIterator 

type PrefixIterator struct {
	Iterator     EntriesIterator
	Prefix       []byte
	PartitionKey []byte
	// contains filtered or unexported fields
}

func (*PrefixIterator) Close 

func (b *PrefixIterator) Close()

func (*PrefixIterator) Entry 

func (b *PrefixIterator) Entry() *Entry

func (*PrefixIterator) Err 

func (b *PrefixIterator) Err() error

func (*PrefixIterator) Next 

func (b *PrefixIterator) Next() bool

type PrimaryIterator added in v0.67.0 

type PrimaryIterator struct {
	// contains filtered or unexported fields
}

PrimaryIterator MessageIterator implementation for primary key The iterator iterates over the given prefix and returns the proto message and key

func NewPrimaryIterator added in v0.67.0 

func NewPrimaryIterator(ctx context.Context, store Store, msgType protoreflect.MessageType, partitionKey string, prefix []byte, options IteratorOptions) (*PrimaryIterator, error)

NewPrimaryIterator creates a new PrimaryIterator by scanning the store for the given prefix under the partitionKey. See IteratorOptions for the starting point options.

func (*PrimaryIterator) Close added in v0.67.0 

func (i *PrimaryIterator) Close()

func (*PrimaryIterator) Entry added in v0.67.0 

func (i *PrimaryIterator) Entry() *MessageEntry

func (*PrimaryIterator) Err added in v0.67.0 

func (i *PrimaryIterator) Err() error

func (*PrimaryIterator) Next added in v0.67.0 

func (i *PrimaryIterator) Next() bool

type Record added in v0.87.0 

type Record struct {
	Partition string      `json:"partition"`
	Key       string      `json:"key"`
	Value     interface{} `json:"value"`
}

func NewRecord added in v0.87.0 

func NewRecord(partition, path string, rawValue []byte) (*Record, error)

type SafeEncoder added in v0.67.0 

type SafeEncoder struct {
	Je *json.Encoder
	Mu sync.Mutex
}

func (*SafeEncoder) Encode added in v0.67.0 

func (e *SafeEncoder) Encode(v interface{}) error

type ScanOptions added in v0.89.0 

type ScanOptions struct {
	// KeyStart start key to seek the scan
	KeyStart []byte
	// BatchSize used by each implementation to perform limited query or fetching of data while scanning.
	// The 0 value means - use the default by the implementation.
	BatchSize int
}

type SecondaryIndex added in v0.67.0 

type SecondaryIndex struct {
	PrimaryKey []byte `protobuf:"bytes,1,opt,name=primary_key,json=primaryKey,proto3" json:"primary_key,omitempty"`
	// contains filtered or unexported fields
}

message data model for secondary index

func (*SecondaryIndex) Descriptor deprecated added in v0.67.0 

func (*SecondaryIndex) Descriptor() ([]byte, []int)

Deprecated: Use SecondaryIndex.ProtoReflect.Descriptor instead.

func (*SecondaryIndex) GetPrimaryKey added in v0.67.0 

func (x *SecondaryIndex) GetPrimaryKey() []byte

func (*SecondaryIndex) ProtoMessage added in v0.67.0 

func (*SecondaryIndex) ProtoMessage()

func (*SecondaryIndex) ProtoReflect added in v0.67.0 

func (x *SecondaryIndex) ProtoReflect() protoreflect.Message

func (*SecondaryIndex) Reset added in v0.67.0 

func (x *SecondaryIndex) Reset()

func (*SecondaryIndex) String added in v0.67.0 

func (x *SecondaryIndex) String() string

type SecondaryIterator added in v0.67.0 

type SecondaryIterator struct {
	// contains filtered or unexported fields
}

SecondaryIterator MessageIterator implementation for secondary key The iterator iterates over the given prefix, extracts the primary key value from secondary key and then returns the proto message and primary key

func NewSecondaryIterator added in v0.67.0 

func NewSecondaryIterator(ctx context.Context, store Store, msgType protoreflect.MessageType, partitionKey string, prefix, after []byte) (*SecondaryIterator, error)

func (*SecondaryIterator) Close added in v0.67.0 

func (s *SecondaryIterator) Close()

func (*SecondaryIterator) Entry added in v0.67.0 

func (s *SecondaryIterator) Entry() *MessageEntry

func (*SecondaryIterator) Err added in v0.67.0 

func (s *SecondaryIterator) Err() error

func (*SecondaryIterator) Next added in v0.67.0 

func (s *SecondaryIterator) Next() bool

type SkipFirstIterator added in v0.68.0 

type SkipFirstIterator struct {
	// contains filtered or unexported fields
}

SkipFirstIterator will keep the behaviour of the given EntriesIterator, except for skipping the first Entry if its Key is equal to 'after'.

func (*SkipFirstIterator) Close added in v0.68.0 

func (si *SkipFirstIterator) Close()

func (*SkipFirstIterator) Entry added in v0.68.0 

func (si *SkipFirstIterator) Entry() *Entry

func (*SkipFirstIterator) Err added in v0.68.0 

func (si *SkipFirstIterator) Err() error

func (*SkipFirstIterator) Next added in v0.68.0 

func (si *SkipFirstIterator) Next() bool

type Store 

type Store interface {
	// Get returns a result containing the Value and Predicate for the given key, or ErrNotFound if key doesn't exist
	//   Predicate can be used for SetIf operation
	Get(ctx context.Context, partitionKey, key []byte) (*ValueWithPredicate, error)

	// Set stores the given value, overwriting an existing value if one exists
	Set(ctx context.Context, partitionKey, key, value []byte) error

	// SetIf returns an ErrPredicateFailed error if the key with valuePredicate passed
	//  doesn't match the currently stored value. SetIf is a simple compare-and-swap operator:
	//  valuePredicate is either the existing value, or nil for no previous key exists.
	//  this is intentionally simplistic: we can model a better abstraction on top, keeping this interface simple for implementors
	SetIf(ctx context.Context, partitionKey, key, value []byte, valuePredicate Predicate) error

	// Delete will delete the key, no error in if key doesn't exist
	Delete(ctx context.Context, partitionKey, key []byte) error

	// Scan returns entries that can be read by key order
	// partitionKey is optional, passing it might increase performance.
	// 'options' holds optional parameters to control the batch size and the key to start the scan with.
	Scan(ctx context.Context, partitionKey []byte, options ScanOptions) (EntriesIterator, error)

	// Close access to the database store. After calling Close the instance is unusable.
	Close()
}

func Open 

func Open(ctx context.Context, params kvparams.Config) (Store, error)

Open lookup driver by 'type' and return store based on the configuration. Failed with ErrUnknownDriver in case 'name' is not registered

type StoreLimiter added in v0.92.0 

type StoreLimiter struct {
	Store   Store
	Limiter ratelimit.Limiter
}

func NewStoreLimiter added in v0.92.0 

func NewStoreLimiter(s Store, l ratelimit.Limiter) *StoreLimiter

func (*StoreLimiter) Close added in v0.92.0 

func (s *StoreLimiter) Close()

func (*StoreLimiter) Delete added in v0.92.0 

func (s *StoreLimiter) Delete(ctx context.Context, partitionKey, key []byte) error

func (*StoreLimiter) Get added in v0.92.0 

func (s *StoreLimiter) Get(ctx context.Context, partitionKey, key []byte) (*ValueWithPredicate, error)

func (*StoreLimiter) Scan added in v0.92.0 

func (s *StoreLimiter) Scan(ctx context.Context, partitionKey []byte, options ScanOptions) (EntriesIterator, error)

func (*StoreLimiter) Set added in v0.92.0 

func (s *StoreLimiter) Set(ctx context.Context, partitionKey, key, value []byte) error

func (*StoreLimiter) SetIf added in v0.92.0 

func (s *StoreLimiter) SetIf(ctx context.Context, partitionKey, key, value []byte, valuePredicate Predicate) error

type StoreMetricsWrapper added in v0.84.0 

type StoreMetricsWrapper struct {
	Store
	StoreType string
}

StoreMetricsWrapper wraps any Store with metrics

func (*StoreMetricsWrapper) Close added in v0.84.0 

func (s *StoreMetricsWrapper) Close()

func (*StoreMetricsWrapper) Delete added in v0.84.0 

func (s *StoreMetricsWrapper) Delete(ctx context.Context, partitionKey, key []byte) error

func (*StoreMetricsWrapper) Get added in v0.84.0 

func (s *StoreMetricsWrapper) Get(ctx context.Context, partitionKey, key []byte) (*ValueWithPredicate, error)

func (*StoreMetricsWrapper) Scan added in v0.84.0 

func (s *StoreMetricsWrapper) Scan(ctx context.Context, partitionKey []byte, options ScanOptions) (EntriesIterator, error)

func (*StoreMetricsWrapper) Set added in v0.84.0 

func (s *StoreMetricsWrapper) Set(ctx context.Context, partitionKey, key, value []byte) error

func (*StoreMetricsWrapper) SetIf added in v0.84.0 

func (s *StoreMetricsWrapper) SetIf(ctx context.Context, partitionKey, key, value []byte, valuePredicate Predicate) error

type ValueWithPredicate added in v0.66.0 

type ValueWithPredicate struct {
	Value     []byte
	Predicate Predicate
}

ValueWithPredicate value with predicate - Value holds the data and Predicate a value used for conditional set. 

Get operation will return this struct, holding the key's information
SetIf operation will use the Predicate for conditional set

Source Files

View all Source files

Directories

Path Synopsis
Package mock is a generated GoMock package.
 Package mock is a generated GoMock package.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.