kv

README

KeyValue "kv"

Simple key-value interface backed by multiple backends (multiple sql engines through gorm, pure golang through badger DB; cloud scale through google datastore).

This is not meant for the high performance applications but rather quick prototyping for cases where simple get/upsert/del/iterate in enough to get started. The different backends are meants for various scaling scenarios and deployment needs.

Status

WIP/Experiment. Should not be used. (limited tests, poor code quality, fragile interfaces, etc)

PRs are welcome.

Usage

import "github.com/zatte/kv"

func main(){
  ctx  := context.Background()
  // gorm / sqlite3 in memory, limited transactional support. Not great for testing
  // db, err := kv.New("sqlite3:///file%3A%3Amemory%3A%3Fcache%3Dshared&mode=rwc")

  // gorm / sqlite3 in memory, limited transactional support. Not great for testing
  // db, err := kv.New("psql://user:password@host:port/database")
  // db, err := kv.New("mysql", "mysql://user:password@/dbname?charset=utf8&parseTime=True&loc=Local")
  // db, err := kv.New("mssql", "mssql://username:password@localhost:1433?database=dbname")

  // Badger DB In memory
  // db, err := kv.New("badger:///?memory=true")

  // Badger DB by path
  // db, err := kv.New("badger:///./badger.testing.db")

  // datastore, local emulator for testing / development
  // db, err := kv.New("datastore://" + os.Getenv("DATASTORE_PROJECT_ID"))

  // datastore based on project id
  // db, err := kv.New("datastore://google-cloud-project-id")

  db, err := kv.New("badger:///?memory=true")

  // Put, Get, Del
  err := db.Put(ctx, []byte("key"), []byte("value"))
  key, err := db.Get(ctx, []byte("key"))
  err := db.Delete(ctx, []byte("key"))
  
  // Create a transaction
  tx, err := db.NewTransaction(ctx, false) // read only transactions. Not supported by all backends but some. 
  defer tx.Discard()

  // Same as above: Put, Get, Del
  err := tx.Put(ctx, []byte("key"), []byte("value"))
  key, err := tx.Get(ctx, []byte("key"))
  err := tx.Delete(ctx, []byte("key"))

  // but transactions can also be have iterators 
  it, err := tx.Seek(ctx, []byte("inclusvie_start_key_utf8_sort_order"))
  defer it.Close()

  for key, val, err := it.Next(); err == nil; key, val, err = it.Next() {
    // process keys and values in order.
  }
}

Testing

Doesn't perform integration testing with external databaes except datastore.

gcloud beta emulators datastore start & 
$(gcloud beta emulators datastore env-init)
go test

TODO

• Add Create (failure on existing keys)
• Add Backend Redis
• Improve errors (atm all errors are ErrNotFound)
• Improve docs
• Integration testing with
• • MySQL
• • Postgres
• • MsSQL

License

Copyright 2020 Mikael Rapp, github.com/zatte

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Documentation

Index

• Constants
• type BadgerDB
  • func NewBadgerDbFromUrl(u *url.URL) (*BadgerDB, error)
  • func NewbadgerFromDB(db *badger.DB) (*BadgerDB, error)
  • func (bdb *BadgerDB) Close() error
  • func (bdb *BadgerDB) Delete(ctx context.Context, key []byte) error
  • func (bdb *BadgerDB) Get(ctx context.Context, key []byte) (res []byte, err error)
  • func (bdb *BadgerDB) NewTransaction(ctx context.Context, readOnly bool) (OrderedTransaction, error)
  • func (bdb *BadgerDB) Put(ctx context.Context, key, value []byte) error
• type Basic
• type BasicTransaction
• type BasicTransactional
• type DatastoreDB
  • func NewDatastoreDbFromUrl(u *url.URL) (*DatastoreDB, error)
  • func (dsDb *DatastoreDB) Close() error
  • func (dsDb *DatastoreDB) Delete(ctx context.Context, key []byte) error
  • func (dsDb *DatastoreDB) Get(ctx context.Context, key []byte) (res []byte, err error)
  • func (dsDb *DatastoreDB) NewTransaction(ctx context.Context, readOnly bool) (OrderedTransaction, error)
  • func (dsDb *DatastoreDB) Put(ctx context.Context, key, value []byte) error
• type GormDB
  • func NewGormDbFromUrl(u *url.URL) (*GormDB, error)
  • func NewGormFromDB(db *gorm.DB) (*GormDB, error)
  • func (gdb *GormDB) Close() error
  • func (gdb *GormDB) Delete(ctx context.Context, key []byte) error
  • func (gdb *GormDB) Get(ctx context.Context, key []byte) ([]byte, error)
  • func (gdb *GormDB) NewTransaction(ctx context.Context, readOnly bool) (OrderedTransaction, error)
  • func (gdb *GormDB) Put(ctx context.Context, key, value []byte) error
• type GromKeyValue
• type Iterator
• type KvError
  • func (e KvError) Error() string
• type Ordered
• type OrderedTransaction
• type OrderedTransactional
  • func New(connectionString string) (OrderedTransactional, error)

Constants

const DataStoreKind = "keyvalue"

Types

type BadgerDB

type BadgerDB struct {
	*badger.DB
	// contains filtered or unexported fields
}

func NewBadgerDbFromUrl

func NewBadgerDbFromUrl(u *url.URL) (*BadgerDB, error)

func NewbadgerFromDB

func NewbadgerFromDB(db *badger.DB) (*BadgerDB, error)

func (*BadgerDB) Close

func (bdb *BadgerDB) Close() error

Get gets the value of a key within a single query transaction

func (*BadgerDB) Delete

func (bdb *BadgerDB) Delete(ctx context.Context, key []byte) error

Delete removes a key within a single transaction

func (*BadgerDB) Get

func (bdb *BadgerDB) Get(ctx context.Context, key []byte) (res []byte, err error)

Get gets the value of a key within a single query transaction

func (*BadgerDB) NewTransaction

func (bdb *BadgerDB) NewTransaction(ctx context.Context, readOnly bool) (OrderedTransaction, error)

NewTransaction for batching multiple values inside a transaction

func (*BadgerDB) Put

func (bdb *BadgerDB) Put(ctx context.Context, key, value []byte) error

Put sets the value of a key within a single query transaction

type Basic

type Basic interface {
	Get(ctx context.Context, Key []byte) ([]byte, error)
	Put(ctx context.Context, key, value []byte) error
	Delete(ctx context.Context, key []byte) error
}

Basic is the simplest version of a key/value store

type BasicTransaction

type BasicTransaction interface {
	Basic
	Discard(ctx context.Context) error
	Commit(ctx context.Context) error
}

BasicTransactional bundles all basic operations into an atomic operation. it is safe to call Discard or Commit multiples times; only the first call should be respected. BasicTransactional implementations is expected to be returned by a NewTransaction(readOnly bool) method of a KeyValue store

type BasicTransactional

type BasicTransactional interface {
	Basic
	NewTransaction(ctx context.Context, ReadOnly bool) (BasicTransaction, error)
}

type DatastoreDB

type DatastoreDB struct {
	*datastore.Client
	// contains filtered or unexported fields
}

func NewDatastoreDbFromUrl

func NewDatastoreDbFromUrl(u *url.URL) (*DatastoreDB, error)

func (*DatastoreDB) Close

func (dsDb *DatastoreDB) Close() error

Get gets the value of a key within a single query transaction

func (*DatastoreDB) Delete

func (dsDb *DatastoreDB) Delete(ctx context.Context, key []byte) error

Delete removes a key within a single transaction

func (*DatastoreDB) Get

func (dsDb *DatastoreDB) Get(ctx context.Context, key []byte) (res []byte, err error)

Get gets the value of a key within a single query transaction

func (*DatastoreDB) NewTransaction

func (dsDb *DatastoreDB) NewTransaction(ctx context.Context, readOnly bool) (OrderedTransaction, error)

NewTransaction for batching multiple values inside a transaction

func (*DatastoreDB) Put

func (dsDb *DatastoreDB) Put(ctx context.Context, key, value []byte) error

Put sets the value of a key within a single query transaction

type GormDB

type GormDB struct {
	*gorm.DB
}

func NewGormDbFromUrl

func NewGormDbFromUrl(u *url.URL) (*GormDB, error)

func NewGormFromDB

func NewGormFromDB(db *gorm.DB) (*GormDB, error)

func (*GormDB) Close

func (gdb *GormDB) Close() error

func (*GormDB) Delete

func (gdb *GormDB) Delete(ctx context.Context, key []byte) error

Delete removes a key within a single transaction

func (*GormDB) Get

func (gdb *GormDB) Get(ctx context.Context, key []byte) ([]byte, error)

Get gets the value of a key within a single query transaction

func (*GormDB) NewTransaction

func (gdb *GormDB) NewTransaction(ctx context.Context, readOnly bool) (OrderedTransaction, error)

NewTransaction for batching multiple values inside a transaction

func (*GormDB) Put

func (gdb *GormDB) Put(ctx context.Context, key, value []byte) error

Put sets the value of a key within a single query transaction

type GromKeyValue

type GromKeyValue struct {
	Key []byte `gorm:"primaryKey"`
	Val []byte
}

type Iterator

type Iterator interface {
	Next(ctx context.Context) (key, value []byte, err error)
	Close() error
}

Iterator scans a key space in a memory bound fashion. follow google design guidelines https://github.com/googleapis/google-cloud-go/wiki/Iterator-Guidelines

type KvError

type KvError string

const (
	ErrInvalidDb KvError = "no supported database type"
	ErrNotFound  KvError = "record not found"
)

func (KvError) Error

func (e KvError) Error() string

type Ordered

type Ordered interface {
	Basic
	Seek(ctx context.Context, StartKey []byte) (Iterator, error)
}

Ordered is an extention to the basic store by also providing scan methods. all scans must be byte-wise lexicographical sorting order.

type OrderedTransaction

type OrderedTransaction interface {
	Ordered
	Discard(ctx context.Context) error
	Commit(ctx context.Context) error
}

OrderedTransaction is an extention to the basic store by also providing scan methods. all scans must be byte-wise lexicographical sorting order. OrderedTransactional implementations is expected to be returned by a NewTransaction(readOnly bool) method of a KeyValue store

type OrderedTransactional

type OrderedTransactional interface {
	Basic
	NewTransaction(ctx context.Context, ReadOnly bool) (OrderedTransaction, error)
}

func New

func New(connectionString string) (OrderedTransactional, error)

New opens up a new db based on a connection string.