kvas

README

kvas

kvas is a minimal overhead key value store backed by the filesystem.

It comes with the following features:

• writes only happen for new data (verified with a SHA-256 hash);
• modifications are timestamped, allowing querying last modified by a timestamp;
• an in-memory index, allowing fast operations like confirming whether value set contains a key, enumerations;
• specific values in a set are stored individually as files;
• reductions - a slice of property values per key for fast in-memory operations like finding all keys that match specific value.
• list of reduction - a slice of reductions with additional fabric that establishes data relationships for transparent common operations (aliasing, joins).

Using kvas in your app

• Run go get github.com/boggydigital/kvas
• Import github.com/boggydigital/kvas
• For a key value set - connect to a local store with kvas.ConnectLocal(directory, extension)
  • Extensions supported: kvas.JsonExt (.json), kvas.GobExt (.gob)
• Use this key value set client to Get, Set, Cut values, as well as filter using CreatedAfter, ModifiedAfter, etc.

Key types provided by kvas

kvas comes with the following types:

• KeyValues - key value store backed by local filesystem
• ReduxValues - key reductions store (backed by kvas.KeyValues)
• ReduxAssets - collection of ReduxValues plus additional data relationship fabric

Example usage of kvas.KeyValues

NOTE: Error handling omitted for brevity.

lkv, _ := kvas.ConnectLocal(os.TempDir(), kvas.JsonExt)

key := "value1"
start := time.Now().Unix()

_ = lkv.Set(key, strings.NewReader(key))

if lkv.Has(key) {
    readCloser, _ := lkv.Get(key)
    defer readCloser.Close()
    // use readCloser to read data stored under "value1" key
}

fmt.Println(lkv.ModifiedAfter(start, false)) // prints: [value1]

Example usage of kvas.ReduxValues

NOTE: Error handling omitted for brevity.

rdx, _ := kvas.ConnectRedux(os.TempDir(), "titles")

for _, key := range rdx.Keys() {
    allKeyTitles, _ := rdx.GetAllValues(key)
    // process all titles for a key
}

// find all keys that have values containing "specific_title" (any case) 
matches := rdx.Match([]string{"specific_title"}, nil, true, true)

Example usage of kvas.ReduxAssets

NOTE: Error handling omitted for brevity.

rxa, _ := kvas.ConnectReduxAssets(os.TempDir(), nil, "titles", "country")

query := map[string][]string{
    "title": {"title1", "title2"},
    "country": {"states"},
}

// find all keys across "titles" and "countries" reductions that match query
matches := rxa.Match(query, true)

Documentation

Index

• Constants
• func Sha256(reader io.Reader) (string, error)
• func UnknownReduxAsset(asset string) error
• type IndexVetter
  • func NewReduxVetter(dir string, assets ...string) (IndexVetter, error)
• type KeyValues
  • func ConnectLocal(dir string, ext string) (KeyValues, error)
• type MatchOption
• type ReadableRedux
  • func NewReduxReader(dir string, assets ...string) (ReadableRedux, error)
  • func ReduxProxy(idpv map[string]map[string][]string) ReadableRedux
• type WriteableRedux
  • func NewReduxWriter(dir string, assets ...string) (WriteableRedux, error)

Constants

const (
	JsonExt = ".json"
	GobExt  = ".gob"
	HtmlExt = ".html"
	XmlExt  = ".xml"
)

const (
	CaseSensitive = iota
	FullMatch
)

Functions

func Sha256

func Sha256(reader io.Reader) (string, error)

Sha256 computes SHA-256 hash of a bytes slice

func UnknownReduxAsset

func UnknownReduxAsset(asset string) error

Types

type IndexVetter

type IndexVetter interface {
	VetIndexOnly(fix bool, tpw nod.TotalProgressWriter) ([]string, error)
	VetIndexMissing(fix bool, tpw nod.TotalProgressWriter) ([]string, error)
}

func NewReduxVetter

func NewReduxVetter(dir string, assets ...string) (IndexVetter, error)

type KeyValues

type KeyValues interface {
	Has(key string) bool
	Get(key string) (io.ReadCloser, error)
	GetFromStorage(key string) (io.ReadCloser, error)
	Set(key string, data io.Reader) error
	Cut(key string) (bool, error)

	Keys() []string
	CreatedAfter(timestamp int64) []string
	ModifiedAfter(timestamp int64, strictlyModified bool) []string
	IsModifiedAfter(key string, timestamp int64) bool

	IndexCurrentModTime() (int64, error)
	CurrentModTime(key string) (int64, error)
	IndexRefresh() error

	IndexVetter
}

func ConnectLocal

func ConnectLocal(dir string, ext string) (KeyValues, error)

type MatchOption

type MatchOption int

type ReadableRedux

type ReadableRedux interface {
	MustHave(assets ...string) error
	Keys(asset string) []string
	HasAsset(asset string) bool
	HasKey(asset, key string) bool
	HasValue(asset, key, val string) bool
	GetAllValues(asset, key string) ([]string, bool)
	GetFirstVal(asset, key string) (string, bool)
	GetLastVal(asset, key string) (string, bool)
	ModTime() (int64, error)
	RefreshReader() (ReadableRedux, error)
	MatchAsset(asset string, terms []string, scope []string, options ...MatchOption) []string
	Match(query map[string][]string, options ...MatchOption) []string
	Sort(ids []string, desc bool, sortBy ...string) ([]string, error)
	Export(w io.Writer, keys ...string) error
}

func NewReduxReader

func NewReduxReader(dir string, assets ...string) (ReadableRedux, error)

func ReduxProxy

func ReduxProxy(idpv map[string]map[string][]string) ReadableRedux

type WriteableRedux

type WriteableRedux interface {
	ReadableRedux
	AddValues(asset, key string, values ...string) error
	BatchAddValues(asset string, keyValues map[string][]string) error
	ReplaceValues(asset, key string, values ...string) error
	BatchReplaceValues(asset string, keyValues map[string][]string) error
	CutValues(asset, key string, values ...string) error
	CutKeys(asset string, keys ...string) error
	BatchCutValues(asset string, keyValues map[string][]string) error
	RefreshWriter() (WriteableRedux, error)
}

func NewReduxWriter

func NewReduxWriter(dir string, assets ...string) (WriteableRedux, error)