kvo

package
v0.3.14 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 10, 2025 License: MIT Imports: 7 Imported by: 0

Documentation

Overview

Package kvo (stands for Key Value Objects) manages trees of objects mainly represented as key-value pairs.

It has 3 goals:

1. Manipulation of data objects in a generic way (e.g. handling all integer ledger stats with the same code).

2. A compact binary data representation that can be read from a memory-mapped file without deserialization.

3. A diff/change format that can be used within a transaction (including to serve reads) and then quickly merged with the memory-mapped data when doing updates.

Potentially we also want to have a diff format that can be persisted long-term as part of change history. This might or might not match #3.

Index

Constants

View Source 
const TimeOffsetMicros = 62_135_596_800_000_000

TimeOffsetMicros is offset to Time.UnixMicro() that kvo stores, chosen such that time.Time{}.UnixMicro() = -TimeOffsetMicros. With this offset, 0 micros correspond to zero time instead of Unix epoch, and we can treat all times as unsigned integers.

Variables

View Source 
var (
	TInt64 = NewIntType[int64]("int64", func(fc *FmtContext, v int64) string {
		return strconv.FormatInt(v, 10)
	})
	TUint64 = NewIntType[uint64]("uint64", func(fc *FmtContext, v uint64) string {
		return strconv.FormatUint(v, 10)
	})
	TUint64Hex = NewIntType[uint64]("uint64_hex", func(fc *FmtContext, v uint64) string {
		return "0x" + strconv.FormatUint(v, 16)
	})
	TUnknownUint64 = NewIntType[uint64]("unknown_uint64", func(fc *FmtContext, v uint64) string {
		return strconv.FormatUint(v, 10)
	})
	TUknownKey = NewIntType[uint64]("unknown_key", func(fc *FmtContext, v uint64) string {
		return "0x" + strconv.FormatUint(v, 16)
	})

	TTime = NewScalarType[time.Time]("time", func(fc *FmtContext, v uint64) string {
		return Uint64ToTime(v).Format(time.RFC3339)
	})

	TBool = NewScalarType[bool]("bool", func(fc *FmtContext, v uint64) string {
		switch v {
		case 0:
			return "false"
		case 1:
			return "true"
		default:
			return fmt.Sprintf("?bool(0x%x)", v)
		}
	})
)

Functions

func Dump added in v0.3.1 

func Dump(m AnyMap) string

func Hex added in v0.3.3 

func Hex(v uint64) string

func HexString 

func HexString(data []uint64) string

func TimeToUint64 added in v0.3.1 

func TimeToUint64(t time.Time) uint64

func Uint64ToTime added in v0.3.1 

func Uint64ToTime(u uint64) time.Time

Types

type AnyMap 

type AnyMap interface {
	Type() AnyType
	IsMissing() bool
	Keys() []uint64
	Get(key uint64) uint64
	GetAnyMap(key uint64) AnyMap
	Packable() Packable
	Dump() string
}

type AnyRecord 

type AnyRecord interface {
	AnyRoot() AnyMap
	Packable
}

type AnyScalarType added in v0.3.4 

type AnyScalarType interface {
	AnyType
}

type AnyType 

type AnyType interface {
	Name() string
	String() string
	ValueKind() ValueKind
	Model() *Model
	ItemType() AnyType
	MapKeyType() AnyType
	MapProp(key uint64) PropImpl
	MapValueType(key uint64) AnyType
	Schema() *Schema // can return nil for generic types
	FormatValue(fc *FmtContext, value uint64) string
	// contains filtered or unexported methods
}

type EntityType added in v0.3.4 

type EntityType struct {
	// contains filtered or unexported fields
}

func NewEntityType 

func NewEntityType(schema *Schema, name string) *EntityType

func (*EntityType) FormatValue added in v0.3.4 

func (typ *EntityType) FormatValue(fc *FmtContext, value uint64) string

func (*EntityType) ItemType added in v0.3.4 

func (typ *EntityType) ItemType() AnyType

func (*EntityType) MapKeyType added in v0.3.4 

func (typ *EntityType) MapKeyType() AnyType

func (*EntityType) MapProp added in v0.3.4 

func (typ *EntityType) MapProp(key uint64) PropImpl

func (*EntityType) MapValueType added in v0.3.4 

func (typ *EntityType) MapValueType(key uint64) AnyType

func (*EntityType) Model added in v0.3.4 

func (typ *EntityType) Model() *Model

func (*EntityType) Name added in v0.3.4 

func (typ *EntityType) Name() string

func (*EntityType) Schema added in v0.3.4 

func (typ *EntityType) Schema() *Schema

func (*EntityType) String added in v0.3.5 

func (typ *EntityType) String() string

func (*EntityType) ValueKind added in v0.3.4 

func (typ *EntityType) ValueKind() ValueKind

type FloatValue 

type FloatValue interface {
	float32 | ~float64
}

type FmtContext added in v0.3.4 

type FmtContext struct {
}

type ImmutableMap 

type ImmutableMap struct {
	// contains filtered or unexported fields
}

func (ImmutableMap) Dump added in v0.3.1 

func (m ImmutableMap) Dump() string

func (ImmutableMap) Get 

func (m ImmutableMap) Get(key uint64) uint64

func (ImmutableMap) GetAnyMap 

func (m ImmutableMap) GetAnyMap(key uint64) AnyMap

func (ImmutableMap) GetMap added in v0.3.3 

func (m ImmutableMap) GetMap(key uint64) ImmutableMap

func (ImmutableMap) IsMissing 

func (m ImmutableMap) IsMissing() bool

func (ImmutableMap) Items added in v0.3.3 

func (m ImmutableMap) Items() func(yield func(k, v uint64) bool)

func (ImmutableMap) KeyModel 

func (m ImmutableMap) KeyModel(key uint64) AnyType

func (ImmutableMap) Keys added in v0.3.1 

func (m ImmutableMap) Keys() []uint64

func (ImmutableMap) Packable added in v0.3.4 

func (m ImmutableMap) Packable() Packable

func (ImmutableMap) RecordData added in v0.3.1 

func (m ImmutableMap) RecordData() ImmutableRecordData

func (ImmutableMap) RecordWithThisRoot added in v0.3.1 

func (m ImmutableMap) RecordWithThisRoot() ImmutableRecord

func (ImmutableMap) Type added in v0.3.4 

func (m ImmutableMap) Type() AnyType

type ImmutableObjectData 

type ImmutableObjectData []uint64

ImmutableObjectData represents a single object within ImmutableRecordData. It can be a map, an array or a set. Could even be a string or a blob, but we'd lack the exact size info, and those are better represented as []byte.

func (ImmutableObjectData) MapGet 

func (m ImmutableObjectData) MapGet(key uint64) uint64

func (ImmutableObjectData) MapKeys added in v0.3.1 

func (m ImmutableObjectData) MapKeys() []uint64

type ImmutableRecord 

type ImmutableRecord struct {
	// contains filtered or unexported fields
}

ImmutableRecord is a high-level interface to immutableRecordData, representing an immutable a tree of objects in a binary format suitable for storing on disk and optimized for random access.

func EmptyRecord 

func EmptyRecord(rootType AnyType) ImmutableRecord

func LoadRecord 

func LoadRecord(data []byte, rootType AnyType) ImmutableRecord

func (ImmutableRecord) AnyRoot 

func (r ImmutableRecord) AnyRoot() AnyMap

func (ImmutableRecord) Data 

func (r ImmutableRecord) Data() ImmutableRecordData

func (ImmutableRecord) Pack 

func (r ImmutableRecord) Pack() ImmutableRecordData

func (ImmutableRecord) Root 

func (r ImmutableRecord) Root() ImmutableMap

type ImmutableRecordData 

type ImmutableRecordData []uint64

ImmutableRecordData provides for random access to an immutable tree of objects stored in a binary format suitable for mmap'ing.

The format is as follows: 

 record -> flags:32 count:32 (objectKind:4 offset:28 size:32)... object...

 object -> map | arrayOrSet | stringOrBlob

	map -> key1 ... keyN valueOrRef1 ... valueOrRefN  (N determined by map size; keys are sorted)
	arrayOrSet -> valueOrRef1 ... valueOrRefN  (N is determined by array/set size; set elements are sorted)
	stringOrBlob is just raw bytes  (this is why object directory stores sizes in bytes)

 valueOrRef -> uint:64 | subobject:64

	subobject -> inlineStringOrData:64 | objectRef:64
	  inlineStringOrData -> alwaysOne:1 zeros:4 byteCount:3 inlineBytes:[7]byte
	  objectRef -> alwaysZero:1 zeroes:31 objectIndex:32

Note that only maps are implements so far.

func EmptyImmutableRecordData added in v0.3.4 

func EmptyImmutableRecordData() ImmutableRecordData

func LoadRecordData 

func LoadRecordData(data []byte) ImmutableRecordData

func (ImmutableRecordData) Bytes 

func (r ImmutableRecordData) Bytes() []byte

func (ImmutableRecordData) HexString 

func (r ImmutableRecordData) HexString() string

func (ImmutableRecordData) Object 

func (r ImmutableRecordData) Object(i int) ImmutableObjectData

func (ImmutableRecordData) ObjectCount 

func (r ImmutableRecordData) ObjectCount() int

func (ImmutableRecordData) Pack 

func (r ImmutableRecordData) Pack() ImmutableRecordData

func (ImmutableRecordData) Record 

func (data ImmutableRecordData) Record(rootType AnyType) ImmutableRecord

func (ImmutableRecordData) RootObject added in v0.3.1 

func (r ImmutableRecordData) RootObject() ImmutableObjectData

func (ImmutableRecordData) Uints 

func (r ImmutableRecordData) Uints() []uint64

type IntegerStringer added in v0.3.4 

type IntegerStringer interface {
	IntegerValue
	fmt.Stringer
}

type IntegerValue 

type IntegerValue interface {
	~int | ~int8 | ~int16 | ~int32 | ~int64 | ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr
}

type List 

type List struct {
	// contains filtered or unexported fields
}

TODO

type MapType added in v0.3.4 

type MapType struct {
	// contains filtered or unexported fields
}

func Map added in v0.3.4 

func Map(keyType AnyScalarType, itemType AnyType) *MapType

func (*MapType) FormatValue added in v0.3.4 

func (typ *MapType) FormatValue(fc *FmtContext, value uint64) string

func (*MapType) ItemType added in v0.3.4 

func (typ *MapType) ItemType() AnyType

func (*MapType) MapKeyType added in v0.3.4 

func (typ *MapType) MapKeyType() AnyType

func (*MapType) MapProp added in v0.3.4 

func (typ *MapType) MapProp(key uint64) PropImpl

func (*MapType) MapValueType added in v0.3.4 

func (typ *MapType) MapValueType(key uint64) AnyType

func (*MapType) Model added in v0.3.4 

func (typ *MapType) Model() *Model

func (*MapType) Name added in v0.3.4 

func (typ *MapType) Name() string

func (*MapType) Schema added in v0.3.4 

func (typ *MapType) Schema() *Schema

func (*MapType) String added in v0.3.5 

func (typ *MapType) String() string

func (*MapType) ValueKind added in v0.3.4 

func (typ *MapType) ValueKind() ValueKind

type Model 

type Model struct {
	// contains filtered or unexported fields
}

func NewModel 

func NewModel(schema *Schema, entityType *EntityType, build func(b *ModelBuilder)) *Model

func (*Model) MustPropByCode 

func (model *Model) MustPropByCode(key uint64) PropImpl

func (*Model) Name 

func (model *Model) Name() string

func (*Model) PropByCode added in v0.3.4 

func (model *Model) PropByCode(key uint64) PropImpl

func (*Model) Type added in v0.3.4 

func (model *Model) Type() AnyType

type ModelBuilder 

type ModelBuilder struct {
	// contains filtered or unexported fields
}

func (*ModelBuilder) Prop 

func (b *ModelBuilder) Prop(propCode PropCode, opts ...any)

type ModelDefinition 

type ModelDefinition struct {
	Name  string
	Props []*PropInstance
}

type MutableMap 

type MutableMap struct {
	// contains filtered or unexported fields
}

MutableMap allows mutating a map within a MutableRecord.

func NewRecord 

func NewRecord(rootType AnyType) MutableMap

NewRecord sets up an empty mutable record with the given root model and an empty root map, and returns the root map.

func UpdateRecord 

func UpdateRecord(orig ImmutableRecord) MutableMap

UpdateRecord sets up a mutable variant of the given record and returns its root map.

func (MutableMap) Dump added in v0.3.1 

func (m MutableMap) Dump() string

func (MutableMap) Get 

func (m MutableMap) Get(key uint64) uint64

func (MutableMap) GetAnyMap 

func (m MutableMap) GetAnyMap(key uint64) AnyMap

func (MutableMap) IsEmpty added in v0.3.1 

func (m MutableMap) IsEmpty() bool

func (MutableMap) IsMissing 

func (m MutableMap) IsMissing() bool

func (MutableMap) Keys added in v0.3.1 

func (m MutableMap) Keys() []uint64

func (MutableMap) Packable added in v0.3.4 

func (m MutableMap) Packable() Packable

func (MutableMap) Record 

func (m MutableMap) Record() *MutableRecord

func (MutableMap) Set 

func (m MutableMap) Set(key, value uint64)

func (MutableMap) Type added in v0.3.4 

func (m MutableMap) Type() AnyType

func (MutableMap) UpdateMap 

func (m MutableMap) UpdateMap(key uint64) MutableMap

type MutableRecord 

type MutableRecord struct {
	// contains filtered or unexported fields
}

MutableRecord is a builder of ImmutableRecordDatas. It represents a delta of a tree of objects in a way that's fast to write and reasonably fast to read.

(E.g. we're using unsorted keys, so reading performance could be improved by maybe sorting keys before reading.)

func (*MutableRecord) IsDirty added in v0.3.1 

func (rec *MutableRecord) IsDirty() bool

func (*MutableRecord) Original 

func (rec *MutableRecord) Original() ImmutableRecord

func (*MutableRecord) Pack 

func (rec *MutableRecord) Pack() ImmutableRecordData

Pack produces an on-disk binary encoding of the updated record, merging the changes recorded in MutableRecord with the original record.

func (*MutableRecord) PackedRecord added in v0.3.2 

func (rec *MutableRecord) PackedRecord() ImmutableRecord

func (*MutableRecord) PackedRoot added in v0.3.1 

func (rec *MutableRecord) PackedRoot() ImmutableMap

func (*MutableRecord) Root 

func (rec *MutableRecord) Root() MutableMap

type Packable 

type Packable interface {
	// Pack returns ImmutableRecordData of this record. Must return nil if
	// called on a nil struct pointer.
	Pack() ImmutableRecordData
}

type Prop added in v0.3.4 

type Prop struct {
	// contains filtered or unexported fields
}

func (*Prop) AnyType added in v0.3.4 

func (p *Prop) AnyType() AnyType

func (*Prop) Code added in v0.3.4 

func (p *Prop) Code() PropCode

func (*Prop) Name added in v0.3.4 

func (p *Prop) Name() string

func (*Prop) TypeModel added in v0.3.4 

func (p *Prop) TypeModel() *Model

func (*Prop) ValueKind added in v0.3.4 

func (p *Prop) ValueKind() ValueKind

type PropBuilder 

type PropBuilder struct{}

type PropCode 

type PropCode = uint64

func NewProp 

func NewProp(schema *Schema, code PropCode, name string, typ AnyType, build func(b *PropBuilder)) PropCode

type PropFlag 

type PropFlag int
const (
	PropDense PropFlag = 1 << iota
	PropRequired
)

type PropImpl 

type PropImpl interface {
	Name() string
	Code() PropCode
	AnyType() AnyType
	TypeModel() *Model
	ValueKind() ValueKind
}

type PropInstance 

type PropInstance struct {
	Prop     PropImpl
	Dense    bool
	Required bool
	// contains filtered or unexported fields
}

func (*PropInstance) PropInstance 

func (pi *PropInstance) PropInstance() *PropInstance

type PropOption 

type PropOption interface {
	// contains filtered or unexported methods
}

type ScalarConverter 

type ScalarConverter[T any] interface {
	ValueToScalar(value T) uint64
	ScalarToValue(scalar uint64) T
}

type Schema 

type Schema struct {
	TPropCode AnyType
	// contains filtered or unexported fields
}

func NewSchema added in v0.3.4 

func NewSchema() *Schema

func (*Schema) MustPropByCode 

func (schema *Schema) MustPropByCode(code PropCode) PropImpl

func (*Schema) PropByCode 

func (schema *Schema) PropByCode(code PropCode) PropImpl

type ValueKind added in v0.3.4 

type ValueKind int
const (
	ValueKindUnknown ValueKind = iota
	ValueKindWord
	ValueKindMap
	ValueKindScalarData
	ValueKindPIIData
)

type WordFormatter added in v0.3.4 

type WordFormatter = func(fc *FmtContext, v uint64) string

type WordType added in v0.3.4 

type WordType struct {
	// contains filtered or unexported fields
}

func NewFloatType 

func NewFloatType[T FloatValue](name string, formatter func(fc *FmtContext, v T) string) *WordType

func NewIntStringerType added in v0.3.4 

func NewIntStringerType[T IntegerStringer](name string) *WordType

func NewIntType 

func NewIntType[T IntegerValue](name string, formatter func(fc *FmtContext, v T) string) *WordType

func NewScalarSubtype 

func NewScalarSubtype[T any](name string, base *WordType) *WordType

func NewScalarType 

func NewScalarType[T any](name string, formatter func(fc *FmtContext, v uint64) string) *WordType

func NewUnknownTypeWithErrorCode added in v0.3.5 

func NewUnknownTypeWithErrorCode(errorCode string) *WordType

func (*WordType) FormatValue added in v0.3.4 

func (typ *WordType) FormatValue(fc *FmtContext, v uint64) string

func (*WordType) ItemType added in v0.3.4 

func (typ *WordType) ItemType() AnyType

func (*WordType) MapKeyType added in v0.3.4 

func (typ *WordType) MapKeyType() AnyType

func (*WordType) MapProp added in v0.3.4 

func (typ *WordType) MapProp(key uint64) PropImpl

func (*WordType) MapValueType added in v0.3.4 

func (typ *WordType) MapValueType(key uint64) AnyType

func (*WordType) Model added in v0.3.4 

func (typ *WordType) Model() *Model

func (*WordType) Name added in v0.3.4 

func (typ *WordType) Name() string

func (*WordType) Schema added in v0.3.4 

func (typ *WordType) Schema() *Schema

func (*WordType) String added in v0.3.5 

func (typ *WordType) String() string

func (*WordType) ValueKind added in v0.3.4 

func (typ *WordType) ValueKind() ValueKind

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.