ratel

Documentation

Index

• Constants
• func GetCounterKey(ser *serial.T) (key by)
• func GetIndexKeysForEvent(ev *event.T, ser *serial.T) (keyz []by)
• func GetTagKeyElements(tagKey, tagValue st, CA *createdat.T, ser *serial.T) (prf index.P, elems []keys.Element, err er)
• func GetTagKeyPrefix(tagValue string) (key by, err er)
• func NewLogger(logLevel no, label string) (l *logger)
• func PrepareQueries(f *filter.T) (qs []query, ext *filter.T, since uint64, err er)
• type BackendParams
• type DelItems
• type Results
• type T
  • func GetBackend(Ctx cx, WG *sync.WaitGroup, hasL2, useCompact bo, ...) (b *T)
  • func New(p BackendParams, params ...no) *T
  • func (r *T) Close() (err er)
  • func (r *T) CountEvents(c cx, f *filter.T) (count no, approx bo, err er)
  • func (r *T) DeleteEvent(c cx, eid *eventid.T) (err er)
  • func (r *T) Export(c cx, w io.Writer, pubkeys ...by)
  • func (r *T) GCCount() (unpruned, pruned count.Items, unprunedTotal, prunedTotal no, err er)
  • func (r *T) GCMark() (pruneEvents, pruneIndexes DelItems, err er)
  • func (r *T) GCRun() (err er)
  • func (r *T) GCSweep(evs, idxs DelItems) (err er)
  • func (r *T) GarbageCollector()
  • func (r *T) GetEventHeadroom() (hw, lw no)
  • func (r *T) GetIndexHeadroom() (hw, lw no)
  • func (r *T) Import(rr io.Reader)
  • func (r *T) Init(path st) (err er)
  • func (r *T) Marshal(ev *event.T, dst by) (b by)
  • func (r *T) Nuke() (err er)
  • func (r *T) Path() st
  • func (r *T) QueryEvents(c cx, f *filter.T) (evs event.Ts, err er)
  • func (r *T) SaveEvent(c cx, ev *event.T) (err er)
  • func (r *T) Serial() (ser uint64, err er)
  • func (r *T) SerialBytes() (ser by, err er)
  • func (r *T) SerialKey() (idx by, ser *serial.T)
  • func (r *T) Sync() (err er)
  • func (r *T) Unmarshal(ev *event.T, evb by) (rem by, err er)

Constants

const CounterLen = KeyLen + createdat.Len

const DefaultMaxLimit = 512

const KeyLen = serial.Len + 1

const PrunedLen = sha256.Size + KeyLen

const Version = 1

Functions

func GetCounterKey

func GetCounterKey(ser *serial.T) (key by)

GetCounterKey returns the proper counter key for a given event ID.

func GetIndexKeysForEvent

func GetIndexKeysForEvent(ev *event.T, ser *serial.T) (keyz []by)

GetIndexKeysForEvent generates all the index keys required to filter for events. evtSerial should be the output of Serial() which gets a unique, monotonic counter value for each new event.

func GetTagKeyElements

func GetTagKeyElements(tagKey, tagValue st, CA *createdat.T,
	ser *serial.T) (prf index.P,
	elems []keys.Element, err er)

func GetTagKeyPrefix

func GetTagKeyPrefix(tagValue string) (key by, err er)

GetTagKeyPrefix returns tag index prefixes based on the initial field of a tag.

There is 3 types of index tag keys:

- TagAddr: [ 8 ][ 2b Kind ][ 8b Pubkey ][ address/URL ][ 8b Serial ]

- Tag32: [ 7 ][ 8b Pubkey ][ 8b Serial ]

- Tag: [ 6 ][ address/URL ][ 8b Serial ]

This function produces the initial bytes without the index.

func NewLogger

func NewLogger(logLevel no, label string) (l *logger)

func PrepareQueries

func PrepareQueries(f *filter.T) (
	qs []query,
	ext *filter.T,
	since uint64,
	err er,
)

PrepareQueries analyses a filter and generates a set of query specs that produce key prefixes to search for in the badger key indexes.

Types

type BackendParams

type BackendParams struct {
	Ctx                                cx
	WG                                 *sync.WaitGroup
	HasL2, UseCompact                  bo
	BlockCacheSize, LogLevel, MaxLimit no
	Compression                        st // none,snappy,zstd
	Extra                              []no
}

type DelItems

type DelItems []uint64

type Results

type Results struct {
	Ev  *event.T
	TS  *timestamp.T
	Ser *serial.T
}

type T

type T struct {
	Ctx cx
	WG  *sync.WaitGroup

	// DBSizeLimit is the number of bytes we want to keep the data store from exceeding.
	DBSizeLimit no
	// DBLowWater is the percentage of DBSizeLimit a GC run will reduce the used storage down
	// to.
	DBLowWater no
	// DBHighWater is the trigger point at which a GC run should start if exceeded.
	DBHighWater no
	// GCFrequency is the frequency of checks of the current utilisation.
	GCFrequency    time.Duration
	HasL2          bo
	BlockCacheSize no
	InitLogLevel   no
	Logger         *logger
	// DB is the badger db
	*badger.DB

	// Threads is how many CPU threads we dedicate to concurrent actions, flatten and GC mark
	Threads no
	// MaxLimit is a default limit that applies to a query without a limit, to avoid sending out
	// too many events to a client from a malformed or excessively broad filter.
	MaxLimit no
	// ActuallyDelete sets whether we actually delete or rewrite deleted entries with a modified
	// deleted prefix value (8th bit set)
	ActuallyDelete bo
	// Flatten should be set to true to trigger a flatten at close... this is mainly
	// triggered by running an import
	Flatten bo
	// UseCompact uses a compact encoding based on the canonical format (generate
	// hash of it to get ID field with the signature in raw binary after.
	UseCompact bo
	// Compression sets the compression to use, none/snappy/zstd
	Compression st
	// contains filtered or unexported fields
}

func GetBackend

func GetBackend(Ctx cx, WG *sync.WaitGroup, hasL2, useCompact bo,
	blockCacheSize, logLevel, maxLimit no, compression st, params ...no) (b *T)

GetBackend returns a reasonably configured badger.Backend.

The variadic params correspond to DBSizeLimit, DBLowWater, DBHighWater and GCFrequency as an integer multiplier of number of seconds.

Note that the cancel function for the context needs to be managed by the caller.

Deprecated: use New instead.

func New

func New(p BackendParams, params ...no) *T

func (*T) Close

func (r *T) Close() (err er)

func (*T) CountEvents

func (r *T) CountEvents(c cx, f *filter.T) (count no, approx bo, err er)

func (*T) DeleteEvent

func (r *T) DeleteEvent(c cx, eid *eventid.T) (err er)

func (*T) Export

func (r *T) Export(c cx, w io.Writer, pubkeys ...by)

func (*T) GCCount

func (r *T) GCCount() (unpruned, pruned count.Items, unprunedTotal,
	prunedTotal no, err er)

GCCount performs a census of events in the event store. It counts the number of events and their size, and if there is a layer 2 enabled, it counts the number of events that have been pruned and thus have indexes to count.

Both operations are more efficient combined together rather than separated, thus this is a fairly long function.

func (*T) GCMark

func (r *T) GCMark() (pruneEvents, pruneIndexes DelItems, err er)

GCMark first gathers the serial, data size and last accessed information about all events and pruned events using GCCount then sorts the results of the events and indexes by least recently accessed and generates the set of serials of events that need to be deleted

func (*T) GCRun

func (r *T) GCRun() (err er)

func (*T) GCSweep

func (r *T) GCSweep(evs, idxs DelItems) (err er)

func (*T) GarbageCollector

func (r *T) GarbageCollector()

GarbageCollector starts up a ticker that runs a check on space utilisation and when it exceeds the high-water mark, prunes back to the low-water mark.

This function should be invoked as a goroutine, and will terminate when the backend context is canceled.

func (*T) GetEventHeadroom

func (r *T) GetEventHeadroom() (hw, lw no)

func (*T) GetIndexHeadroom

func (r *T) GetIndexHeadroom() (hw, lw no)

func (*T) Import

func (r *T) Import(rr io.Reader)

Import accepts an event

func (*T) Init

func (r *T) Init(path st) (err er)

func (*T) Marshal

func (r *T) Marshal(ev *event.T, dst by) (b by)

func (*T) Nuke

func (r *T) Nuke() (err er)

func (*T) Path

func (r *T) Path() st

func (*T) QueryEvents

func (r *T) QueryEvents(c cx, f *filter.T) (evs event.Ts, err er)

func (*T) SaveEvent

func (r *T) SaveEvent(c cx, ev *event.T) (err er)

func (*T) Serial

func (r *T) Serial() (ser uint64, err er)

Serial returns the next monotonic conflict free unique serial on the database.

func (*T) SerialBytes

func (r *T) SerialBytes() (ser by, err er)

SerialBytes returns a new serial value, used to store an event record with a conflict-free unique code (it is a monotonic, atomic, ascending counter).

func (*T) SerialKey

func (r *T) SerialKey() (idx by, ser *serial.T)

SerialKey returns a key used for storing events, and the raw serial counter bytes to copy into index keys.

func (*T) Sync

func (r *T) Sync() (err er)

func (*T) Unmarshal

func (r *T) Unmarshal(ev *event.T, evb by) (rem by, err er)