badger

Documentation

Index

• Constants
• func GetCounterKey(ser *serial.T) (key []byte)
• func GetIndexKeysForEvent(ev *event.T, ser *serial.T) (keyz [][]byte)
• func GetTagKeyElements(tagValue string, CA *createdat.T, ser *serial.T) (prf index.P, elems []keys.Element, err error)
• func GetTagKeyPrefix(tagValue string) (key []byte, err error)
• func NewLogger(logLevel int, label string) (l *logger)
• func PrepareQueries(f *filter.T) (qs []query, ext *filter.T, since uint64, err error)
• type AccessEvent
  • func MakeAccessEvent(EvID eventid.T, Ser *serial.T) (ae *AccessEvent)
  • func (a AccessEvent) String() (s string)
• type Backend
  • func GetBackend(Ctx context.T, WG *sync.WaitGroup, path string, hasL2 bool, blockCacheSize int, ...) (b *Backend)
  • func (b *Backend) AccessLoop(c context.T, accCh chan *AccessEvent)
  • func (b *Backend) Close()
  • func (b *Backend) CountEvents(c context.T, f *filter.T) (count int, err error)
  • func (b *Backend) DeleteEvent(c context.T, ev *event.T) (err error)
  • func (b *Backend) GCCount() (unpruned, pruned count.Items, unprunedTotal, prunedTotal int, err error)
  • func (b *Backend) GCMark() (pruneEvents, pruneIndexes DelItems, err error)
  • func (b *Backend) GCRun() (err error)
  • func (b *Backend) GCSweep(evs, idxs DelItems) (err error)
  • func (b *Backend) GarbageCollector()
  • func (b *Backend) GetEventHeadroom() (hw, lw int)
  • func (b *Backend) GetIndexHeadroom() (hw, lw int)
  • func (b *Backend) IncrementAccesses(acc *AccessEvent) (err error)
  • func (b *Backend) Init() (err error)
  • func (b *Backend) QueryEvents(c context.T, f *filter.T) (ch event.C, err error)
  • func (b *Backend) SaveEvent(c context.T, ev *event.T) (err error)
  • func (b *Backend) Serial() (ser uint64, err error)
  • func (b *Backend) SerialBytes() (ser []byte, err error)
  • func (b *Backend) SerialKey() (idx []byte, ser *serial.T)
  • func (b *Backend) Update(fn func(txn *badger.Txn) (err error)) (err error)
  • func (b *Backend) View(fn func(txn *badger.Txn) (err error)) (err error)
  • func (b *Backend) Wipe() (err error)
• type DelItems
• type GCCountFunc
• type PruneFunc
• type Results

Constants

const CounterLen = KeyLen + createdat.Len

const DefaultMaxLimit = 1024

const KeyLen = serial.Len + 1

const PrunedLen = sha256.Size + KeyLen

Functions

func GetCounterKey

func GetCounterKey(ser *serial.T) (key []byte)

GetCounterKey returns the proper counter key for a given event ID.

func GetIndexKeysForEvent

func GetIndexKeysForEvent(ev *event.T, ser *serial.T) (keyz [][]byte)

GetIndexKeysForEvent generates all the index keys required to filter for events. evtSerial should be the output of Serial() which gets a unique, monotonic counter value for each new event.

func GetTagKeyElements

func GetTagKeyElements(tagValue string, CA *createdat.T, ser *serial.T) (prf index.P,
	elems []keys.Element, err error)

func GetTagKeyPrefix

func GetTagKeyPrefix(tagValue string) (key []byte, err error)

GetTagKeyPrefix returns tag index prefixes based on the initial field of a tag.

There is 3 types of index tag keys:

- TagAddr: [ 8 ][ 2b Kind ][ 8b Pubkey ][ address/URL ][ 8b Serial ] - Tag32: [ 7 ][ 8b Pubkey ][ 8b Serial ] - Tag: [ 6 ][ address/URL ][ 8b Serial ]

This function produces the initial bytes without the index.

func NewLogger

func NewLogger(logLevel int, label string) (l *logger)

func PrepareQueries

func PrepareQueries(f *filter.T) (
	qs []query,
	ext *filter.T,
	since uint64,
	err error,
)

PrepareQueries analyses a filter and generates a set of query specs that produce key prefixes to search for in the badger key indexes.

Types

type AccessEvent

type AccessEvent struct {
	EvID eventid.T
	Ts   timestamp.T
	Ser  *serial.T
}

func MakeAccessEvent

func MakeAccessEvent(EvID eventid.T, Ser *serial.T) (ae *AccessEvent)

MakeAccessEvent generates an *AccessEvent from an event ID and serial.

func (AccessEvent) String

func (a AccessEvent) String() (s string)

type Backend

type Backend struct {
	Ctx  context.T
	WG   *sync.WaitGroup
	Path string
	// MaxLimit is the largest a single event JSON can be, in bytes.
	MaxLimit int
	// DBSizeLimit is the number of bytes we want to keep the data store from
	// exceeding.
	DBSizeLimit int
	// DBLowWater is the percentage of DBSizeLimit a GC run will reduce the used
	// storage down to.
	DBLowWater int
	// DBHighWater is the trigger point at which a GC run should start if
	// exceeded.
	DBHighWater int
	// GCFrequency is the frequency of checks of the current utilisation.
	GCFrequency    time.Duration
	HasL2          bool
	BlockCacheSize int
	InitLogLevel   int
	Logger         *logger
	// DB is the badger db interface
	*badger.DB
	// contains filtered or unexported fields
}

func GetBackend

func GetBackend(
	Ctx context.T,
	WG *sync.WaitGroup,
	path string,
	hasL2 bool,
	blockCacheSize int,
	params ...int,
) (b *Backend)

GetBackend returns a reasonably configured badger.Backend.

The variadic params correspond to DBSizeLimit, DBLowWater, DBHighWater and GCFrequency as an integer multiplier of number of seconds.

Note that the cancel function for the context needs to be managed by the caller.

func (*Backend) AccessLoop

func (b *Backend) AccessLoop(c context.T, accCh chan *AccessEvent)

AccessLoop is meant to be run as a goroutine to gather access events in a query and when it finishes, bump all the access records

func (*Backend) Close

func (b *Backend) Close()

func (*Backend) CountEvents

func (b *Backend) CountEvents(c context.T, f *filter.T) (count int, err error)

func (*Backend) DeleteEvent

func (b *Backend) DeleteEvent(c context.T, ev *event.T) (err error)

func (*Backend) GCCount

func (b *Backend) GCCount() (unpruned, pruned count.Items, unprunedTotal,
	prunedTotal int, err error)

GCCount performs a census of events in the event store. It counts the number of events and their size, and if there is a layer 2 enabled, it counts the number of events that have been pruned and thus have indexes to count.

Both operations are more efficient combined together rather than separated, thus this is a fairly long function.

func (*Backend) GCMark

func (b *Backend) GCMark() (pruneEvents, pruneIndexes DelItems, err error)

GCMark first gathers the serial, data size and last accessed information about all events and pruned events using GCCount then sorts the results of the events and indexes by least recently accessed and generates the set of serials of events that need to be deleted

func (*Backend) GCRun

func (b *Backend) GCRun() (err error)

func (*Backend) GCSweep

func (b *Backend) GCSweep(evs, idxs DelItems) (err error)

func (*Backend) GarbageCollector

func (b *Backend) GarbageCollector()

GarbageCollector starts up a ticker that runs a check on space utilisation and when it exceeds the high-water mark, prunes back to the low-water mark.

This function should be invoked as a goroutine, and will terminate when the backend context is canceled.

func (*Backend) GetEventHeadroom

func (b *Backend) GetEventHeadroom() (hw, lw int)

func (*Backend) GetIndexHeadroom

func (b *Backend) GetIndexHeadroom() (hw, lw int)

func (*Backend) IncrementAccesses

func (b *Backend) IncrementAccesses(acc *AccessEvent) (err error)

IncrementAccesses takes a list of event IDs of events that were accessed in a query and updates their access counter records.

func (*Backend) Init

func (b *Backend) Init() (err error)

func (*Backend) QueryEvents

func (b *Backend) QueryEvents(c context.T, f *filter.T) (ch event.C, err error)

func (*Backend) SaveEvent

func (b *Backend) SaveEvent(c context.T, ev *event.T) (err error)

func (*Backend) Serial

func (b *Backend) Serial() (ser uint64, err error)

func (*Backend) SerialBytes

func (b *Backend) SerialBytes() (ser []byte, err error)

SerialBytes returns a new serial value, used to store an event record with a conflict-free unique code (it is a monotonic, atomic, ascending counter).

func (*Backend) SerialKey

func (b *Backend) SerialKey() (idx []byte, ser *serial.T)

SerialKey returns a key used for storing events, and the raw serial counter bytes to copy into index keys.

func (*Backend) Update

func (b *Backend) Update(fn func(txn *badger.Txn) (err error)) (err error)

func (*Backend) View

func (b *Backend) View(fn func(txn *badger.Txn) (err error)) (err error)

func (*Backend) Wipe

func (b *Backend) Wipe() (err error)

type DelItems

type DelItems []uint64

type GCCountFunc

type GCCountFunc func(ifc any) (deleteItems del.Items, err error)

type PruneFunc

type PruneFunc func(ifc any, deleteItems del.Items) (err error)

type Results

type Results struct {
	Ev  *event.T
	TS  timestamp.T
	Ser *serial.T
}