database

Documentation

Index

• Constants
• Variables
• type Callbacks
• type Capabilities
• type CollectionName
• type HashCollectionNS
• type OrderedCollectionNS
• type OrderedUUIDCollectionNS
• type OtherCollection
• type PersistenceInterface
• type Plugin
• type PostCompletionHook
• type UUIDCollectionNS
• type UpsertOptimization

Constants

const (
	// Pseudo-namespace to register a global callback handler, which will receive all namespaced and non-namespaced events
	GlobalHandler = "ff:global"
)

Variables

var (
	// HashMismatch sentinel error
	HashMismatch = i18n.NewError(context.Background(), coremsgs.MsgHashMismatch)
	// IDMismatch sentinel error
	IDMismatch = i18n.NewError(context.Background(), coremsgs.MsgIDMismatch)
	// DeleteRecordNotFound sentinel error
	DeleteRecordNotFound = i18n.NewError(context.Background(), coremsgs.Msg404NotFound)
)

var BatchQueryFactory = &ffapi.QueryFields{
	"id":         &ffapi.UUIDField{},
	"type":       &ffapi.StringField{},
	"author":     &ffapi.StringField{},
	"key":        &ffapi.StringField{},
	"group":      &ffapi.Bytes32Field{},
	"hash":       &ffapi.Bytes32Field{},
	"payloadref": &ffapi.StringField{},
	"created":    &ffapi.TimeField{},
	"confirmed":  &ffapi.TimeField{},
	"tx.type":    &ffapi.StringField{},
	"tx.id":      &ffapi.UUIDField{},
	"node":       &ffapi.UUIDField{},
}

BatchQueryFactory filter fields for batches

var BlobQueryFactory = &ffapi.QueryFields{
	"hash":       &ffapi.Bytes32Field{},
	"size":       &ffapi.Int64Field{},
	"payloadref": &ffapi.StringField{},
	"created":    &ffapi.TimeField{},
	"data_id":    &ffapi.UUIDField{},
}

BlobQueryFactory filter fields for config records

var BlockchainEventQueryFactory = &ffapi.QueryFields{
	"id":              &ffapi.UUIDField{},
	"source":          &ffapi.StringField{},
	"name":            &ffapi.StringField{},
	"protocolid":      &ffapi.StringField{},
	"listener":        &ffapi.StringField{},
	"tx.type":         &ffapi.StringField{},
	"tx.id":           &ffapi.UUIDField{},
	"tx.blockchainid": &ffapi.StringField{},
	"timestamp":       &ffapi.TimeField{},
}

BlockchainEventQueryFactory filter fields for contract events

var ContractAPIQueryFactory = &ffapi.QueryFields{
	"id":        &ffapi.UUIDField{},
	"name":      &ffapi.StringField{},
	"interface": &ffapi.UUIDField{},
}

ContractAPIQueryFactory filter fields for Contract APIs

var ContractListenerQueryFactory = &ffapi.QueryFields{
	"id":        &ffapi.UUIDField{},
	"name":      &ffapi.StringField{},
	"interface": &ffapi.UUIDField{},
	"location":  &ffapi.JSONField{},
	"topic":     &ffapi.StringField{},
	"signature": &ffapi.StringField{},
	"backendid": &ffapi.StringField{},
	"created":   &ffapi.TimeField{},
	"updated":   &ffapi.TimeField{},
	"state":     &ffapi.JSONField{},
}

ContractListenerQueryFactory filter fields for contract listeners

var DataQueryFactory = &ffapi.QueryFields{
	"id":               &ffapi.UUIDField{},
	"validator":        &ffapi.StringField{},
	"datatype.name":    &ffapi.StringField{},
	"datatype.version": &ffapi.StringField{},
	"hash":             &ffapi.Bytes32Field{},
	"blob.hash":        &ffapi.Bytes32Field{},
	"blob.public":      &ffapi.StringField{},
	"blob.name":        &ffapi.StringField{},
	"blob.size":        &ffapi.Int64Field{},
	"created":          &ffapi.TimeField{},
	"value":            &ffapi.JSONField{},
	"public":           &ffapi.StringField{},
}

DataQueryFactory filter fields for data

var DatatypeQueryFactory = &ffapi.QueryFields{
	"id":        &ffapi.UUIDField{},
	"message":   &ffapi.UUIDField{},
	"validator": &ffapi.StringField{},
	"name":      &ffapi.StringField{},
	"version":   &ffapi.StringField{},
	"created":   &ffapi.TimeField{},
}

DatatypeQueryFactory filter fields for data definitions

var EventQueryFactory = &ffapi.QueryFields{
	"id":         &ffapi.UUIDField{},
	"type":       &ffapi.StringField{},
	"reference":  &ffapi.UUIDField{},
	"correlator": &ffapi.UUIDField{},
	"tx":         &ffapi.UUIDField{},
	"topic":      &ffapi.StringField{},
	"sequence":   &ffapi.Int64Field{},
	"created":    &ffapi.TimeField{},
}

EventQueryFactory filter fields for data events

var FFIErrorQueryFactory = &ffapi.QueryFields{
	"id":          &ffapi.UUIDField{},
	"name":        &ffapi.StringField{},
	"pathname":    &ffapi.StringField{},
	"interface":   &ffapi.UUIDField{},
	"description": &ffapi.StringField{},
}

FFIErrorQueryFactory filter fields for contract errors

var FFIEventQueryFactory = &ffapi.QueryFields{
	"id":          &ffapi.UUIDField{},
	"name":        &ffapi.StringField{},
	"pathname":    &ffapi.StringField{},
	"interface":   &ffapi.UUIDField{},
	"description": &ffapi.StringField{},
}

FFIEventQueryFactory filter fields for contract events

var FFIMethodQueryFactory = &ffapi.QueryFields{
	"id":          &ffapi.UUIDField{},
	"name":        &ffapi.StringField{},
	"pathname":    &ffapi.StringField{},
	"interface":   &ffapi.UUIDField{},
	"description": &ffapi.StringField{},
}

FFIMethodQueryFactory filter fields for contract methods

var FFIQueryFactory = &ffapi.QueryFields{
	"id":      &ffapi.UUIDField{},
	"name":    &ffapi.StringField{},
	"version": &ffapi.StringField{},
}

FFIQueryFactory filter fields for contract definitions

var GroupQueryFactory = &ffapi.QueryFields{
	"hash":        &ffapi.Bytes32Field{},
	"message":     &ffapi.UUIDField{},
	"description": &ffapi.StringField{},
	"ledger":      &ffapi.UUIDField{},
	"created":     &ffapi.TimeField{},
}

GroupQueryFactory filter fields for groups

var IdentityQueryFactory = &ffapi.QueryFields{
	"id":                    &ffapi.UUIDField{},
	"did":                   &ffapi.StringField{},
	"parent":                &ffapi.UUIDField{},
	"messages.claim":        &ffapi.UUIDField{},
	"messages.verification": &ffapi.UUIDField{},
	"messages.update":       &ffapi.UUIDField{},
	"type":                  &ffapi.StringField{},
	"name":                  &ffapi.StringField{},
	"description":           &ffapi.StringField{},
	"profile":               &ffapi.JSONField{},
	"created":               &ffapi.TimeField{},
	"updated":               &ffapi.TimeField{},
}

IdentityQueryFactory filter fields for identities

var MessageQueryFactory = &ffapi.QueryFields{
	"id":             &ffapi.UUIDField{},
	"cid":            &ffapi.UUIDField{},
	"type":           &ffapi.StringField{},
	"author":         &ffapi.StringField{},
	"key":            &ffapi.StringField{},
	"topics":         &ffapi.FFStringArrayField{},
	"tag":            &ffapi.StringField{},
	"group":          &ffapi.Bytes32Field{},
	"created":        &ffapi.TimeField{},
	"datahash":       &ffapi.Bytes32Field{},
	"idempotencykey": &ffapi.StringField{},
	"hash":           &ffapi.Bytes32Field{},
	"pins":           &ffapi.FFStringArrayField{},
	"state":          &ffapi.StringField{},
	"confirmed":      &ffapi.TimeField{},
	"sequence":       &ffapi.Int64Field{},
	"txtype":         &ffapi.StringField{},
	"batch":          &ffapi.UUIDField{},
}

MessageQueryFactory filter fields for messages

var NextPinQueryFactory = &ffapi.QueryFields{
	"context":  &ffapi.Bytes32Field{},
	"identity": &ffapi.StringField{},
	"hash":     &ffapi.Bytes32Field{},
	"nonce":    &ffapi.Int64Field{},
}

NextPinQueryFactory filter fields for next pins

var NonceQueryFactory = &ffapi.QueryFields{
	"hash":  &ffapi.StringField{},
	"nonce": &ffapi.Int64Field{},
}

NonceQueryFactory filter fields for nonces

var OffsetQueryFactory = &ffapi.QueryFields{
	"name":    &ffapi.StringField{},
	"type":    &ffapi.StringField{},
	"current": &ffapi.Int64Field{},
}

OffsetQueryFactory filter fields for data offsets

var OperationQueryFactory = &ffapi.QueryFields{
	"id":      &ffapi.UUIDField{},
	"tx":      &ffapi.UUIDField{},
	"type":    &ffapi.StringField{},
	"status":  &ffapi.StringField{},
	"error":   &ffapi.StringField{},
	"plugin":  &ffapi.StringField{},
	"input":   &ffapi.JSONField{},
	"output":  &ffapi.JSONField{},
	"created": &ffapi.TimeField{},
	"updated": &ffapi.TimeField{},
	"retry":   &ffapi.UUIDField{},
}

OperationQueryFactory filter fields for data operations

var PinQueryFactory = &ffapi.QueryFields{
	"sequence":   &ffapi.Int64Field{},
	"masked":     &ffapi.BoolField{},
	"hash":       &ffapi.Bytes32Field{},
	"batch":      &ffapi.UUIDField{},
	"index":      &ffapi.Int64Field{},
	"dispatched": &ffapi.BoolField{},
	"created":    &ffapi.TimeField{},
}

PinQueryFactory filter fields for parked contexts

var SubscriptionQueryFactory = &ffapi.QueryFields{
	"id":        &ffapi.UUIDField{},
	"name":      &ffapi.StringField{},
	"transport": &ffapi.StringField{},
	"events":    &ffapi.StringField{},
	"filters":   &ffapi.JSONField{},
	"options":   &ffapi.StringField{},
	"created":   &ffapi.TimeField{},
}

SubscriptionQueryFactory filter fields for data subscriptions

var TokenAccountPoolQueryFactory = &ffapi.QueryFields{
	"pool":    &ffapi.UUIDField{},
	"updated": &ffapi.TimeField{},
}

TokenAccountPoolQueryFactory filter fields for token account pools

var TokenAccountQueryFactory = &ffapi.QueryFields{
	"key":     &ffapi.StringField{},
	"updated": &ffapi.TimeField{},
}

TokenAccountQueryFactory filter fields for token accounts

var TokenApprovalQueryFactory = &ffapi.QueryFields{
	"localid":         &ffapi.StringField{},
	"pool":            &ffapi.UUIDField{},
	"connector":       &ffapi.StringField{},
	"key":             &ffapi.StringField{},
	"operator":        &ffapi.StringField{},
	"approved":        &ffapi.BoolField{},
	"protocolid":      &ffapi.StringField{},
	"subject":         &ffapi.StringField{},
	"active":          &ffapi.BoolField{},
	"created":         &ffapi.TimeField{},
	"tx.type":         &ffapi.StringField{},
	"tx.id":           &ffapi.UUIDField{},
	"blockchainevent": &ffapi.UUIDField{},
	"message":         &ffapi.UUIDField{},
	"messagehash":     &ffapi.Bytes32Field{},
}

var TokenBalanceQueryFactory = &ffapi.QueryFields{
	"pool":       &ffapi.UUIDField{},
	"tokenindex": &ffapi.StringField{},
	"uri":        &ffapi.StringField{},
	"connector":  &ffapi.StringField{},
	"key":        &ffapi.StringField{},
	"balance":    &ffapi.Int64Field{},
	"updated":    &ffapi.TimeField{},
}

TokenBalanceQueryFactory filter fields for token balances

var TokenPoolQueryFactory = &ffapi.QueryFields{
	"id":              &ffapi.UUIDField{},
	"type":            &ffapi.StringField{},
	"name":            &ffapi.StringField{},
	"standard":        &ffapi.StringField{},
	"locator":         &ffapi.StringField{},
	"symbol":          &ffapi.StringField{},
	"decimals":        &ffapi.Int64Field{},
	"message":         &ffapi.UUIDField{},
	"state":           &ffapi.StringField{},
	"created":         &ffapi.TimeField{},
	"connector":       &ffapi.StringField{},
	"tx.type":         &ffapi.StringField{},
	"tx.id":           &ffapi.UUIDField{},
	"interface":       &ffapi.UUIDField{},
	"interfaceformat": &ffapi.StringField{},
}

TokenPoolQueryFactory filter fields for token pools

var TokenTransferQueryFactory = &ffapi.QueryFields{
	"localid":         &ffapi.StringField{},
	"pool":            &ffapi.UUIDField{},
	"tokenindex":      &ffapi.StringField{},
	"uri":             &ffapi.StringField{},
	"connector":       &ffapi.StringField{},
	"key":             &ffapi.StringField{},
	"from":            &ffapi.StringField{},
	"to":              &ffapi.StringField{},
	"amount":          &ffapi.Int64Field{},
	"protocolid":      &ffapi.StringField{},
	"message":         &ffapi.UUIDField{},
	"messagehash":     &ffapi.Bytes32Field{},
	"created":         &ffapi.TimeField{},
	"tx.type":         &ffapi.StringField{},
	"tx.id":           &ffapi.UUIDField{},
	"blockchainevent": &ffapi.UUIDField{},
	"type":            &ffapi.StringField{},
}

TokenTransferQueryFactory filter fields for token transfers

var TransactionQueryFactory = &ffapi.QueryFields{
	"id":             &ffapi.UUIDField{},
	"type":           &ffapi.StringField{},
	"created":        &ffapi.TimeField{},
	"idempotencykey": &ffapi.StringField{},
	"blockchainids":  &ffapi.FFStringArrayField{},
}

TransactionQueryFactory filter fields for transactions

var VerifierQueryFactory = &ffapi.QueryFields{
	"hash":     &ffapi.Bytes32Field{},
	"identity": &ffapi.UUIDField{},
	"type":     &ffapi.StringField{},
	"value":    &ffapi.StringField{},
	"created":  &ffapi.TimeField{},
}

VerifierQueryFactory filter fields for identities

Types

type Callbacks

type Callbacks interface {
	// OrderedUUIDCollectionNSEvent emits the sequence on insert, but it will be -1 on update
	OrderedUUIDCollectionNSEvent(resType OrderedUUIDCollectionNS, eventType core.ChangeEventType, namespace string, id *fftypes.UUID, sequence int64)
	OrderedCollectionNSEvent(resType OrderedCollectionNS, eventType core.ChangeEventType, namespace string, sequence int64)
	UUIDCollectionNSEvent(resType UUIDCollectionNS, eventType core.ChangeEventType, namespace string, id *fftypes.UUID)
	HashCollectionNSEvent(resType HashCollectionNS, eventType core.ChangeEventType, namespace string, hash *fftypes.Bytes32)
}

Callbacks are the methods for passing data from plugin to core

If Capabilities returns ClusterEvents=true then these should be broadcast to every instance within a cluster that is connected to the database.

If Capabilities returns ClusterEvents=false then these events can be simply coupled in-process to update activities.

The system does not rely on these events exclusively for data/transaction integrity, but if an event is missed/delayed it might result in slower processing. For example, the batch interface will initiate a batch as soon as an event is triggered, but it will use a subsequent database query as the source of truth of the latest set/order of data, and it will periodically check for new messages even if it does not receive any events.

Events are emitted locally to the individual FireFly core process. However, a WebSocket interface is available for remote listening to these events. That allows the UI to listen to the events, as well as providing a building block for a cluster of FireFly servers to directly propgate events to each other.

type Capabilities

type Capabilities struct {
	Concurrency bool
}

Capabilities defines the capabilities a plugin can report as implementing or not

type CollectionName

type CollectionName string

CollectionName represents all collections

type HashCollectionNS

type HashCollectionNS CollectionName

HashCollectionNS is a collection where the primary key is a hash, such that it can by identified by any member of the network at any time, without it first having been broadcast.

const (
	CollectionGroups    HashCollectionNS = "groups"
	CollectionVerifiers HashCollectionNS = "verifiers"
)

type OrderedCollectionNS

type OrderedCollectionNS CollectionName

OrderedCollectionNS is a collection that is ordered, and that sequence is the only key

const (
	CollectionPins OrderedCollectionNS = "pins"
)

type OrderedUUIDCollectionNS

type OrderedUUIDCollectionNS CollectionName

OrderedUUIDCollectionNS collections have a strong order that includes a sequence integer that uniquely identifies the entry in a sequence. The sequence is LOCAL to this FireFly node. We try to minimize adding new collections of this type, as they have implementation complexity in some databases (such as NoSQL databases)

const (
	CollectionMessages OrderedUUIDCollectionNS = "messages"
	CollectionEvents   OrderedUUIDCollectionNS = "events"
)

type OtherCollection

type OtherCollection CollectionName

OtherCollection are odd balls, that don't fit any of the categories above. These collections do not support change events, and generally their creation is coordinated with creation of another object that does support change events. Mainly they are entries that require lookup by compound IDs.

const (
	CollectionBlobs         OtherCollection = "blobs"
	CollectionNextpins      OtherCollection = "nextpins"
	CollectionNonces        OtherCollection = "nonces"
	CollectionOffsets       OtherCollection = "offsets"
	CollectionTokenBalances OtherCollection = "tokenbalances"
)

type PersistenceInterface

type PersistenceInterface interface {
	core.Named

	// RunAsGroup instructs the database plugin that all database operations performed within the context
	// function can be grouped into a single transaction (if supported).
	// Requirements:
	// - Firefly must not depend on this to guarantee ACID properties (it is only a suggestion/optimization)
	// - The database implementation must support nested RunAsGroup calls (ie by reusing a transaction if one exists)
	// - The caller is responsible for passing the supplied context to all database operations within the callback function
	RunAsGroup(ctx context.Context, fn func(ctx context.Context) error) error
	// contains filtered or unexported methods
}

PeristenceInterface are the operations that must be implemented by a database interface plugin. The database mechanism of Firefly is designed to provide the balance between being able to query the data a member of the network has transferred/received via Firefly efficiently, while not trying to become the core database of the application (where full deeply nested rich query is needed).

This means that we treat business data as opaque within the storage, only verifying it against a data definition within the Firefly core runtime itself. The data types, indexes and relationships are designed to be simple, and map closely to the REST semantics of the Firefly API itself.

As a result, the database interface could be implemented efficiently by most database technologies. Including both Relational/SQL and Document/NoSQL database technologies.

As such we suggest the factors in choosing your database should be non-functional, such as: - Which provides you with the HA/DR capabilities you require - Which is most familiar within your existing devops pipeline for the application - Whether you can consolidate the HA/DR and server infrastructure for your app DB with the Firefly DB

Each database does need an update to the core codebase, to provide a plugin that implements this interface. For SQL databases the process of adding a new database is simplified via the common SQL layer. For NoSQL databases, the code should be straight forward to map the collections, indexes, and operations.

type Plugin

type Plugin interface {
	PersistenceInterface // Split out to aid pluggability the next level down (SQL provider etc.)

	// InitConfig initializes the set of configuration options that are valid, with defaults. Called on all plugins.
	InitConfig(config config.Section)

	// Init initializes the plugin, with configuration
	Init(ctx context.Context, config config.Section) error

	// SetHandler registers a handler to receive callbacks
	// Plugin will attempt (but is not guaranteed) to deliver events only for the given namespace
	SetHandler(namespace string, handler Callbacks)

	// Capabilities returns capabilities - not called until after Init
	Capabilities() *Capabilities
}

Plugin is the interface implemented by each plugin

type PostCompletionHook

type PostCompletionHook func()

PostCompletionHook is a closure/function that will be called after a successful insertion. This includes where the insert is nested in a RunAsGroup, and the database is transactional. These hooks are useful when triggering code that relies on the inserted database object being available.

type UUIDCollectionNS

type UUIDCollectionNS CollectionName

UUIDCollectionNS is the most common type of collection - each entry has a UUID that is globally unique, and used externally by apps to address entries in the collection. Objects in these collections are all namespaced,.

const (
	CollectionBatches           UUIDCollectionNS = "batches"
	CollectionBlockchainEvents  UUIDCollectionNS = "blockchainevents"
	CollectionData              UUIDCollectionNS = "data"
	CollectionDataTypes         UUIDCollectionNS = "datatypes"
	CollectionOperations        UUIDCollectionNS = "operations"
	CollectionSubscriptions     UUIDCollectionNS = "subscriptions"
	CollectionTransactions      UUIDCollectionNS = "transactions"
	CollectionTokenPools        UUIDCollectionNS = "tokenpools"
	CollectionTokenTransfers    UUIDCollectionNS = "tokentransfers"
	CollectionTokenApprovals    UUIDCollectionNS = "tokenapprovals"
	CollectionFFIs              UUIDCollectionNS = "ffi"
	CollectionFFIMethods        UUIDCollectionNS = "ffimethods"
	CollectionFFIEvents         UUIDCollectionNS = "ffievents"
	CollectionFFIErrors         UUIDCollectionNS = "ffierrors"
	CollectionContractAPIs      UUIDCollectionNS = "contractapis"
	CollectionContractListeners UUIDCollectionNS = "contractlisteners"
	CollectionIdentities        UUIDCollectionNS = "identities"
)

type UpsertOptimization

type UpsertOptimization int

const (
	UpsertOptimizationSkip UpsertOptimization = iota
	UpsertOptimizationNew
	UpsertOptimizationExisting
)