Documentation
¶
Overview ¶
Package stark is an IPFS-backed database for distributed Sequence Recording And Record Keeping.
It is both a library and a Command Line Utility for running and interacting with `stark databases`.
Features include:
- snapshot and sync entire databases over the IPFS
- use PubSub messaging to share and collect data records as they are created
- track record history and rollback revisions (rollback feature WIP)
- attach and sync files to records (WIP)
- encrypt record fields
Index ¶
- Constants
- Variables
- func GetBaseVersion() string
- func GetVersion() string
- func RegisterStarkDbServer(s *grpc.Server, srv StarkDbServer)
- type Db
- func (starkdb *Db) Delete(key string) error
- func (starkdb *Db) Dump(ctx context.Context, key *Key) (*DbMeta, error)
- func (starkdb *Db) Get(ctx context.Context, key *Key) (*Record, error)
- func (starkdb *Db) GetCIDs() map[string]string
- func (starkdb *Db) GetConnectedPeers() ([]string, error)
- func (starkdb *Db) GetNodeAddr() (string, error)
- func (starkdb *Db) GetNumEntries() int
- func (starkdb *Db) GetSnapshot() string
- func (starkdb *Db) Listen(terminator chan struct{}) (chan *Record, chan error, error)
- func (starkdb *Db) PinataPublish(apiKey, apiSecret string) (*starkpinata.APIResponse, error)
- func (starkdb *Db) Set(ctx context.Context, record *Record) (*Record, error)
- type DbMeta
- func (*DbMeta) Descriptor() ([]byte, []int)deprecated
- func (x *DbMeta) GetAnnouncing() bool
- func (x *DbMeta) GetCurrEntries() int32
- func (x *DbMeta) GetNodeAddress() string
- func (x *DbMeta) GetPairs() map[string]string
- func (x *DbMeta) GetPeers() []string
- func (x *DbMeta) GetPinning() bool
- func (x *DbMeta) GetProject() string
- func (x *DbMeta) GetSnapshot() string
- func (*DbMeta) ProtoMessage()
- func (x *DbMeta) ProtoReflect() protoreflect.Message
- func (x *DbMeta) Reset()
- func (x *DbMeta) String() string
- type DbOption
- func SetProject(project string) DbOption
- func SetSnapshotCID(path string) DbOption
- func WithAnnouncing() DbOption
- func WithEncryption() DbOption
- func WithLogging(loggingChan chan interface{}) DbOption
- func WithNoPinning() DbOption
- func WithPeers(nodeList []string) DbOption
- func WithPinata(interval int) DbOption
- type Key
- type Record
- func (x *Record) AddComment(text string)
- func (x *Record) Decrypt(cipherKey []byte) error
- func (*Record) Descriptor() ([]byte, []int)deprecated
- func (x *Record) Encrypt(cipherKey []byte) error
- func (x *Record) GetAlias() string
- func (x *Record) GetBarcodes() map[string]int32
- func (x *Record) GetCreatedTimestamp() *timestamp.Timestamp
- func (x *Record) GetDescription() string
- func (x *Record) GetEncrypted() bool
- func (x *Record) GetHistory() []*RecordComment
- func (x *Record) GetLastUpdatedTimestamp() *timestamp.Timestamp
- func (x *Record) GetLinkedLibraries() map[string]string
- func (x *Record) GetLinkedSamples() map[string]string
- func (x *Record) GetLocalSequencerOutputDir() string
- func (x *Record) GetPreviousCID() string
- func (x *Record) GetStatus() Status
- func (x *Record) GetUuid() string
- func (x *Record) LinkLibrary(libraryUUID uuid.UUID, libraryLocation string) error
- func (x *Record) LinkSample(sampleUUID uuid.UUID, sampleLocation string) error
- func (*Record) ProtoMessage()
- func (x *Record) ProtoReflect() protoreflect.Message
- func (x *Record) Reset()
- func (x *Record) String() string
- type RecordComment
- func (*RecordComment) Descriptor() ([]byte, []int)deprecated
- func (x *RecordComment) GetPreviousCID() string
- func (x *RecordComment) GetText() string
- func (x *RecordComment) GetTimestamp() *timestamp.Timestamp
- func (*RecordComment) ProtoMessage()
- func (x *RecordComment) ProtoReflect() protoreflect.Message
- func (x *RecordComment) Reset()
- func (x *RecordComment) String() string
- type RecordOption
- type StarkDbClient
- type StarkDbServer
- type Status
- type UnimplementedStarkDbServer
Constants ¶
const ( // DefaultBufferSize is the maximum number of records stored in channels. DefaultBufferSize = 42 // DefaultMinBootstrappers is the minimum number of reachable bootstrappers required. DefaultMinBootstrappers = 3 // DefaultProject is the default project name used if none provided to the OpenDB function. DefaultProject = "starkDB-default-project" // DefaultPinataAPIkey is the env variable for the pinata API. DefaultPinataAPIkey = "PINATA_API_KEY" // DefaultPinataSecretKey is the env variable for the pinata secret key. DefaultPinataSecretKey = "PINATA_SECRET_KEY" // DefaultStarkEnvVariable is the env variable starkDB looks for when told to use encryption. DefaultStarkEnvVariable = "STARK_DB_PASSWORD" )
Variables ¶
var ( // ErrAttemptedOverwrite indicates a starkDB key is already in use for a Record with non-matching UUID. ErrAttemptedOverwrite = fmt.Errorf("starkDB key is already in use for a Record with non-matching UUID") // ErrAttemptedUpdate indicates a Record with matching UUID is already in the IPFS and has a more recent update timestamp. ErrAttemptedUpdate = fmt.Errorf("cannot update a Record in starkDB with an older version") // ErrBootstrappers is issued when not enough bootstrappers are accessible. ErrBootstrappers = fmt.Errorf("not enough bootstrappers found (minimum required: %d)", DefaultMinBootstrappers) // ErrCipherPasswordMismatch is issued when a password does not decrypt a Record. ErrCipherPasswordMismatch = fmt.Errorf("provided password cannot decrypt Record") // ErrDbOption is issued for incorrect database initialisation options. ErrDbOption = fmt.Errorf("starkDB option could not be set") // ErrEncrypted is issued when an encryption is attempted on an encrypted Record. ErrEncrypted = fmt.Errorf("data is encrypted with passphrase") // ErrInvalidSnapshot indicates a snapshotted IPFS DAG node can't be accessed. ErrInvalidSnapshot = fmt.Errorf("cannot access the database snapshot") // ErrLinkExists indicates a Record is already linked to the provided UUID. ErrLinkExists = fmt.Errorf("Record already linked to the provided UUID") // ErrNoCID indicates no CID was provided. ErrNoCID = fmt.Errorf("no CID was provided") // ErrNoKey indicates no record key was provided. ErrNoKey = fmt.Errorf("no key could be found, make sure Record Alias is set") // ErrNotFound indicates a key was not found in the starkDB. ErrNotFound = func(key string) error { return fmt.Errorf("key not found: %v", key) } // ErrNodeFormat is issued when a CID points to a node with an unsupported format. ErrNodeFormat = fmt.Errorf("database entry points to a non-CBOR node") // ErrNodeOffline indicates the node is offline. ErrNodeOffline = fmt.Errorf("IPFS node is offline") // ErrNodeOnline indicates the node is online. ErrNodeOnline = fmt.Errorf("IPFS node is online") // ErrNoEnvSet is issued when no env variable is found. ErrNoEnvSet = fmt.Errorf("no %s environment variable found", DefaultStarkEnvVariable) // ErrNoPeerID indicates the IPFS node has no peer ID. ErrNoPeerID = fmt.Errorf("no PeerID listed for the current IPFS node") // ErrNoProject indicates no project name was given. ErrNoProject = fmt.Errorf("project name is required for a starkDB") // ErrNoSub indicates the IPFS node is not registered for PubSub. ErrNoSub = fmt.Errorf("IPFS node has no topic registered for PubSub") // ErrNoPeers indicates no peers were proided but announcing via PubSub was requested. ErrNoPeers = fmt.Errorf("no peers given but PubSub announcements requested, chances of messages being received is low") // ErrPinataAPI indicates the Pinata API can't be reached. ErrPinataAPI = func(err error) error { return fmt.Errorf("failed to reach Pinata API: %w", err) } // ErrPinataOpt is issued for a db option pinning conflict. ErrPinataOpt = fmt.Errorf("can't use WithPinata when WithNoPinning") // ErrPinataKey is issued when the no env variable for the Pinata API key is set. ErrPinataKey = fmt.Errorf("no %s environment variable found", DefaultPinataAPIkey) // ErrPinataSecret is issued when the no env variable for the Pinata secret is set. ErrPinataSecret = fmt.Errorf("no %s environment variable found", DefaultPinataSecretKey) // ErrRecordHistory indicates two Records with the same UUID a gap in their history. ErrRecordHistory = fmt.Errorf("both Records share UUID but have a gap in their history") // ErrSnapshotUpdate is issued when a link can't be made between the new Record and existing project base node. ErrSnapshotUpdate = fmt.Errorf("could not update database snapshot") )
var ( Status_name = map[int32]string{ 0: "UN_INITIALIZED", 1: "untagged", 2: "tagged", } Status_value = map[string]int32{ "UN_INITIALIZED": 0, "untagged": 1, "tagged": 2, } )
Enum value maps for Status.
var File_stark_proto protoreflect.FileDescriptor
Functions ¶
func GetBaseVersion ¶
func GetBaseVersion() string
GetBaseVersion returns the major minor version string for the current stark package.
func GetVersion ¶
func GetVersion() string
GetVersion returns the full version string for the current stark package.
func RegisterStarkDbServer ¶
func RegisterStarkDbServer(s *grpc.Server, srv StarkDbServer)
Types ¶
type Db ¶
type Db struct { sync.RWMutex // protects access to the bound IPFS node // contains filtered or unexported fields }
Db is the starkDB database.
func OpenDB ¶
OpenDB opens a new instance of a starkdb.
If there is an existing database in the specified local storage location, which has the specified project name, the DB will open that.
It returns the initialised database, a teardown function and any error encountered.
func (*Db) Delete ¶
Delete will delete an entry from starkdb. This involves removing the key and Record CID from the local store, as well as unpinning the Record from the IPFS.
Note: I'm not sure how this behaves if the Record wasn't pinned in the IPFS in the first place.
func (*Db) Dump ¶
Dump returns the metadata from a starkDB instance.
Note: input key is currently unused.
func (*Db) Get ¶
Get will retrieve a copy of a Record from the starkDB using the provided lookup key.
func (*Db) GetCIDs ¶
GetCIDs will return a map of keys to CIDs for all Records currently held in the database.
func (*Db) GetConnectedPeers ¶
GetConnectedPeers returns the addresses of the currently connected IPFS peers.
func (*Db) GetNodeAddr ¶
GetNodeAddr returns the public address of the underlying IPFS node for the starkDB instance.
func (*Db) GetNumEntries ¶
GetNumEntries returns the number of entries in the current database instance.
func (*Db) GetSnapshot ¶
GetSnapshot returns the current database snapshot CID, which can be used to rebuild the current database instance.
Note: if the database has no entries, the returned snapshot will be a nil string.
func (*Db) Listen ¶
Listen will start a subscription to the IPFS PubSub network for messages matching the current database's project. It tries pulling Records from the IPFS via the announced CIDs, then returns them via a channel to the caller.
It returns the Record channel, an Error channel which reports errors during message processing and Record retrieval, as well as any error during the PubSub setup.
func (*Db) PinataPublish ¶
func (starkdb *Db) PinataPublish(apiKey, apiSecret string) (*starkpinata.APIResponse, error)
PinataPublish will issue an API call to the pinata pinByHash endpoint and pin the current database instance.
func (*Db) Set ¶
Set will add a copy of a Record to the starkDB, using the Record Alias as a lookup key.
This method will add comments to the Record's history before adding it to the IPFS.
It will return a pointer to a copy of the Record that was added to the starkDB, which contains the updated Record history and the CID for the Record in the IPFS.
type DbMeta ¶
type DbMeta struct { Project string `protobuf:"bytes,1,opt,name=Project,proto3" json:"Project,omitempty"` // project name for the current database Snapshot string `protobuf:"bytes,2,opt,name=Snapshot,proto3" json:"Snapshot,omitempty"` // CID for the current database state NodeAddress string `protobuf:"bytes,3,opt,name=NodeAddress,proto3" json:"NodeAddress,omitempty"` // the IPFS node address for the database Peers []string `protobuf:"bytes,4,rep,name=Peers,proto3" json:"Peers,omitempty"` // the IPFS peers in the swarm Pinning bool `protobuf:"varint,5,opt,name=Pinning,proto3" json:"Pinning,omitempty"` // if true, database is pinning entries Announcing bool `protobuf:"varint,6,opt,name=Announcing,proto3" json:"Announcing,omitempty"` // if true, database is announcing entries on PubSub CurrEntries int32 `protobuf:"varint,7,opt,name=CurrEntries,proto3" json:"CurrEntries,omitempty"` // current number of entries in the database Pairs map[string]string `` // pairs of Keys -> Record CIDs held in the database /* 151-byte string literal not displayed */ // contains filtered or unexported fields }
DbMeta.
This message is used to encode runtime information for a StarkDb.
func (*DbMeta) Descriptor
deprecated
func (*DbMeta) GetAnnouncing ¶
func (*DbMeta) GetCurrEntries ¶
func (*DbMeta) GetNodeAddress ¶
func (*DbMeta) GetPinning ¶
func (*DbMeta) GetProject ¶
func (*DbMeta) GetSnapshot ¶
func (*DbMeta) ProtoMessage ¶
func (*DbMeta) ProtoMessage()
func (*DbMeta) ProtoReflect ¶
func (x *DbMeta) ProtoReflect() protoreflect.Message
type DbOption ¶
DbOption is a wrapper struct used to pass functional options to the starkDB constructor.
func SetProject ¶
SetProject is an option setter for the OpenDB constructor that sets the project for the database instance. The project name is used to broadcast messages when Records are added to the database instance.
func SetSnapshotCID ¶
SetSnapshotCID is an option setter for the OpenDB constructor that sets the base CID to use for the database instance. If none provided it will open an empty database, otherwise it will check the provided CID and populate the starkDB from the existing records contained in the snapshot.
func WithAnnouncing ¶
func WithAnnouncing() DbOption
WithAnnouncing is an option setter for the OpenDB constructor that sets the database to announcing new records via PubSub as they are added to the database.
When Set is called and WithAnnouncing is set, the CID of the set Record is broadcast on IPFS with the database project as the topic.
func WithEncryption ¶
func WithEncryption() DbOption
WithEncryption is an option setter for the OpenDB constructor that tells starkDB to make encrypted writes to IPFS using the password in STARK_DB_PASSWORD env variable.
Note: If existing Records were encrypted, Get operations will fail unless this option is set.
func WithLogging ¶
func WithLogging(loggingChan chan interface{}) DbOption
WithLogging is an option setter for the OpenDB constructor that provides starkDB with a logging channel to send internal state messages during the lifetime of the starkDB instance back to the caller.
Note: The caller should close this channel when done.
func WithNoPinning ¶
func WithNoPinning() DbOption
WithNoPinning is an option setter that specifies the IPFS node should NOT pin entries.
Note: If not provided to the constructor, the node will pin entries by default.
func WithPeers ¶
WithPeers is an option setter for the OpenDB constructor that adds a list of nodes to the default IPFS bootstrappers.
func WithPinata ¶
WithPinata is an option setter for the OpenDB constructor that tells starkDB to pin it's contents with pinata every time the interval is passed during set operations. A value of < 1 tells starkDB NOT to use pinata (default).
Note: This option requires the PINATA_API_KEY and the PINATA_SECRET_KEY environment variables to be set.
type Key ¶
type Key struct { Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // contains filtered or unexported fields }
Key.
This message is used to send queries from gRPC clients to a stark database that implements the StarkDb interface.
func (*Key) Descriptor
deprecated
func (*Key) ProtoMessage ¶
func (*Key) ProtoMessage()
func (*Key) ProtoReflect ¶
func (x *Key) ProtoReflect() protoreflect.Message
type Record ¶
type Record struct { // reserved: Uuid string `protobuf:"bytes,1,opt,name=uuid,proto3" json:"uuid,omitempty"` // universally unique id for the record PreviousCID string `protobuf:"bytes,2,opt,name=previousCID,proto3" json:"previousCID,omitempty"` // the last known CID this record was pulled from (set during a Get operation) History []*RecordComment `protobuf:"bytes,3,rep,name=history,proto3" json:"history,omitempty"` // describes the history of the record - can be used to get timestamps for creation and last updated Encrypted bool `protobuf:"varint,4,opt,name=encrypted,proto3" json:"encrypted,omitempty"` // set true to indicate if fields have been encrypted Status Status `protobuf:"varint,5,opt,name=status,proto3,enum=stark.Status" json:"status,omitempty"` // describes if untagged/tagged/etc. // user updateable: Alias string `protobuf:"bytes,6,opt,name=alias,proto3" json:"alias,omitempty"` // the record name / human readable id (used as the default RecordKey ID) Description string `protobuf:"bytes,7,opt,name=description,proto3" json:"description,omitempty"` // a short description of the record LocalSequencerOutputDir string `protobuf:"bytes,9,opt,name=localSequencerOutputDir,proto3" json:"localSequencerOutputDir,omitempty"` // where the sequencer is outputing data for this record LinkedSamples map[string]string `` // all samples linked to this record (map relates sample UUIDs to a metadata location (e.g. a CID)) /* 168-byte string literal not displayed */ LinkedLibraries map[string]string `` // all libraries linked to this record (map relates library UUIDs to a metadata location (e.g. a CID)) /* 172-byte string literal not displayed */ Barcodes map[string]int32 `` // all barcodes used by this record (map links library UUID to barcode for that library) /* 159-byte string literal not displayed */ // contains filtered or unexported fields }
Record.
This message is used to describe a sequencing metadata. Record's will be able to encode runs, samples and libraries. WIP.
func NewRecord ¶
func NewRecord(options ...RecordOption) (*Record, error)
NewRecord creates a record.
func NewRecordFromReader ¶
NewRecordFromReader creates a Record from a reader. Accepts either JSON or Protobuf encoded Record.
func (*Record) AddComment ¶
AddComment adds a timestamped comment to the Record's history, along with the last known CID to enable rollbacks.
func (*Record) Decrypt ¶
Decrypt will decrypt certain fields of a Record. Unencrypted Records are ignored and errors are reported for unsuccessful decrypts.
Note: Currently only the Record UUID is decrypted.
func (*Record) Descriptor
deprecated
func (*Record) Encrypt ¶
Encrypt will encrypt certain fields of a Record.
Note: Currently only the Record UUID is encrypted.
func (*Record) GetBarcodes ¶
func (*Record) GetCreatedTimestamp ¶
GetCreatedTimestamp returns the timestamp for when the record was created.
func (*Record) GetDescription ¶
func (*Record) GetEncrypted ¶
func (*Record) GetHistory ¶
func (x *Record) GetHistory() []*RecordComment
func (*Record) GetLastUpdatedTimestamp ¶
GetLastUpdatedTimestamp returns the timestamp for when the record was created.
func (*Record) GetLinkedLibraries ¶
func (*Record) GetLinkedSamples ¶
func (*Record) GetLocalSequencerOutputDir ¶
func (*Record) GetPreviousCID ¶
func (*Record) LinkLibrary ¶
LinkLibrary links a Library to a Record.
func (*Record) LinkSample ¶
LinkSample links a Sample to a Record.
func (*Record) ProtoMessage ¶
func (*Record) ProtoMessage()
func (*Record) ProtoReflect ¶
func (x *Record) ProtoReflect() protoreflect.Message
type RecordComment ¶
type RecordComment struct { Timestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` // timestamp for change Text string `protobuf:"bytes,2,opt,name=text,proto3" json:"text,omitempty"` // description of the change PreviousCID string `protobuf:"bytes,3,opt,name=previousCID,proto3" json:"previousCID,omitempty"` // last known CID for a Record (used to rollback the Record and undo the newly commented change) // contains filtered or unexported fields }
RecordComment is used to describe message history.
func NewComment ¶
func NewComment(comment, prevCID string) *RecordComment
NewComment creates a comment.
func (*RecordComment) Descriptor
deprecated
func (*RecordComment) Descriptor() ([]byte, []int)
Deprecated: Use RecordComment.ProtoReflect.Descriptor instead.
func (*RecordComment) GetPreviousCID ¶
func (x *RecordComment) GetPreviousCID() string
func (*RecordComment) GetText ¶
func (x *RecordComment) GetText() string
func (*RecordComment) GetTimestamp ¶
func (x *RecordComment) GetTimestamp() *timestamp.Timestamp
func (*RecordComment) ProtoMessage ¶
func (*RecordComment) ProtoMessage()
func (*RecordComment) ProtoReflect ¶
func (x *RecordComment) ProtoReflect() protoreflect.Message
func (*RecordComment) Reset ¶
func (x *RecordComment) Reset()
func (*RecordComment) String ¶
func (x *RecordComment) String() string
type RecordOption ¶
RecordOption is a wrapper struct used to pass functional options to the Record constructor.
func SetAlias ¶
func SetAlias(alias string) RecordOption
SetAlias is an option setter for the NewRecord constructor that sets the human readable label of a Record. Spaces are replaced with underscores.
Note: the Record's alias is used as the key in the StarkDB.
func SetDescription ¶
func SetDescription(description string) RecordOption
SetDescription is an option setter for the NewRecord constructor that sets the description of a Record.
type StarkDbClient ¶
type StarkDbClient interface { Get(ctx context.Context, in *Key, opts ...grpc.CallOption) (*Record, error) Set(ctx context.Context, in *Record, opts ...grpc.CallOption) (*Record, error) Dump(ctx context.Context, in *Key, opts ...grpc.CallOption) (*DbMeta, error) }
StarkDbClient is the client API for StarkDb service.
For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
func NewStarkDbClient ¶
func NewStarkDbClient(cc grpc.ClientConnInterface) StarkDbClient
type StarkDbServer ¶
type StarkDbServer interface { Get(context.Context, *Key) (*Record, error) Set(context.Context, *Record) (*Record, error) Dump(context.Context, *Key) (*DbMeta, error) }
StarkDbServer is the server API for StarkDb service.
type Status ¶
type Status int32
Status is currently un-used.
func (Status) Descriptor ¶
func (Status) Descriptor() protoreflect.EnumDescriptor
func (Status) EnumDescriptor
deprecated
func (Status) Number ¶
func (x Status) Number() protoreflect.EnumNumber
func (Status) Type ¶
func (Status) Type() protoreflect.EnumType
Source Files
¶
Directories
¶
Path | Synopsis |
---|---|
src
|
|
crypto
Package crypto is used to encrypt and decrypt string data using symmetric key encryption.
|
Package crypto is used to encrypt and decrypt string data using symmetric key encryption. |
helpers
Package helpers contains some basic helper functions for stark.
|
Package helpers contains some basic helper functions for stark. |
ipfs
Package ipfs wraps the ipfs core api with some useful client methods for PubSub and IO.
|
Package ipfs wraps the ipfs core api with some useful client methods for PubSub and IO. |
pinata
Package pinata is used to send requests to the Pinata pinByHash endpoint (https://pinata.cloud/documentation#PinByHash).
|
Package pinata is used to send requests to the Pinata pinByHash endpoint (https://pinata.cloud/documentation#PinByHash). |
cmd
Package cmd is the command line utility for managing a stark database.
|
Package cmd is the command line utility for managing a stark database. |
config
Package config is used to set up and manage the stark config file.
|
Package config is used to set up and manage the stark config file. |