metadata

package
v1.0.0-beta.55 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 28, 2023 License: Apache-2.0 Imports: 28 Imported by: 0

Documentation

Index

Constants

View Source
const (
	MainBranch          = "main"
	BranchNameSeparator = "_$branch$_"
)

Variables

View Source
var (
	ErrMissingId       = fmt.Errorf("cannot enqueue without an id")
	ErrNotFound        = fmt.Errorf("QueueItem was not found")
	ErrClaimed         = fmt.Errorf("another worker has claimed this item")
	ErrJsonDeSerialize = fmt.Errorf("failed to unmarshal queue item")
	ErrJsonSerialize   = fmt.Errorf("failed to marshal queue item")
)
View Source
var DefaultNameRegistry = &NameRegistry{
	ReserveSB:   "reserved",
	EncodingSB:  "encoding",
	SchemaSB:    "schema",
	SearchSB:    "search_schema",
	UserSB:      "user",
	NamespaceSB: "namespace",
	ClusterSB:   "cluster",
	QueueSB:     "queue",

	BaseCounterValue: reservedBaseValue,
}

Functions

func ByteToUInt32

func ByteToUInt32(b []byte) uint32

func NewBranchMismatchErr

func NewBranchMismatchErr(old string, newName string) error

func NewBranchNotFoundErr

func NewBranchNotFoundErr(name string) error

func NewCacheExistsErr

func NewCacheExistsErr(name string) error

func NewCacheNotFoundErr

func NewCacheNotFoundErr(name string) error

func NewDatabaseBranchExistsErr

func NewDatabaseBranchExistsErr(name string) error

func NewDatabaseMismatchErr

func NewDatabaseMismatchErr(old string, newName string) error

func NewMetadataError

func NewMetadataError(code ErrorCode, msg string, args ...interface{}) error

func NewProjectNotFoundErr

func NewProjectNotFoundErr(name string) error

func NewSearchIndexExistsErr

func NewSearchIndexExistsErr(name string) error

func NewSearchIndexNotFoundErr

func NewSearchIndexNotFoundErr(name string) error

func UInt32ToByte

func UInt32ToByte(v uint32) []byte

Types

type CacheEncoder

type CacheEncoder interface {
	EncodeCacheTableName(tenantId uint32, projId uint32, name string) (string, error)
	DecodeCacheTableName(stream string) (uint32, uint32, string, bool)
	DecodeInternalCacheKeyNameToExternal(internalKey string) string
}

func NewCacheEncoder

func NewCacheEncoder() CacheEncoder

NewCacheEncoder creates CacheEncoder to encode cache tenant, project and keys.

type CacheMetadata

type CacheMetadata struct {
	Name      string
	Creator   string
	CreatedAt int64
}

type CacheTracker

type CacheTracker struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

CacheTracker is to track if tenant cache is stale and trigger reloads on it. CacheTracker is used by the session manager to identify during the running transaction whether there is a need to fill tenant state from the database or whether the cached version is up-to-date.

func NewCacheTracker

func NewCacheTracker(tenantMgr *TenantManager, txMgr *transaction.Manager) *CacheTracker

NewCacheTracker creates and returns the cache tracker. It uses tenant manager state to populate in-memory version tracking for each tenant.

func (*CacheTracker) DeferredTracking

func (cacheTracker *CacheTracker) DeferredTracking(ctx context.Context, tx transaction.Tx, tenant *Tenant) (*Tracker, error)

DeferredTracking returns a tracker that has a future attached to the caller's transaction.

func (*CacheTracker) InstantTracking

func (cacheTracker *CacheTracker) InstantTracking(ctx context.Context, tx transaction.Tx, tenant *Tenant) (*Tracker, error)

InstantTracking is when a tracker is needed outside the caller’s transaction. This is used by DDL transactions that are also bumping up the metadata version.

type ClusterMetadata

type ClusterMetadata struct {
	ID uuid.UUID
}

ClusterMetadata keeps cluster wide metadata.

type ClusterSubspace

type ClusterSubspace struct {
	// contains filtered or unexported fields
}

ClusterSubspace is used to store metadata about Tigris clusters.

func NewClusterStore

func NewClusterStore(nameRegistry *NameRegistry) *ClusterSubspace

func (*ClusterSubspace) Get

func (*ClusterSubspace) Update

func (u *ClusterSubspace) Update(ctx context.Context, tx transaction.Tx, metadata *ClusterMetadata) error

type CollectionMetadata

type CollectionMetadata struct {
	ID uint32 `json:"id,omitempty"`
}

CollectionMetadata contains collection wide metadata.

type CollectionSubspace

type CollectionSubspace struct {
	// contains filtered or unexported fields
}

CollectionSubspace is used to store metadata about Tigris collections.

func (*CollectionSubspace) Get

func (c *CollectionSubspace) Get(ctx context.Context, tx transaction.Tx, nsID uint32, dbID uint32, name string,
) (*CollectionMetadata, error)

func (*CollectionSubspace) Update

func (c *CollectionSubspace) Update(ctx context.Context, tx transaction.Tx, nsID uint32, dbID uint32, name string,
	metadata *CollectionMetadata,
) error

type Database

type Database struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

Database is to manage the collections for this database. Check the Clone method before changing this struct.

func NewDatabase

func NewDatabase(id uint32, name string) *Database

func (*Database) BranchName

func (d *Database) BranchName() string

func (*Database) Clone

func (d *Database) Clone() *Database

Clone is used to stage the database.

func (*Database) DbName

func (d *Database) DbName() string

func (*Database) GetCollection

func (d *Database) GetCollection(cname string) *schema.DefaultCollection

GetCollection returns the collection object, or null if the collection map contains no mapping for the database. At this point collection is fully formed and safe to use.

func (*Database) Id

func (d *Database) Id() uint32

Id returns the dictionary encoded value of this collection.

func (*Database) IsBranch

func (d *Database) IsBranch() bool

func (*Database) ListCollection

func (d *Database) ListCollection() []*schema.DefaultCollection

ListCollection returns the collection object of all the collections in this database.

func (*Database) Name

func (d *Database) Name() string

Name returns the internal database name.

func (*Database) SameBranch

func (d *Database) SameBranch(branch string) bool

type DatabaseMetadata

type DatabaseMetadata struct {
	ID uint32 `json:"id,omitempty"`
}

DatabaseMetadata contains database wide metadata.

type DatabaseName

type DatabaseName struct {
	// contains filtered or unexported fields
}

DatabaseName represents a primary database and its branch name.

func NewDatabaseName

func NewDatabaseName(key string) *DatabaseName

func NewDatabaseNameWithBranch

func NewDatabaseNameWithBranch(db string, branch string) *DatabaseName

func (*DatabaseName) Branch

func (b *DatabaseName) Branch() string

Branch belonging to Db.

func (*DatabaseName) Db

func (b *DatabaseName) Db() string

Db is the user facing name of the primary database.

func (*DatabaseName) IsMainBranch

func (b *DatabaseName) IsMainBranch() bool

IsMainBranch returns "True" if this is primary Db or "False" if a branch.

func (*DatabaseName) Name

func (b *DatabaseName) Name() string

Name returns the internal name of the database db with name "catalog" and branch "main" will have internal name as "catalog" db with name "catalog" and branch "feature" will have internal name as "catalog_$branch$_feature".

type DatabaseSubspace

type DatabaseSubspace struct {
	// contains filtered or unexported fields
}

DatabaseSubspace is used to store metadata about Tigris databases.

func (*DatabaseSubspace) Get

func (*DatabaseSubspace) Update

func (d *DatabaseSubspace) Update(ctx context.Context, tx transaction.Tx, nsID uint32, name string, metadata *DatabaseMetadata) error

type DefaultNamespace

type DefaultNamespace struct{}

DefaultNamespace is for "default" namespace in the cluster. This is useful when there is no need to logically group databases. All databases will be created under a single namespace. It is totally fine for a deployment to choose this and just have one namespace. The default assigned value for this namespace is 1.

func NewDefaultNamespace

func NewDefaultNamespace() *DefaultNamespace

func (*DefaultNamespace) Id

func (n *DefaultNamespace) Id() uint32

Id returns id assigned to the namespace.

func (*DefaultNamespace) Metadata

func (n *DefaultNamespace) Metadata() NamespaceMetadata

Metadata returns metadata assigned to the namespace.

func (*DefaultNamespace) StrId

func (n *DefaultNamespace) StrId() string

StrId returns id assigned to the namespace.

type DictKeyEncoder

type DictKeyEncoder struct{}

func (*DictKeyEncoder) DecodeCacheTableName

func (d *DictKeyEncoder) DecodeCacheTableName(name string) (uint32, uint32, string, bool)

func (*DictKeyEncoder) DecodeIndexName

func (d *DictKeyEncoder) DecodeIndexName(indexName []byte) uint32

func (*DictKeyEncoder) DecodeInternalCacheKeyNameToExternal

func (d *DictKeyEncoder) DecodeInternalCacheKeyNameToExternal(internalKey string) string

func (*DictKeyEncoder) DecodeSearchTableName

func (d *DictKeyEncoder) DecodeSearchTableName(name string) (uint32, uint32, string, bool)

DecodeSearchTableName will decode the information from encoded search index Name. This method returns tenant id, project id, and index name.

func (*DictKeyEncoder) DecodeTableName

func (d *DictKeyEncoder) DecodeTableName(tableName []byte) (uint32, uint32, uint32, bool)

func (*DictKeyEncoder) EncodeCacheTableName

func (d *DictKeyEncoder) EncodeCacheTableName(tenantId uint32, projId uint32, name string) (string, error)

func (*DictKeyEncoder) EncodeFDBSearchKey

func (d *DictKeyEncoder) EncodeFDBSearchKey(searchTable string, id string) (keys.Key, error)

func (*DictKeyEncoder) EncodeFDBSearchTableName

func (d *DictKeyEncoder) EncodeFDBSearchTableName(searchTable string) []byte

func (*DictKeyEncoder) EncodeIndexName

func (d *DictKeyEncoder) EncodeIndexName(idx *schema.Index) []byte

func (*DictKeyEncoder) EncodeKey

func (d *DictKeyEncoder) EncodeKey(encodedTable []byte, idx *schema.Index, idxParts []interface{}) (keys.Key, error)

func (*DictKeyEncoder) EncodePartitionTableName

func (d *DictKeyEncoder) EncodePartitionTableName(ns Namespace, db *Database, coll *schema.DefaultCollection) ([]byte, error)

func (*DictKeyEncoder) EncodeSearchTableName

func (d *DictKeyEncoder) EncodeSearchTableName(tenantId uint32, projId uint32, indexName string) string

EncodeSearchTableName will encode search index created by the user and return an encoded string that will be use as an index name in the underlying search store.

func (*DictKeyEncoder) EncodeSecondaryIndexTableName

func (d *DictKeyEncoder) EncodeSecondaryIndexTableName(ns Namespace, db *Database, coll *schema.DefaultCollection) ([]byte, error)

func (*DictKeyEncoder) EncodeTableName

func (d *DictKeyEncoder) EncodeTableName(ns Namespace, db *Database, coll *schema.DefaultCollection) ([]byte, error)

EncodeTableName creates storage friendly table name from namespace, database and collection ids Database and collection objects can be omitted to get table name prefix. If the collection is omitted then result name includes all the collections in the database If both database and collections are omitted then result name includes all databases in the namespace.

type Dictionary

type Dictionary struct {
	NameRegistry
	// contains filtered or unexported fields
}

Dictionary is used to replace variable length strings to their corresponding codes to allocateAndSave it. Compression is achieved by replacing long strings with a simple 4byte representation.

func NewMetadataDictionary

func NewMetadataDictionary(mdNameRegistry *NameRegistry) *Dictionary

func (*Dictionary) Cluster

func (k *Dictionary) Cluster() *ClusterSubspace

func (*Dictionary) Collection

func (k *Dictionary) Collection() *CollectionSubspace

func (*Dictionary) CreateCollection

func (k *Dictionary) CreateCollection(ctx context.Context, tx transaction.Tx, name string,
	namespaceId uint32, dbId uint32,
) (*CollectionMetadata, error)

func (*Dictionary) CreateDatabase

func (k *Dictionary) CreateDatabase(ctx context.Context, tx transaction.Tx, name string, namespaceId uint32,
) (*DatabaseMetadata, error)

func (*Dictionary) CreateIndex

func (k *Dictionary) CreateIndex(ctx context.Context, tx transaction.Tx, name string, namespaceId uint32,
	dbId uint32, collId uint32,
) (*IndexMetadata, error)

func (*Dictionary) Database

func (k *Dictionary) Database() *DatabaseSubspace

func (*Dictionary) DropCollection

func (k *Dictionary) DropCollection(ctx context.Context, tx transaction.Tx, collection string,
	namespaceId uint32, dbId uint32,
) error

func (*Dictionary) DropDatabase

func (k *Dictionary) DropDatabase(ctx context.Context, tx transaction.Tx, dbName string, namespaceId uint32,
) error

DropDatabase will remove the "created" entry from the encoding subspace and will add a "dropped" entry with the same value.

func (*Dictionary) DropIndex

func (k *Dictionary) DropIndex(ctx context.Context, tx transaction.Tx, indexName string, namespaceId uint32,
	dbId uint32, collId uint32,
) error

func (*Dictionary) GetCollection

func (k *Dictionary) GetCollection(ctx context.Context, tx transaction.Tx, collName string,
	namespaceId uint32, dbId uint32,
) (*CollectionMetadata, error)

func (*Dictionary) GetCollections

func (k *Dictionary) GetCollections(ctx context.Context, tx transaction.Tx, namespaceId uint32,
	databaseId uint32,
) (map[string]*CollectionMetadata, error)

func (*Dictionary) GetDatabase

func (k *Dictionary) GetDatabase(ctx context.Context, tx transaction.Tx, dbName string, namespaceId uint32,
) (*DatabaseMetadata, error)

func (*Dictionary) GetDatabases

func (k *Dictionary) GetDatabases(ctx context.Context, tx transaction.Tx, namespaceId uint32,
) (map[string]*DatabaseMetadata, error)

func (*Dictionary) GetIndex

func (k *Dictionary) GetIndex(ctx context.Context, tx transaction.Tx, indexName string, namespaceId uint32,
	dbId uint32, collId uint32,
) (*IndexMetadata, error)

func (*Dictionary) GetIndexes

func (k *Dictionary) GetIndexes(ctx context.Context, tx transaction.Tx, namespaceId uint32, databaseId uint32,
	collId uint32,
) (map[string]*IndexMetadata, error)

func (*Dictionary) GetNamespaces

func (k *Dictionary) GetNamespaces(ctx context.Context, tx transaction.Tx,
) (map[string]NamespaceMetadata, error)

func (*Dictionary) Index

func (k *Dictionary) Index() *IndexSubspace

func (*Dictionary) Namespace

func (k *Dictionary) Namespace() *NamespaceSubspace

func (*Dictionary) ReserveNamespace

func (k *Dictionary) ReserveNamespace(ctx context.Context, tx transaction.Tx, namespaceId string,
	namespaceMetadata NamespaceMetadata,
) error

ReserveNamespace is the first step in the encoding and the mapping is passed the caller. As this is the first encoded integer the caller needs to make sure a unique value is assigned to this namespace.

func (*Dictionary) Schema

func (k *Dictionary) Schema() *SchemaSubspace

func (*Dictionary) SearchSchema

func (k *Dictionary) SearchSchema() *SearchSchemaSubspace

type Encoder

type Encoder interface {
	SearchEncoder

	// EncodeTableName returns encoded bytes which are formed by combining namespace, database, and collection.
	EncodeTableName(ns Namespace, db *Database, coll *schema.DefaultCollection) ([]byte, error)
	// EncodeSecondaryIndexTableName returns encoded bytes for the table name of a collections secondary index.
	EncodeSecondaryIndexTableName(ns Namespace, db *Database, coll *schema.DefaultCollection) ([]byte, error)
	EncodePartitionTableName(ns Namespace, db *Database, coll *schema.DefaultCollection) ([]byte, error)
	// EncodeIndexName returns encoded bytes for the index name
	EncodeIndexName(idx *schema.Index) []byte
	// EncodeKey returns encoded bytes of the key which will be used to store the values in fdb. The Key return by this
	// method has two parts,
	//   - tableName: This is set with an encoding of namespace, database and collection id.
	//   - IndexParts: This has the index identifier and value(s) associated with a single or composite index. This is appended
	//	   to the table name to form the Key. The first element of this list is the dictionary encoding of index type key
	//	   information i.e. whether the index is pkey, etc. The remaining elements are values for this index.
	EncodeKey(encodedTable []byte, idx *schema.Index, idxParts []interface{}) (keys.Key, error)

	// DecodeTableName is used to decode the key stored in FDB and extract namespace name, database name and collection ids.
	DecodeTableName(tableName []byte) (uint32, uint32, uint32, bool)
	DecodeIndexName(indexName []byte) uint32
}

Encoder is used to encode/decode values of the Key.

func NewEncoder

func NewEncoder() Encoder

NewEncoder creates Dictionary metaStore to encode keys.

type Error

type Error struct {
	// contains filtered or unexported fields
}

func (Error) Code

func (e Error) Code() ErrorCode

func (Error) Error

func (e Error) Error() string

type ErrorCode

type ErrorCode byte
const (
	ErrCodeDatabaseNotFound     ErrorCode = 0x00
	ErrCodeDatabaseExists       ErrorCode = 0x01
	ErrCodeDatabaseBranchExists ErrorCode = 0x02
	ErrCodeBranchNotFound       ErrorCode = 0x03
	ErrCodeCannotDeleteBranch   ErrorCode = 0x04
	ErrCodeProjectNotFound      ErrorCode = 0x05
	ErrCodeSearchIndexExists    ErrorCode = 0x06
	ErrCodeSearchIndexNotFound  ErrorCode = 0x07
	ErrCodeCacheExists          ErrorCode = 0x08
	ErrCodeCacheNotFound        ErrorCode = 0x09

	ErrDBMismatch     ErrorCode = 0x0A
	ErrBranchMismatch ErrorCode = 0x0B
)

type IndexMetadata

type IndexMetadata struct {
	ID   uint32 `json:"id"`
	Name string `json:"name"`
}

IndexMetadata contains index wide metadata.

type IndexSubspace

type IndexSubspace struct {
	// contains filtered or unexported fields
}

IndexSubspace is used to store metadata about Tigris secondary indexes.

func (*IndexSubspace) Get

func (c *IndexSubspace) Get(ctx context.Context, tx transaction.Tx, nsID uint32, dbID uint32, collID uint32, name string) (*IndexMetadata, error)

func (*IndexSubspace) Update

func (c *IndexSubspace) Update(ctx context.Context, tx transaction.Tx, nsID uint32, dbID uint32, collID uint32, name string, metadata *IndexMetadata) error

type NameRegistry

type NameRegistry struct {
	// ReservedSubspaceName is the name of the table(subspace) where all the counters are stored.
	ReserveSB string
	// EncodingSubspaceName is the name of the table(subspace) which is used by the dictionary encoder to store all the
	// dictionary encoded values.
	EncodingSB string
	// SchemaSubspaceName (the schema subspace) will be storing the actual schema of the user for a collection. The schema subspace will
	// look like below
	//    ["schema", 0x01, x, 0x01, 0x03, "created", 0x01] => {"title": "t1", properties: {"a": int}, primary_key: ["a"]}
	//
	//  where,
	//    - schema is the keyword for this table.
	//    - 0x01 is the schema subspace version
	//    - x is the value assigned for the namespace
	//    - 0x01 is the value for the database.
	//    - 0x03 is the value for the collection.
	//    - "created" is keyword.
	//
	SchemaSB    string
	SearchSB    string
	UserSB      string
	NamespaceSB string
	ClusterSB   string
	VersionKey  string
	QueueSB     string

	BaseCounterValue uint32
}

NameRegistry is used by tests to inject table names that can be used by tests. NameRegistry provides the names of the internal tables(subspaces) maintained by the metadata package. The interface helps in creating test tables for these structures.

func (*NameRegistry) ClusterSubspaceName

func (d *NameRegistry) ClusterSubspaceName() []byte

func (*NameRegistry) EncodingSubspaceName

func (d *NameRegistry) EncodingSubspaceName() []byte

func (*NameRegistry) GetVersionKey

func (d *NameRegistry) GetVersionKey() []byte

func (*NameRegistry) NamespaceSubspaceName

func (d *NameRegistry) NamespaceSubspaceName() []byte

func (*NameRegistry) QueueSubspaceName

func (d *NameRegistry) QueueSubspaceName() []byte

func (*NameRegistry) ReservedSubspaceName

func (d *NameRegistry) ReservedSubspaceName() []byte

func (*NameRegistry) SchemaSubspaceName

func (d *NameRegistry) SchemaSubspaceName() []byte

func (*NameRegistry) SearchSchemaSubspaceName

func (d *NameRegistry) SearchSchemaSubspaceName() []byte

func (*NameRegistry) UserSubspaceName

func (d *NameRegistry) UserSubspaceName() []byte

type Namespace

type Namespace interface {
	// Id for the namespace is used by the cluster to append as the first element in the key.
	Id() uint32
	// StrId is the name used for the lookup.
	StrId() string
	// Metadata for the namespace
	Metadata() NamespaceMetadata
}

A Namespace is a logical grouping of databases.

type NamespaceMetadata

type NamespaceMetadata struct {
	// unique namespace Id
	Id uint32
	// unique namespace name StrId
	StrId string
	// displayName for the namespace
	Name string
}

NamespaceMetadata - This structure is persisted as the namespace in DB.

func NewNamespaceMetadata

func NewNamespaceMetadata(id uint32, name string, displayName string) NamespaceMetadata

type NamespaceSubspace

type NamespaceSubspace struct {
	// contains filtered or unexported fields
}

NamespaceSubspace is used to store metadata about Tigris namespaces.

func NewNamespaceStore

func NewNamespaceStore(mdNameRegistry *NameRegistry) *NamespaceSubspace

func (*NamespaceSubspace) DeleteNamespace

func (n *NamespaceSubspace) DeleteNamespace(ctx context.Context, tx transaction.Tx, namespaceId uint32) error

func (*NamespaceSubspace) DeleteNamespaceMetadata

func (n *NamespaceSubspace) DeleteNamespaceMetadata(ctx context.Context, tx transaction.Tx, namespaceId uint32,
	metadataKey string,
) error

func (*NamespaceSubspace) DeleteProjectMetadata

func (n *NamespaceSubspace) DeleteProjectMetadata(ctx context.Context, tx transaction.Tx, namespaceId uint32,
	projName string,
) error

func (*NamespaceSubspace) GetNamespaceMetadata

func (n *NamespaceSubspace) GetNamespaceMetadata(ctx context.Context, tx transaction.Tx, namespaceId uint32,
	metadataKey string,
) ([]byte, error)

func (*NamespaceSubspace) GetProjectMetadata

func (n *NamespaceSubspace) GetProjectMetadata(ctx context.Context, tx transaction.Tx, namespaceId uint32,
	projName string,
) (*ProjectMetadata, error)

func (*NamespaceSubspace) InsertNamespaceMetadata

func (n *NamespaceSubspace) InsertNamespaceMetadata(ctx context.Context, tx transaction.Tx, namespaceId uint32,
	metadataKey string, payload []byte,
) error

func (*NamespaceSubspace) InsertProjectMetadata

func (n *NamespaceSubspace) InsertProjectMetadata(ctx context.Context, tx transaction.Tx, namespaceId uint32,
	projName string, projMetadata *ProjectMetadata,
) error

func (*NamespaceSubspace) UpdateNamespaceMetadata

func (n *NamespaceSubspace) UpdateNamespaceMetadata(ctx context.Context, tx transaction.Tx, namespaceId uint32,
	metadataKey string, payload []byte,
) error

func (*NamespaceSubspace) UpdateProjectMetadata

func (n *NamespaceSubspace) UpdateProjectMetadata(ctx context.Context, tx transaction.Tx, namespaceId uint32,
	projName string, projMetadata *ProjectMetadata,
) error

type NamespaceType

type NamespaceType string

type Project

type Project struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func NewProject

func NewProject(id uint32, name string) *Project

NewProject is to create a project, this is only done during reloading from the database as tenant attaches the main database and branches to this object.

func (*Project) GetDatabase

func (p *Project) GetDatabase(databaseName *DatabaseName) (*Database, error)

GetDatabase returns either the main database or a database branch. This depends on the DatabaseName object.

func (*Project) GetDatabaseWithBranches

func (p *Project) GetDatabaseWithBranches() []*Database

GetDatabaseWithBranches returns main database and all the corresponding database branches.

func (*Project) GetMainDatabase

func (p *Project) GetMainDatabase() *Database

GetMainDatabase returns the main database of this project.

func (*Project) GetSearch

func (p *Project) GetSearch() *Search

GetSearch returns the search for this project which will have all search indexes.

func (*Project) Id

func (p *Project) Id() uint32

Id returns the dictionary encoded value of the main database of this project.

func (*Project) Name

func (p *Project) Name() string

Name returns the project name.

type ProjectMetadata

type ProjectMetadata struct {
	ID             uint32
	Creator        string
	CreatedAt      int64
	CachesMetadata []CacheMetadata
	SearchMetadata []SearchMetadata
}

type QueueItem

type QueueItem struct {
	Id         string    `json:"id"`
	Priority   int64     `json:"priority"`
	ErrorCount uint8     `json:"error_count"`
	LeaseId    string    `json:"lease_id,omitempty"`
	Data       []byte    `json:"data"`
	Vesting    time.Time `json:"vesting_time"`
}

func NewQueueItem

func NewQueueItem(priority int64, data []byte) *QueueItem

type QueueSubspace

type QueueSubspace struct {
	// contains filtered or unexported fields
}

func (*QueueSubspace) Complete

func (q *QueueSubspace) Complete(ctx context.Context, tx transaction.Tx, item *QueueItem) error

func (*QueueSubspace) Dequeue

func (q *QueueSubspace) Dequeue(ctx context.Context, tx transaction.Tx, item *QueueItem) error

func (*QueueSubspace) Enqueue

func (q *QueueSubspace) Enqueue(ctx context.Context, tx transaction.Tx, item *QueueItem, delay time.Duration) error

func (*QueueSubspace) Find

func (q *QueueSubspace) Find(ctx context.Context, tx transaction.Tx, item *QueueItem) (*QueueItem, error)

func (*QueueSubspace) ObtainLease

func (q *QueueSubspace) ObtainLease(ctx context.Context, tx transaction.Tx, item *QueueItem, leaseTime time.Duration) (*QueueItem, error)

func (*QueueSubspace) Peek

func (q *QueueSubspace) Peek(ctx context.Context, tx transaction.Tx, max int) ([]QueueItem, error)

func (*QueueSubspace) RenewLease

func (q *QueueSubspace) RenewLease(ctx context.Context, tx transaction.Tx, item *QueueItem, leaseTime time.Duration) error

type SchemaSubspace

type SchemaSubspace struct {
	SubspaceName []byte
	// contains filtered or unexported fields
}

SchemaSubspace is used to manage schemas in schema subspace.

func NewSchemaStore

func NewSchemaStore(mdNameRegistry *NameRegistry) *SchemaSubspace

func (*SchemaSubspace) Delete

func (s *SchemaSubspace) Delete(ctx context.Context, tx transaction.Tx, namespaceId uint32, dbId uint32, collId uint32) error

Delete is to remove schema for a given namespace, database and collection.

func (*SchemaSubspace) Get

func (s *SchemaSubspace) Get(ctx context.Context, tx transaction.Tx, namespaceId uint32, dbId uint32, collId uint32) (schema.Versions, error)

Get returns all the version stored for a collection inside a given namespace and database.

func (*SchemaSubspace) GetLatest

func (s *SchemaSubspace) GetLatest(ctx context.Context, tx transaction.Tx, namespaceId uint32, dbId uint32, collId uint32) (*schema.Version, error)

GetLatest returns the latest version stored for a collection inside a given namespace and database.

func (*SchemaSubspace) Put

func (s *SchemaSubspace) Put(ctx context.Context, tx transaction.Tx, namespaceId uint32, dbId uint32, collId uint32, schema []byte, revision int) error

Put is to persist schema for a given namespace, database and collection.

type Search struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

Search is to manage all the search indexes that are explicitly created by the user.

func NewSearch

func NewSearch() *Search

func (*Search) AddIndex

func (s *Search) AddIndex(index *schema.SearchIndex)

func (*Search) GetIndex

func (s *Search) GetIndex(name string) (*schema.SearchIndex, bool)

func (*Search) GetIndexes

func (s *Search) GetIndexes() []*schema.SearchIndex

type SearchEncoder

type SearchEncoder interface {
	// EncodeSearchTableName will encode search index created by the user and return an encoded string that will be use
	// as an index name in the underlying search store.
	EncodeSearchTableName(tenantId uint32, projId uint32, indexName string) string
	// DecodeSearchTableName will decode the information from encoded search index Name. This method returns tenant id,
	// project id, index name.
	DecodeSearchTableName(name string) (uint32, uint32, string, bool)
	// EncodeFDBSearchTableName is the search index table in FDB
	EncodeFDBSearchTableName(searchTable string) []byte
	// EncodeFDBSearchKey is the search row-key in FDB
	EncodeFDBSearchKey(searchTable string, id string) (keys.Key, error)
}

type SearchMetadata

type SearchMetadata struct {
	Name      string
	Creator   string
	CreatedAt int64
}

type SearchSchemaSubspace

type SearchSchemaSubspace struct {
	SubspaceName []byte
	// contains filtered or unexported fields
}

SearchSchemaSubspace is used to manage search schemas.

func NewSearchSchemaStore

func NewSearchSchemaStore(mdNameRegistry *NameRegistry) *SearchSchemaSubspace

func (*SearchSchemaSubspace) Delete

func (s *SearchSchemaSubspace) Delete(ctx context.Context, tx transaction.Tx, namespaceId uint32, dbId uint32, index string) error

Delete is to remove schema for a given namespace, database and collection.

func (*SearchSchemaSubspace) Get

func (s *SearchSchemaSubspace) Get(ctx context.Context, tx transaction.Tx, namespaceId uint32, dbId uint32, index string) (schema.Versions, error)

Get returns all the version stored for a collection inside a given namespace and database.

func (*SearchSchemaSubspace) GetLatest

func (s *SearchSchemaSubspace) GetLatest(ctx context.Context, tx transaction.Tx, namespaceId uint32, dbId uint32, index string) (*schema.Version, error)

GetLatest returns the latest version stored for a collection inside a given namespace and database.

func (*SearchSchemaSubspace) Put

func (s *SearchSchemaSubspace) Put(ctx context.Context, tx transaction.Tx, namespaceId uint32, dbId uint32, search string, schema []byte, revision int) error

Put is to persist schema for a given namespace, database and search index.

type TableKeyGenerator

type TableKeyGenerator struct{}

TableKeyGenerator is used to generated keys that may need persistence like counter.

func NewTableKeyGenerator

func NewTableKeyGenerator() *TableKeyGenerator

func (*TableKeyGenerator) GenerateCounter

func (g *TableKeyGenerator) GenerateCounter(ctx context.Context, txMgr *transaction.Manager, table []byte) (int32, error)

GenerateCounter is used to generate an id in a transaction for int32 field only. This is mainly used to guarantee uniqueness with auto-incremented ids, so what we are doing is reserving this id in storage before returning to the caller so that only one id is assigned to one caller.

type Tenant

type Tenant struct {
	sync.RWMutex

	Encoder Encoder

	TableKeyGenerator *TableKeyGenerator
	// contains filtered or unexported fields
}

Tenant is a logical grouping of databases. The tenant is used to manage all the databases that belongs to this tenant and the corresponding collections for these databases. Operations performed on the tenant object are thread-safe.

func NewTenant

func NewTenant(namespace Namespace, kvStore kv.TxStore, searchStore search.Store, dict *Dictionary,
	encoder Encoder, versionH *VersionHandler, currentVersion Version, _ *TableKeyGenerator,
) *Tenant

func (*Tenant) CollectionSize

func (tenant *Tenant) CollectionSize(ctx context.Context, db *Database, coll *schema.DefaultCollection) (int64, error)

CollectionSize returns approximate data size on disk for all the collections for the database provided by the caller.

func (*Tenant) CreateBranch

func (tenant *Tenant) CreateBranch(ctx context.Context, tx transaction.Tx, projName string, dbName *DatabaseName) error

CreateBranch is used to create a database branch. A database branch is essentially a schema-only copy of a database. A new database is created in the tenant namespace and all the collection schemas from primary database are created in this branch. A branch may drift overtime from the primary database.

func (*Tenant) CreateCache

func (tenant *Tenant) CreateCache(ctx context.Context, tx transaction.Tx, project string, cache string, currentSub string) (bool, error)

func (*Tenant) CreateCollection

func (tenant *Tenant) CreateCollection(ctx context.Context, tx transaction.Tx, database *Database, schFactory *schema.Factory) error

CreateCollection is to create a collection inside tenant namespace.

func (*Tenant) CreateProject

func (tenant *Tenant) CreateProject(ctx context.Context, tx transaction.Tx, projName string, projMetadata *ProjectMetadata) error

CreateProject is responsible for creating a Project. This includes creating a dictionary encoding entry for the main database that will be attached to this project. This method is not adding the entry to the tenant because the outer layer may still roll back the transaction. The session manager is bumping the metadata version once the commit is successful so reloading happens at the next call when a transaction sees a stale tenant version. This applies to the reloading mechanism on all the servers. It returns "true" If the project already exists, else "false" and an error. The project metadata if not nil is also added inside this transaction.

func (*Tenant) CreateSearchIndex

func (tenant *Tenant) CreateSearchIndex(ctx context.Context, tx transaction.Tx, project *Project, factory *schema.SearchFactory) error

func (*Tenant) DatabaseSize

func (tenant *Tenant) DatabaseSize(ctx context.Context, db *Database) (int64, error)

DatabaseSize returns approximate data size on disk for all the database for this tenant.

func (*Tenant) DeleteBranch

func (tenant *Tenant) DeleteBranch(ctx context.Context, tx transaction.Tx, projName string, dbBranch *DatabaseName) error

DeleteBranch is responsible for deleting a database branch. Throws error if database/branch does not exist or if 'main' branch is being deleted.

func (*Tenant) DeleteCache

func (tenant *Tenant) DeleteCache(ctx context.Context, tx transaction.Tx, project string, cache string) (bool, error)

func (*Tenant) DeleteProject

func (tenant *Tenant) DeleteProject(ctx context.Context, tx transaction.Tx, projName string) (bool, error)

DeleteProject is responsible for first dropping a dictionary encoding of the main database attached to this project and then adding a corresponding dropped encoding entry in the encoding table. This API returns "false" if the project doesn't exist so that caller can reason about it. DeleteProject is more involved than CreateProject as with deletion we also need to iterate over all the collections present in the main database and database branches and call drop collection on each one of them. Returns "False" if the project doesn't exist.

func (*Tenant) DeleteSearchIndex

func (tenant *Tenant) DeleteSearchIndex(ctx context.Context, tx transaction.Tx, project *Project, indexName string) error

func (*Tenant) DropCollection

func (tenant *Tenant) DropCollection(ctx context.Context, tx transaction.Tx, db *Database, collectionName string) error

DropCollection is to drop a collection and its associated indexes. It removes the "created" entry from the encoding subspace and adds a "dropped" entry for the same collection key.

func (*Tenant) GetNamespace

func (tenant *Tenant) GetNamespace() Namespace

GetNamespace returns the namespace of this tenant.

func (*Tenant) GetProject

func (tenant *Tenant) GetProject(projName string) (*Project, error)

GetProject returns the project object, or null if there is no project with the name passed in the param. As reloading of tenant state is happening at the session manager layer so GetProject calls assume that the caller just needs the state from the cache.

func (*Tenant) GetSearchIndex

func (tenant *Tenant) GetSearchIndex(_ context.Context, _ transaction.Tx, project *Project, indexName string) (*schema.SearchIndex, error)

func (*Tenant) ListCaches

func (tenant *Tenant) ListCaches(ctx context.Context, tx transaction.Tx, project string) ([]string, error)

func (*Tenant) ListDatabaseBranches

func (tenant *Tenant) ListDatabaseBranches(projName string) []string

ListDatabaseBranches returns an array of branch names associated with this database including "main" branch.

func (*Tenant) ListProjects

func (tenant *Tenant) ListProjects(_ context.Context) []string

ListProjects is used to list all projects available for this tenant.

func (*Tenant) ListSearchIndexes

func (tenant *Tenant) ListSearchIndexes(_ context.Context, _ transaction.Tx, project *Project) ([]*schema.SearchIndex, error)

func (*Tenant) Reload

func (tenant *Tenant) Reload(ctx context.Context, tx transaction.Tx, version Version) error

Reload is used to reload this tenant. The reload method compares the currently attached version to the tenant to the version passed in the API call to detect whether reloading is needed. This check is needed to ensure only a single thread will actually perform reload. This is a blocking API which means if most of the requests detected that the tenant state is stale then they all will block till one of them will reload the tenant state from the database. All the blocking transactions will be restarted to ensure they see the latest view of the tenant.

func (*Tenant) Size

func (tenant *Tenant) Size(ctx context.Context) (int64, error)

Size returns approximate data size on disk for all the collections, databases for this tenant.

func (*Tenant) String

func (tenant *Tenant) String() string

type TenantGetter

type TenantGetter interface {
	GetTenant(ctx context.Context, id string) (*Tenant, error)
}

type TenantManager

type TenantManager struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

TenantManager is to manage all the tenants ToDo: start a background thread to reload the mapping.

func NewTenantManager

func NewTenantManager(kvStore kv.TxStore, searchStore search.Store, txMgr *transaction.Manager) *TenantManager

func NewTestTenantMgr

func NewTestTenantMgr(t *testing.T, kvStore kv.TxStore) (*TenantManager, context.Context, context.CancelFunc)

NewTestTenantMgr creates new TenantManager for tests.

func (*TenantManager) CreateOrGetTenant

func (m *TenantManager) CreateOrGetTenant(ctx context.Context, namespace Namespace) (tenant *Tenant, err error)

CreateOrGetTenant is a thread safe implementation of creating a new tenant. It returns the tenant if it already exists. This is mainly returning the tenant to avoid calling "Get" again after creating the tenant. This method is expensive as it reloads the existing tenants from the disk if it sees the tenant is not present in the cache.

func (*TenantManager) CreateTenant

func (m *TenantManager) CreateTenant(ctx context.Context, tx transaction.Tx, namespace Namespace) (Namespace, error)

CreateTenant is a thread safe implementation of creating a new tenant. It returns an error if it already exists.

func (*TenantManager) DecodeTableName

func (m *TenantManager) DecodeTableName(tableName []byte) (string, *Database, string, bool)

func (*TenantManager) EnsureDefaultNamespace

func (m *TenantManager) EnsureDefaultNamespace() error

func (*TenantManager) GetEncoder

func (m *TenantManager) GetEncoder() Encoder

func (*TenantManager) GetNamespaceId

func (m *TenantManager) GetNamespaceId(namespaceName string) (uint32, error)

func (*TenantManager) GetNamespaceNames

func (m *TenantManager) GetNamespaceNames() []string

func (*TenantManager) GetNamespaceStore

func (m *TenantManager) GetNamespaceStore() *NamespaceSubspace

func (*TenantManager) GetTableFromIds

func (m *TenantManager) GetTableFromIds(tenantId uint32, dbId uint32, collId uint32) (string, *Database, string, bool)

GetTableFromIds returns tenant name, database object, collection name corresponding to their encoded ids.

func (*TenantManager) GetTenant

func (m *TenantManager) GetTenant(ctx context.Context, namespaceName string) (*Tenant, error)

GetTenant is responsible for returning the tenant from the cache. If the tenant is not available in the cache then this method will attempt to load it from the database and will update the tenant manager cache accordingly.

func (*TenantManager) GetVersionHandler

func (m *TenantManager) GetVersionHandler() *VersionHandler

func (*TenantManager) ListNamespaces

func (m *TenantManager) ListNamespaces(ctx context.Context, tx transaction.Tx) ([]Namespace, error)

ListNamespaces returns all the namespaces(tenants) exist in this cluster.

func (*TenantManager) Reload

func (m *TenantManager) Reload(ctx context.Context, tx transaction.Tx, collectionsInSearch map[string]*tsApi.CollectionResponse) error

Reload reads all the tenants exist in the database and builds an in-memory view of the manager to track the tenants. As this is an expensive call, the reloading happens only during the start of the server. It is possible that reloading fails during start time then we rely on each transaction to detect it and trigger reload. The consistency shouldn’t be impacted if we fail to load the in-memory view.

type TenantNamespace

type TenantNamespace struct {
	// contains filtered or unexported fields
}

TenantNamespace is used when there is a finer isolation of databases is needed. The caller provides a unique id and strId to this namespace which is used by the cluster to create a namespace.

func NewTenantNamespace

func NewTenantNamespace(name string, metadata NamespaceMetadata) *TenantNamespace

func (*TenantNamespace) Id

func (n *TenantNamespace) Id() uint32

Id returns assigned code for the namespace.

func (*TenantNamespace) Metadata

func (n *TenantNamespace) Metadata() NamespaceMetadata

Metadata returns assigned metadata for the namespace.

func (*TenantNamespace) StrId

func (n *TenantNamespace) StrId() string

StrId returns assigned id for the namespace.

type Tracker

type Tracker struct {
	// contains filtered or unexported fields
}

Tracker is an object attached to a transaction so that a transaction can identify if metadata is changed and whether it needs to restart the transaction. Tracker is not thread-safe and should be used only in a single session.

func (*Tracker) Stop

func (tracker *Tracker) Stop(ctx context.Context) (bool, error)

Stop is needed to stop the tracker and reload the tenant if needed.

type UserSubspace

type UserSubspace struct {
	// contains filtered or unexported fields
}

UserSubspace is used to store metadata about Tigris users.

func NewUserStore

func NewUserStore(mdNameRegistry *NameRegistry) *UserSubspace

func (*UserSubspace) DeleteUser

func (u *UserSubspace) DeleteUser(ctx context.Context, tx transaction.Tx, namespaceId uint32, userType UserType,
	userId string,
) error

func (*UserSubspace) DeleteUserMetadata

func (u *UserSubspace) DeleteUserMetadata(ctx context.Context, tx transaction.Tx, namespaceId uint32, userType UserType,
	userId string, metadataKey string,
) error

func (*UserSubspace) GetUserMetadata

func (u *UserSubspace) GetUserMetadata(ctx context.Context, tx transaction.Tx, namespaceId uint32, userType UserType,
	userId string, metadataKey string,
) ([]byte, error)

func (*UserSubspace) InsertUserMetadata

func (u *UserSubspace) InsertUserMetadata(ctx context.Context, tx transaction.Tx, namespaceId uint32, userType UserType,
	userId string, metadataKey string, payload []byte,
) error

func (*UserSubspace) UpdateUserMetadata

func (u *UserSubspace) UpdateUserMetadata(ctx context.Context, tx transaction.Tx, namespaceId uint32, userType UserType,
	userId string, metadataKey string, payload []byte,
) error

type UserType

type UserType uint32
const (
	User        UserType = 0
	Application UserType = 1
)

type Version

type Version []byte

type VersionFuture

type VersionFuture kv.Future

type VersionHandler

type VersionHandler struct {
	Key   []byte
	Value []byte
}

VersionHandler is used to maintain a version for each schema change. Using this we can implement transactional DDL APIs. This will also be used to provide a strongly consistent Cache lookup on the schemas i.e. anytime version changes we know that a DDL operation is performed which means we can invalidate the cache and reload from the disk.

func (*VersionHandler) Increment

func (m *VersionHandler) Increment(ctx context.Context, tx transaction.Tx) error

Increment is used to increment the metadata version.

func (*VersionHandler) Read

func (m *VersionHandler) Read(ctx context.Context, tx transaction.Tx, isSnapshot bool) (Version, error)

Read is blocking and returns the latest metadata version.

func (*VersionHandler) ReadFuture

func (m *VersionHandler) ReadFuture(ctx context.Context, tx transaction.Tx, isSnapshot bool) (VersionFuture, error)

ReadFuture is a non-blocking API to return the future corresponding to the latest metadata version.

func (*VersionHandler) ReadInOwnTxn

func (m *VersionHandler) ReadInOwnTxn(ctx context.Context, txMgr *transaction.Manager, isSnapshot bool) (version Version, err error)

ReadInOwnTxn creates a transaction and then reads the version. This is useful when a transaction is also changing the metadata then it is better to read the metadata version in its own transaction as the read-write-read or write-read metadata version is not allowed in a transaction.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL