schema

package
v1.0.0-beta.46 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 3, 2023 License: Apache-2.0 Imports: 28 Imported by: 0

Documentation

Index

Constants

View Source
const (
	PrimaryKeyIndexName = "pkey"
	AutoPrimaryKeyF     = "id"
	PrimaryKeySchemaK   = "primary_key"
	// DateTimeFormat represents the supported date time format.
	DateTimeFormat               = time.RFC3339Nano
	CollectionTypeF              = "collection_type"
	IndexingSchemaVersionKey     = "indexing_version"
	DefaultIndexingSchemaVersion = "v1"

	SecondaryKeyIndexName = "skey"
)
View Source
const (
	ObjFlattenDelimiter = "."
)
View Source
const (
	SearchId = "id"
)

Variables

View Source
var (
	MsgFieldNameAsLanguageKeyword = "Invalid collection field name, It contains language keyword for fieldName = '%s'"
	MsgFieldNameInvalidPattern    = "" /* 131-byte string literal not displayed */
	ValidFieldNamePattern         = regexp.MustCompile(`^[a-zA-Z_$][a-zA-Z0-9_$]*$`)
)
View Source
var (
	ErrIncompatibleSchema = fmt.Errorf("error incompatible schema")
	ErrExpectedString     = fmt.Errorf("expected string type")
	ErrExpectedNumber     = fmt.Errorf("expected json.Number")
	ErrUnsupportedType    = fmt.Errorf("unsupported type")
)

Supported subtypes.

View Source
var (
	ErrMissingField           = errors.InvalidArgument("removing a field is a backward incompatible change")
	ErrCollectionNameMismatch = errors.InvalidArgument("mismatch in the collection name")
	ErrIndexNameMismatch      = errors.InvalidArgument("mismatch in the index name")
)
View Source
var FieldNames = [...]string{
	UnknownType:  "unknown",
	NullType:     "null",
	BoolType:     "bool",
	Int32Type:    "int32",
	Int64Type:    "int64",
	DoubleType:   "double",
	StringType:   "string",
	ByteType:     "byte",
	UUIDType:     "uuid",
	DateTimeType: "datetime",
	ArrayType:    "array",
	ObjectType:   "object",
}
View Source
var ReservedFields = [...]string{
	CreatedAt:           "_tigris_created_at",
	UpdatedAt:           "_tigris_updated_at",
	Metadata:            "_tigris_metadata",
	IdToSearchKey:       "_tigris_id",
	DateSearchKeyPrefix: "_tigris_date_",
	SearchArrNullItem:   "_tigris_null",
}
View Source
var SupportedFieldProperties = container.NewHashSet(
	"type",
	"format",
	"items",
	"maxLength",
	"description",
	"contentEncoding",
	"properties",
	"autoGenerate",
	"sorted",
	"sort",
	"index",
	"facet",
	"searchIndex",
	"default",
	"createdAt",
	"updatedAt",
	"title",
	"required",
	"index",
	"facet",
	"searchIndex",
)

Functions

func ApplySchemaRules

func ApplySchemaRules(existing *DefaultCollection, current *Factory) error

ApplySchemaRules is to validate incoming collection request against the existing present collection. It performs following validations,

  • Primary Key Changed, or order of fields part of the primary key is changed
  • Collection name change
  • A validation on field property is also applied like for instance if existing field has some property, but it is removed in the new schema
  • Any index exist on the collection will also have same checks like type, etc

func ApplySearchIndexSchemaRules

func ApplySearchIndexSchemaRules(existing *SearchIndex, current *SearchFactory) error

ApplySearchIndexSchemaRules is to validate incoming index schema against the existing index version. It performs following validations,

  • Collection name change.
  • A validation on field like changing the type is not allowed.
  • A validation on index source, it is defined during index creation and should not change afterwards.

func DefaultSortableField

func DefaultSortableField(fieldType FieldType) bool

func FacetableField

func FacetableField(fieldType FieldType) bool

func Generate

func Generate(jsonSchema []byte, format string) ([]byte, error)

Generate schema in the requested format.

func GetSearchDeltaFields

func GetSearchDeltaFields(forSearchIndex bool, existingFields []*QueryableField, incomingFields []*Field, fieldsInSearch []tsApi.Field) []tsApi.Field

func IndexableField

func IndexableField(fieldType FieldType) bool

func Infer

func Infer(sch *schema.Schema, name string, docs [][]byte, primaryKey []string, autoGenerate []string,
	depth int,
) error

func IsPrimitiveType

func IsPrimitiveType(fieldType FieldType) bool

func IsReservedField

func IsReservedField(name string) bool

func IsSearchID

func IsSearchID(name string) bool

func IsValidKeyType

func IsValidKeyType(t FieldType) bool

func RemoveIndexingVersion

func RemoveIndexingVersion(schema jsoniter.RawMessage) jsoniter.RawMessage

func SearchIndexableField

func SearchIndexableField(fieldType FieldType, subType FieldType) bool

func SetIndexingVersion

func SetIndexingVersion(factory *Factory) error

func ToSearchDateKey

func ToSearchDateKey(key string) string

ToSearchDateKey can be used to generate storage field for search backend Original date strings are persisted as it is under this field.

func ValidateSearchSchema

func ValidateSearchSchema(factory *SearchFactory) error

Types

type CollectionType

type CollectionType string
const (
	DocumentsType CollectionType = "documents"
)

func GetCollectionType

func GetCollectionType(_ jsoniter.RawMessage) (CollectionType, error)

type DefaultCollection

type DefaultCollection struct {
	// Id is the dictionary encoded value for this collection.
	Id uint32
	// SchVer returns the schema version
	SchVer int
	// Name is the name of the collection.
	Name string
	// EncodedName is the encoded name of the collection.
	EncodedName []byte
	// EncodedTableIndexName is the encoded name of the collection's Secondary Index.
	EncodedTableIndexName []byte
	// Fields are derived from the user schema.
	Fields []*Field
	// Indexes is a wrapper on the indexes part of this collection.
	Indexes *Indexes
	// Validator is used to validate the JSON document. As it is expensive to create this, it is only created once
	// during constructor of the collection.
	Validator *jsonschema.Schema
	// JSON schema
	Schema jsoniter.RawMessage
	// SchemaDeltas contains incompatible schema changes from version to version
	SchemaDeltas []VersionDelta
	// FieldVersions contains the list of schema versions at which the field had incompatible change
	FieldVersions map[string]*FieldVersions
	// ImplicitSearchIndex is created by the Tigris to use a search index for in-memory indexes. This is needed till we move
	// to secondary indexes which will be stored in FDB.
	ImplicitSearchIndex *ImplicitSearchIndex
	// search indexes are indexes that are explicitly created by the user and tagged Tigris as source. Collection will be
	// responsible for ensuring these indexes are in sync when any mutation happens to this collection.
	SearchIndexes map[string]*SearchIndex
	// QueryableFields are similar to Fields but these are flattened forms of fields. For instance, a simple field
	// will be one to one mapped to queryable field but complex fields like object type field there may be more than
	// one queryableFields. As queryableFields represent a flattened state these can be used as-is to index in memory.
	QueryableFields []*QueryableField
	// CollectionType is the type of the collection. Only two types of collections are supported "messages" and "documents"
	CollectionType CollectionType

	// This is the existing fields in search
	FieldsInSearch []tsApi.Field
	// contains filtered or unexported fields
}

DefaultCollection is used to represent a collection. The tenant in the metadata package is responsible for creating the collection.

func NewDefaultCollection

func NewDefaultCollection(id uint32, schVer int, factory *Factory, schemas Versions,
	implicitSearchIndex *ImplicitSearchIndex,
) (*DefaultCollection, error)

func (*DefaultCollection) AddSearchIndex

func (d *DefaultCollection) AddSearchIndex(index *SearchIndex)

func (*DefaultCollection) CompatibleSchemaSince

func (d *DefaultCollection) CompatibleSchemaSince(version int32) bool

CompatibleSchemaSince determines if there was incompatible schema change since given version.

func (*DefaultCollection) GetField

func (d *DefaultCollection) GetField(name string) *Field

func (*DefaultCollection) GetFields

func (d *DefaultCollection) GetFields() []*Field

func (*DefaultCollection) GetImplicitSearchIndex

func (d *DefaultCollection) GetImplicitSearchIndex() *ImplicitSearchIndex

func (*DefaultCollection) GetIndexedFields

func (d *DefaultCollection) GetIndexedFields() []*QueryableField

func (*DefaultCollection) GetIndexes

func (d *DefaultCollection) GetIndexes() *Indexes

func (*DefaultCollection) GetInt64FieldsPath

func (d *DefaultCollection) GetInt64FieldsPath() map[string]struct{}

func (*DefaultCollection) GetName

func (d *DefaultCollection) GetName() string

func (*DefaultCollection) GetQueryableField

func (d *DefaultCollection) GetQueryableField(name string) (*QueryableField, error)

func (*DefaultCollection) GetQueryableFields

func (d *DefaultCollection) GetQueryableFields() []*QueryableField

func (*DefaultCollection) GetVersion

func (d *DefaultCollection) GetVersion() int32

func (*DefaultCollection) LookupFieldVersion

func (d *DefaultCollection) LookupFieldVersion(keyPath []string) []*FieldVersion

LookupFieldVersion returns the list of the schema versions for the specified keyPath, when this field had incompatible change. Returns nil if the field has never been changed, meaning that it has version 1.

func (*DefaultCollection) TaggedDefaultsForInsert

func (d *DefaultCollection) TaggedDefaultsForInsert() map[string]struct{}

func (*DefaultCollection) TaggedDefaultsForUpdate

func (d *DefaultCollection) TaggedDefaultsForUpdate() map[string]struct{}

func (*DefaultCollection) Type

func (*DefaultCollection) UpdateRowSchema

func (d *DefaultCollection) UpdateRowSchema(doc map[string]any, version int32)

UpdateRowSchema fixes the schema of provided, unmarshalled, document from given Version to the latest collection schema.

func (*DefaultCollection) UpdateRowSchemaRaw

func (d *DefaultCollection) UpdateRowSchemaRaw(doc []byte, version int32) ([]byte, error)

UpdateRowSchemaRaw fixes the schema of provided document from given Version to the latest collection schema.

func (*DefaultCollection) Validate

func (d *DefaultCollection) Validate(document interface{}) error

Validate expects an unmarshalled document which it will validate again the schema of this collection.

type Factory

type Factory struct {
	// Name is the collection name of this schema.
	Name string
	// Fields are derived from the user schema.
	Fields []*Field
	// Indexes is a wrapper on the indexes part of this collection. At this point the dictionary encoded value is not
	// set for these indexes which is set as part of collection creation.
	Indexes *Indexes
	// Schema is the raw JSON schema received as part of CreateOrUpdateCollection request. This is stored as-is in the
	// schema subspace.
	Schema jsoniter.RawMessage
	// CollectionType is the type of the collection. Only two types of collections are supported "messages" and "documents"
	CollectionType  CollectionType
	IndexingVersion string
	Version         int32
}

Factory is used as an intermediate step so that collection can be initialized with properly encoded values.

func Build

func Build(collection string, reqSchema jsoniter.RawMessage) (*Factory, error)

Build is used to deserialize the user json schema into a schema factory.

type Field

type Field struct {
	FieldName       string
	Defaulter       *FieldDefaulter
	DataType        FieldType
	MaxLength       *int32
	FillCreatedAt   *bool
	FillUpdatedAt   *bool
	UniqueKeyField  *bool
	PrimaryKeyField *bool
	AutoGenerated   *bool
	Sorted          *bool
	Indexed         *bool
	Faceted         *bool
	SearchIndexed   *bool
	// Nested fields are the fields where we know the schema of nested attributes like if properties are
	Fields []*Field
}

func (*Field) GetDefaulter

func (f *Field) GetDefaulter() *FieldDefaulter

func (*Field) GetNestedField

func (f *Field) GetNestedField(name string) *Field

func (*Field) IsAutoGenerated

func (f *Field) IsAutoGenerated() bool

func (*Field) IsCompatible

func (f *Field) IsCompatible(f1 *Field) error

func (*Field) IsIndexable

func (f *Field) IsIndexable() bool

func (*Field) IsPrimaryKey

func (f *Field) IsPrimaryKey() bool

func (*Field) IsSorted

func (f *Field) IsSorted() bool

func (*Field) Name

func (f *Field) Name() string

func (*Field) Type

func (f *Field) Type() FieldType

type FieldBuilder

type FieldBuilder struct {
	FieldName   string
	Description string              `json:"description,omitempty"`
	Type        string              `json:"type,omitempty"`
	Format      string              `json:"format,omitempty"`
	Encoding    string              `json:"contentEncoding,omitempty"`
	Default     interface{}         `json:"default,omitempty"`
	CreatedAt   *bool               `json:"createdAt,omitempty"`
	UpdatedAt   *bool               `json:"updatedAt,omitempty"`
	MaxLength   *int32              `json:"maxLength,omitempty"`
	Auto        *bool               `json:"autoGenerate,omitempty"`
	Sorted      *bool               `json:"sort,omitempty"`
	Index       *bool               `json:"index,omitempty"`
	Facet       *bool               `json:"facet,omitempty"`
	SearchIndex *bool               `json:"searchIndex,omitempty"`
	Items       *FieldBuilder       `json:"items,omitempty"`
	Properties  jsoniter.RawMessage `json:"properties,omitempty"`
	Primary     *bool
	Fields      []*Field
}

func (*FieldBuilder) Build

func (f *FieldBuilder) Build(isArrayElement bool) (*Field, error)

func (*FieldBuilder) Validate

func (f *FieldBuilder) Validate(v []byte) error

func (*FieldBuilder) ValidateIndexOnField

func (f *FieldBuilder) ValidateIndexOnField(field *Field) error

type FieldDefaulter

type FieldDefaulter struct {
	// contains filtered or unexported fields
}

func (*FieldDefaulter) GetValue

func (defaulter *FieldDefaulter) GetValue() interface{}

GetValue returns the value if there is default tag set for a field. For functions, it will execute it and return the value. If the function is now() then it is converted to createdAt during the construction of the defaulter.

func (*FieldDefaulter) TaggedWithCreatedAt

func (defaulter *FieldDefaulter) TaggedWithCreatedAt() bool

func (*FieldDefaulter) TaggedWithUpdatedAt

func (defaulter *FieldDefaulter) TaggedWithUpdatedAt() bool

type FieldSchemaValidator

type FieldSchemaValidator struct{}

func (*FieldSchemaValidator) Validate

func (v *FieldSchemaValidator) Validate(existing *DefaultCollection, current *Factory) error

func (*FieldSchemaValidator) ValidateIndex

func (v *FieldSchemaValidator) ValidateIndex(existing *SearchIndex, current *SearchFactory) error

type FieldType

type FieldType int
const (
	UnknownType FieldType = iota
	NullType
	BoolType
	Int32Type
	Int64Type
	DoubleType
	StringType
	// ByteType is a base64 encoded characters, this means if this type is used as key then we need to decode it
	// and then use it as key.
	ByteType
	UUIDType
	// DateTimeType is a valid date representation as defined by RFC 3339, see https://datatracker.ietf.org/doc/html/rfc3339#section-5.6
	DateTimeType
	ArrayType
	ObjectType
)

func ToFieldType

func ToFieldType(jsonType string, encoding string, format string) FieldType

type FieldVersion

type FieldVersion struct {
	Change  *VersionDeltaField
	Version int
}

FieldVersion contains individual field change, along with the schema version at which this change has happened.

type FieldVersions

type FieldVersions struct {
	// contains filtered or unexported fields
}

FieldVersions contains all the changes of the field, across all the schema versions.

type ImplicitSearchIndex

type ImplicitSearchIndex struct {
	// Name is the name of the index.
	Name string
	// StoreSchema is the search schema of the underlying search engine.
	StoreSchema *tsApi.CollectionSchema
	// QueryableFields are similar to Fields but these are flattened forms of fields. For instance, a simple field
	// will be one to one mapped to queryable field but complex fields like object type field there may be more than
	// one queryableFields. As queryableFields represent a flattened state these can be used as-is to index in memory.
	QueryableFields []*QueryableField
	// contains filtered or unexported fields
}

ImplicitSearchIndex is a search index that is automatically created by Tigris when a collection is created. Lifecycle of this index is tied to the collection.

func NewImplicitSearchIndex

func NewImplicitSearchIndex(name string, searchStoreName string, fields []*Field, fieldsInSearch []tsApi.Field) *ImplicitSearchIndex

func (*ImplicitSearchIndex) StoreIndexName

func (s *ImplicitSearchIndex) StoreIndexName() string

type Index

type Index struct {
	// Fields that are part of this index. An index can have a single or composite fields.
	Fields []*Field
	// Name is used by dictionary encoder for this index.
	Name string
	// Id is assigned to this index by the dictionary encoder.
	Id uint32
}

Index can be composite, so it has a list of fields, each index has name and encoded id. The encoded is used for key building.

func (*Index) IsCompatible

func (i *Index) IsCompatible(i1 *Index) error

type IndexSourceValidator

type IndexSourceValidator struct{}

func (*IndexSourceValidator) ValidateIndex

func (v *IndexSourceValidator) ValidateIndex(existing *SearchIndex, current *SearchFactory) error

type Indexes

type Indexes struct {
	PrimaryKey     *Index
	SecondaryIndex *Index
}

Indexes is to wrap different index that a collection can have.

func (*Indexes) GetIndexes

func (i *Indexes) GetIndexes() []*Index

type JSONSchema

type JSONSchema struct {
	Name            string              `json:"title,omitempty"`
	Description     string              `json:"description,omitempty"`
	Properties      jsoniter.RawMessage `json:"properties,omitempty"`
	PrimaryKeys     []string            `json:"primary_key,omitempty"`
	CollectionType  string              `json:"collection_type,omitempty"`
	IndexingVersion string              `json:"indexing_version,omitempty"`
	Version         int32               `json:"version,omitempty"`
}

type PrimaryIndexSchemaValidator

type PrimaryIndexSchemaValidator struct{}

func (*PrimaryIndexSchemaValidator) Validate

func (v *PrimaryIndexSchemaValidator) Validate(existing *DefaultCollection, current *Factory) error

type QueryableField

type QueryableField struct {
	FieldName     string
	Indexed       bool // Secondary Index
	InMemoryAlias string
	Faceted       bool
	SearchIndexed bool
	Sortable      bool
	DataType      FieldType
	SubType       FieldType
	SearchType    string
	// contains filtered or unexported fields
}

func (*QueryableField) InMemoryName

func (q *QueryableField) InMemoryName() string

InMemoryName returns key name that is used to index this field in the indexing store. For example, an "id" key is indexed with "_tigris_id" name.

func (*QueryableField) IsReserved

func (q *QueryableField) IsReserved() bool

IsReserved returns true if the queryable field is internal field.

func (*QueryableField) KeyPath

func (q *QueryableField) KeyPath() []string

func (*QueryableField) Name

func (q *QueryableField) Name() string

Name returns the name of this field as defined in the schema.

func (*QueryableField) ShouldPack

func (q *QueryableField) ShouldPack() bool

ShouldPack returns true if we need to pack this field before sending to indexing store.

func (*QueryableField) Type

func (q *QueryableField) Type() FieldType

Type returns the data type of this field.

type QueryableFieldsBuilder

type QueryableFieldsBuilder struct {
	ForSearchIndex bool
}

func NewQueryableFieldsBuilder

func NewQueryableFieldsBuilder(forSearchIndex bool) *QueryableFieldsBuilder

func (*QueryableFieldsBuilder) BuildQueryableFields

func (builder *QueryableFieldsBuilder) BuildQueryableFields(fields []*Field, fieldsInSearch []tsApi.Field) []*QueryableField

func (*QueryableFieldsBuilder) NewQueryableField

func (builder *QueryableFieldsBuilder) NewQueryableField(name string, f *Field, fieldsInSearch []tsApi.Field) *QueryableField

type ReservedField

type ReservedField uint8
const (
	CreatedAt ReservedField = iota
	UpdatedAt
	Metadata
	IdToSearchKey
	DateSearchKeyPrefix
	SearchArrNullItem
)

type SearchFactory

type SearchFactory struct {
	// Name is the index name.
	Name string
	// Fields are derived from the user schema.
	Fields []*Field
	// Schema is the raw JSON schema received
	Schema jsoniter.RawMessage
	Sub    string
	Source SearchSource
}

SearchFactory is used as an intermediate step so that collection can be initialized with properly encoded values.

func BuildSearch

func BuildSearch(index string, reqSchema jsoniter.RawMessage) (*SearchFactory, error)

type SearchIndex

type SearchIndex struct {
	// Name is the name of the index.
	Name string
	// index version
	Version int
	// Fields are derived from the user schema.
	Fields []*Field
	// JSON schema.
	Schema jsoniter.RawMessage
	// StoreSchema is the search schema of the underlying search engine.
	StoreSchema *tsApi.CollectionSchema
	// QueryableFields are similar to Fields but these are flattened forms of fields. For instance, a simple field
	// will be one to one mapped to queryable field but complex fields like object type field there may be more than
	// one queryableFields. As queryableFields represent a flattened state these can be used as-is to index in memory.
	QueryableFields []*QueryableField
	// Source of this index
	Source SearchSource
}

SearchIndex is to manage search index created by the user.

func NewSearchIndex

func NewSearchIndex(ver int, searchStoreName string, factory *SearchFactory, fieldsInSearch []tsApi.Field) *SearchIndex

func (*SearchIndex) GetQueryableField

func (s *SearchIndex) GetQueryableField(name string) (*QueryableField, error)

func (*SearchIndex) StoreIndexName

func (s *SearchIndex) StoreIndexName() string

type SearchIndexValidator

type SearchIndexValidator interface {
	ValidateIndex(existing *SearchIndex, current *SearchFactory) error
}

type SearchJSONSchema

type SearchJSONSchema struct {
	Name        string              `json:"title,omitempty"`
	Description string              `json:"description,omitempty"`
	Properties  jsoniter.RawMessage `json:"properties,omitempty"`
	Source      *SearchSource       `json:"source,omitempty"`
}

type SearchSource

type SearchSource struct {
	// Type is the source type i.e. either it is Tigris or the index will be maintained by the user.
	Type SearchSourceType `json:"type,omitempty"`
	// CollectionName is the source name i.e. collection name in case of Tigris otherwise it is optional.
	CollectionName string `json:"collection,omitempty"`
	// DatabaseBranch is in case the collection is part of a database branch. Only applicable if Type is Tigris.
	DatabaseBranch string `json:"branch,omitempty"`
}

type SearchSourceType

type SearchSourceType string
const (
	// SearchSourceTigris is when the source type is Tigris for the search index.
	SearchSourceTigris SearchSourceType = "tigris"
	// SearchSourceExternal is when the source type is external for the search index.
	SearchSourceExternal SearchSourceType = "external"
)

type Validator

type Validator interface {
	Validate(existing *DefaultCollection, current *Factory) error
}

type Version

type Version struct {
	Version int
	Schema  []byte
}

Version is schema associated with it version.

type VersionDelta

type VersionDelta struct {
	Version int
	Fields  []*VersionDeltaField
}

VersionDelta contains all fields schema changes in particular schema Version.

type VersionDeltaField

type VersionDeltaField struct {
	KeyPath []string  // Key path of the field
	From    FieldType // The type is changing from this
	To      FieldType // to this

	MaxLength int
}

VersionDeltaField describes field schema change.

If field is deleted then To is equal to `UnknownType`. If field is added then From is equal to `UnknownType`.

type Versions

type Versions []Version

Versions is an array of all collections schemas. Sorted in ascending schema version order.

func (Versions) Latest

func (x Versions) Latest() Version

func (Versions) Len

func (x Versions) Len() int

func (Versions) Less

func (x Versions) Less(i, j int) bool

func (Versions) Swap

func (x Versions) Swap(i, j int)

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL