collection

package
v0.10.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 11, 2024 License: Apache-2.0 Imports: 28 Imported by: 3

Documentation

Index

Constants

View Source 
const (

	// CSVHeaderKey is the key used to store the header of the csv file
	CSVHeaderKey = "__csv_header__"
)
View Source 
const (

	// DefaultIndexFieldName is the default index identifier
	DefaultIndexFieldName = "id"
)

Variables

View Source 
var (
	// ErrEmptyIndex is returned when the index is empty
	ErrEmptyIndex = errors.New("empty Index")
	// ErrEntryNotFound is returned when the entry is not found
	ErrEntryNotFound = errors.New("entry not found")
	// ErrNoNextElement is returned when there is no next element
	ErrNoNextElement = errors.New("no next element")
	// ErrNoManifestFound is returned when there is no manifest found
	ErrNoManifestFound = errors.New("no Manifest found")
	// ErrManifestUnmarshall is returned when the manifest cannot be unmarshalled
	ErrManifestUnmarshall = errors.New("could not unmarshall Manifest")
	// ErrManifestCreate is returned when the manifest cannot be created
	ErrManifestCreate = errors.New("could not create new Manifest")
	// ErrDeleteingIndex is returned when the index cannot be deleted
	ErrDeleteingIndex = errors.New("could not delete index")
	// ErrIndexAlreadyPresent is returned when the index is already present
	ErrIndexAlreadyPresent = errors.New("index already present")
	// ErrIndexNotPresent is returned when the index is not present
	ErrIndexNotPresent = errors.New("index not present")
	// ErrIndexNotSupported is returned when the index is not supported
	ErrIndexNotSupported = errors.New("index not supported")
	// ErrInvalidIndexType is returned when the index type is invalid
	ErrInvalidIndexType = errors.New("invalid index type")
	// ErrKvTableAlreadyPresent is returned when the kv table is already present
	ErrKvTableAlreadyPresent = errors.New("kv table already present")
	// ErrKVTableNotPresent is returned when the kv table is not present
	ErrKVTableNotPresent = errors.New("kv table not present")
	// ErrKVTableNotOpened is returned when the kv table is not opened
	ErrKVTableNotOpened = errors.New("kv table not opened")
	// ErrKVInvalidIndexType is returned when the kv index type is invalid
	ErrKVInvalidIndexType = errors.New("kv invalid index type")
	// ErrKVNilIterator is returned when the kv iterator is nil
	ErrKVNilIterator = errors.New("iterator not set, seek first")
	// ErrKVIndexTypeNotSupported is returned when the kv index type is not supported
	ErrKVIndexTypeNotSupported = errors.New("kv index type not supported yet")
	// ErrKVKeyNotANumber is returned when the kv key is not a number
	ErrKVKeyNotANumber = errors.New("kv key not a number")
	// ErrUnmarshallingDBSchema is returned when the db schema cannot be unmarshalled
	ErrUnmarshallingDBSchema = errors.New("could not unmarshall document db schema")
	// ErrMarshallingDBSchema is returned when the db schema cannot be marshalled
	ErrMarshallingDBSchema = errors.New("could not marshall document db schema")
	// ErrDocumentDBAlreadyPresent is returned when the document db is already present
	ErrDocumentDBAlreadyPresent = errors.New("document db already present")
	// ErrDocumentDBNotPresent is returned when the document db is not present
	ErrDocumentDBNotPresent = errors.New("document db  not present")
	// ErrDocumentDBNotOpened is returned when the document db is not opened
	ErrDocumentDBNotOpened = errors.New("document db not opened")
	// ErrDocumentDBAlreadyOpened is returned when the document db is already opened
	ErrDocumentDBAlreadyOpened = errors.New("document db already opened")
	// ErrDocumentDBIndexFieldNotPresent is returned when the document db index field is not present
	ErrDocumentDBIndexFieldNotPresent = errors.New("document db index field not present")
	// ErrModifyingImmutableDocDB is returned when the document db is immutable
	ErrModifyingImmutableDocDB = errors.New("trying to modify immutable document db")
	// ErrInvalidOperator is returned when the operator is invalid
	ErrInvalidOperator = errors.New("invalid operator")
	// ErrDocumentNotPresent is returned when the document is not present
	ErrDocumentNotPresent = errors.New("document not present")
	// ErrInvalidDocumentId is returned when the document id is invalid
	ErrInvalidDocumentId = errors.New("invalid document id")
	// ErrReadOnlyIndex is returned when the index is read only
	ErrReadOnlyIndex = errors.New("read only index")
	// ErrCannotModifyImmutableIndex is returned when the index is immutable
	ErrCannotModifyImmutableIndex = errors.New("trying to modify immutable index")
	// ErrUnknownJsonFormat is returned when the json format is unknown
	ErrUnknownJsonFormat = errors.New("unknown json format")
)
View Source 
var (
	//  NoOfParallelWorkers is the number of parallel workers to be used for index creation
	NoOfParallelWorkers = runtime.NumCPU() * 4
)

Functions

func CreateIndex 

func CreateIndex(podName, collectionName, indexName, encryptionPassword string, indexType IndexType, fd *feed.API, user utils.Address, client blockstore.Client, mutable bool) error

CreateIndex creates a common index file to be used in kv or document tables.

func NormalizedDistance added in v0.10.0 

func NormalizedDistance(a, b []float32) (float32, error)

NormalizedDistance between two arbitrary vectors, errors if dimensions don't match, will return results between 0 (no distance) and 1 (maximum distance)

Types

type Batch 

type Batch struct {
	// contains filtered or unexported fields
}

Batch is to be used in KV table or a Document database

func NewBatch 

func NewBatch(idx *Index) (*Batch, error)

NewBatch creates a new batch index to be used in a KV table or a Document database.

func (*Batch) Del 

func (b *Batch) Del(key string) ([][]byte, error)

Del deletes a index entry.

func (*Batch) DelNumber 

func (b *Batch) DelNumber(key float64) ([][]byte, error)

DelNumber deletes a number index key and value. skipcq: TCV-001

func (*Batch) Get 

func (b *Batch) Get(key string) ([][]byte, error)

Get extracts an index value from an index given a key.

func (*Batch) Put 

func (b *Batch) Put(key string, value []byte, apnd, memory bool) error

Put creates an index entry given a key string and value.

func (*Batch) PutNumber 

func (b *Batch) PutNumber(key float64, refValue []byte, apnd, memory bool) error

PutNumber inserts index as a number.

func (*Batch) Write 

func (b *Batch) Write(podFile string) (*Manifest, error)

Write commits the raw index file in to the Swarm network.

type CIndex 

type CIndex struct {
	SimpleIndexes []SIndex
}

CIndex is a compound index

type DBSchema 

type DBSchema struct {
	Name            string   `json:"name"`
	Mutable         bool     `json:"mutable"`
	SimpleIndexes   []SIndex `json:"simple_indexes,omitempty"`
	MapIndexes      []SIndex `json:"map_indexes,omitempty"`
	ListIndexes     []SIndex `json:"list_indexes,omitempty"`
	VectorIndexes   []SIndex `json:"vector_indexes,omitempty"`
	CompoundIndexes []CIndex `json:"compound_indexes,omitempty"`
}

DBSchema is the schema of a document DB

type DocBatch 

type DocBatch struct {
	// contains filtered or unexported fields
}

DocBatch is a batch of documents

type Document 

type Document struct {
	// contains filtered or unexported fields
}

Document is the main object to handle document DBs

func NewDocumentStore 

func NewDocumentStore(podName string, fd *feed.API, ai *account.Info, user utils.Address, file *file.File,
	tm taskmanager.TaskManagerGO, client blockstore.Client, logger logging.Logger) *Document

NewDocumentStore instantiates a document DB object through which all document DB are spawned.

func (*Document) Count 

func (d *Document) Count(dbName, expr string) (uint64, error)

Count counts the number of document in a document DB which matches a given expression

func (*Document) CreateDocBatch 

func (d *Document) CreateDocBatch(dbName, podPassword string) (*DocBatch, error)

CreateDocBatch creates a batch index instead of normal index. This is used when doing a bulk insert.

func (*Document) CreateDocumentDB 

func (d *Document) CreateDocumentDB(dbName, encryptionPassword string, indexes map[string]IndexType, mutable bool) error

CreateDocumentDB creates a new document database and its related indexes.

func (*Document) Del 

func (d *Document) Del(dbName, id string) error

Del deletes a specific document from a document database matching a document id.

func (*Document) DeleteAllDocumentDBs added in v0.9.1 

func (d *Document) DeleteAllDocumentDBs(encryptionPassword string) error

DeleteAllDocumentDBs deletes all document DBs, all their data and related indxes.

func (*Document) DeleteDocumentDB 

func (d *Document) DeleteDocumentDB(dbName, encryptionPassword string) error

DeleteDocumentDB a document DB, all its data and its related indxes.

func (*Document) DocBatchPut 

func (d *Document) DocBatchPut(docBatch *DocBatch, doc []byte, index int64) error

DocBatchPut is used to insert a single document to the batch index.

func (*Document) DocBatchWrite 

func (d *Document) DocBatchWrite(docBatch *DocBatch, podFile string) error

DocBatchWrite commits the batch index into the Swarm network.

func (*Document) DocFileIndex 

func (d *Document) DocFileIndex(dbName, podFile, podPassword string) error

DocFileIndex indexes an existing json file in the pod with the document DB. skipcq: TCV-001

func (*Document) Find 

func (d *Document) Find(dbName, expr, podPassword string, limit int) ([][]byte, error)

Find selects a number of rows from a document database matching an expression.

func (*Document) Get 

func (d *Document) Get(dbName, id, podPassword string) ([]byte, error)

Get retrieves a specific document from a document database matching the dcument id.

func (*Document) IsDBOpened 

func (d *Document) IsDBOpened(dbName string) bool

IsDBOpened is used to check if a document DB is opened or not.

func (*Document) LoadDocumentDBSchemas 

func (d *Document) LoadDocumentDBSchemas(encryptionPassword string) (map[string]DBSchema, error)

LoadDocumentDBSchemas loads the schema of all documents belonging to a pod.

func (*Document) NearestNodes added in v0.10.0 

func (d *Document) NearestNodes(dbName, podPassword, index string, v []float32, force float32, limit int) ([][]byte, error)

NearestNodes does something TODO

func (*Document) OpenDocumentDB 

func (d *Document) OpenDocumentDB(dbName, encryptionPassword string) error

OpenDocumentDB open a document database and its related indexes.

func (*Document) Put 

func (d *Document) Put(dbName string, doc []byte) error

Put inserts a document in to a document database.

type DocumentDB 

type DocumentDB struct {
	// contains filtered or unexported fields
}

DocumentDB is the main object to handle a document DB

type Entry 

type Entry struct {
	Name     string    `json:"name"`
	EType    string    `json:"type"`
	Ref      [][]byte  `json:"ref,omitempty"`
	Manifest *Manifest `json:"Manifest,omitempty"`
}

Entry is the structure of the entry

type Index 

type Index struct {
	// contains filtered or unexported fields
}

Index is the structure of the index

func OpenIndex 

func OpenIndex(podName, collectionName, indexName, podPassword string, fd *feed.API, ai *account.Info, user utils.Address, client blockstore.Client, logger logging.Logger) (*Index, error)

OpenIndex open the index and load any index in to the memory.

func (*Index) CountIndex 

func (idx *Index) CountIndex(encryptionPassword string) (uint64, error)

CountIndex counts the entries in an index.

func (*Index) Delete 

func (idx *Index) Delete(key string) ([][]byte, error)

Delete removes an entry from index where the key is of type string.

func (*Index) DeleteIndex 

func (idx *Index) DeleteIndex(encryptionPassword string) error

DeleteIndex delete the index from file and all its entries.

func (*Index) DeleteNumber 

func (idx *Index) DeleteNumber(key float64) ([][]byte, error)

DeleteNumber removes an entry from index where the key is of type number.

func (*Index) Get 

func (idx *Index) Get(key string) ([][]byte, error)

Get retrieves an element from the index where the key is of type string.

func (*Index) GetNumber 

func (idx *Index) GetNumber(key float64) ([][]byte, error)

GetNumber retrieves an element from the index where the key is of type number. skipcq: TCV-001

func (*Index) IsEmpty added in v0.10.0 

func (idx *Index) IsEmpty(encryptionPassword string) (bool, error)

func (*Index) NewIntIterator 

func (idx *Index) NewIntIterator(start, end, limit int64) (*Iterator, error)

NewIntIterator creates a new index iterator with start prefix, endPrefix and the limit to iterate.

func (*Index) NewStringIterator 

func (idx *Index) NewStringIterator(start, end string, limit int64) (*Iterator, error)

NewStringIterator creates a new iterator object which is used to create new index iterators.

func (*Index) Put 

func (idx *Index) Put(key string, refValue []byte, idxType IndexType, apnd bool) error

Put inserts an entry in to index with a string as key.

func (*Index) PutNumber 

func (idx *Index) PutNumber(key float64, refValue []byte, idxType IndexType, apnd bool) error

PutNumber inserts an entry in to index with a number as a key.

type IndexType 

type IndexType int

IndexType is the type of the index 

const (
	// InvalidIndex is returned when the index type is invalid
	InvalidIndex IndexType = iota
	// BytesIndex is returned when the index type is bytes
	BytesIndex
	// StringIndex is returned when the index type is string
	StringIndex
	// NumberIndex is returned when the index type is number
	NumberIndex
	// MapIndex is returned when the index type is map
	MapIndex
	// ListIndex is returned when the index type is list
	ListIndex

	VectorIndex
)

func (IndexType) String 

func (e IndexType) String() string

type Iterator 

type Iterator struct {
	// contains filtered or unexported fields
}

Iterator is used to iterate over the index.

func (*Iterator) IntegerKey 

func (itr *Iterator) IntegerKey() int64

IntegerKey returns the current key as an integer.

func (*Iterator) Next 

func (itr *Iterator) Next() bool

Next moves the seek pointer one step ahead.

func (*Iterator) Seek 

func (itr *Iterator) Seek(key string) error

Seek seeks to the given key prefix.

func (*Iterator) StringKey 

func (itr *Iterator) StringKey() string

StringKey returns the current key.

func (*Iterator) Value 

func (itr *Iterator) Value() []byte

Value returns the current value.

func (*Iterator) ValueAll 

func (itr *Iterator) ValueAll() [][]byte

ValueAll returns all the values for the current key.

type KVTable 

type KVTable struct {
	// contains filtered or unexported fields
}

KVTable is the object used to do all operation on a key value table.

type KeyValue 

type KeyValue struct {
	// contains filtered or unexported fields
}

KeyValue is the main object used to do all operation on the key value tables.

func NewKeyValueStore 

func NewKeyValueStore(podName string, fd *feed.API, ai *account.Info, user utils.Address, client blockstore.Client, logger logging.Logger) *KeyValue

NewKeyValueStore is the main object used to do all operation on the key value tables.

func (*KeyValue) Commit added in v0.10.0 

func (kv *KeyValue) Commit()

func (*KeyValue) CreateKVTable 

func (kv *KeyValue) CreateKVTable(name, encryptionPassword string, indexType IndexType) error

CreateKVTable creates the key value table with a given index type.

func (*KeyValue) DeleteAllKVTables added in v0.9.1 

func (kv *KeyValue) DeleteAllKVTables(encryptionPassword string) error

DeleteAllKVTables deletes all key value tables with all their index and data entries.

func (*KeyValue) DeleteKVTable 

func (kv *KeyValue) DeleteKVTable(name, encryptionPassword string) error

DeleteKVTable deletes a given key value table with all it's index and data entries.

func (*KeyValue) IsEmpty added in v0.10.0 

func (kv *KeyValue) IsEmpty(name string) (bool, error)

IsEmpty checks if the given key value table is empty.

func (*KeyValue) KVBatch 

func (kv *KeyValue) KVBatch(name string, columns []string) (*Batch, error)

KVBatch prepares the index to do a batch insert if keys and values.

func (*KeyValue) KVBatchPut 

func (kv *KeyValue) KVBatchPut(batch *Batch, key string, value []byte) error

KVBatchPut inserts a key and value in to the memory for batch.

func (*KeyValue) KVBatchWrite 

func (kv *KeyValue) KVBatchWrite(batch *Batch) error

KVBatchWrite commits all the batch entries in to the key value table.

func (*KeyValue) KVCount 

func (kv *KeyValue) KVCount(name string) (*TableKeyCount, error)

KVCount counts the number of entries in the given key value table.

func (*KeyValue) KVDelete 

func (kv *KeyValue) KVDelete(name, key string) ([]byte, error)

KVDelete removed a key value entry from the KV table given a key.

func (*KeyValue) KVGet 

func (kv *KeyValue) KVGet(name, key string) ([]string, []byte, error)

KVGet retrieves a value from the KV table given a key.

func (*KeyValue) KVGetNext 

func (kv *KeyValue) KVGetNext(name string) ([]string, string, []byte, error)

KVGetNext retrieve the next key value pair in the iteration.

func (*KeyValue) KVPut 

func (kv *KeyValue) KVPut(name, key string, value []byte) error

KVPut inserts a given key and value in to the KV table.

func (*KeyValue) KVSeek 

func (kv *KeyValue) KVSeek(name, start, end string, limit int64) (*Iterator, error)

KVSeek seek to given key with start prefix and prepare for iterating the table.

func (*KeyValue) LoadKVTables 

func (kv *KeyValue) LoadKVTables(encryptionPassword string) (map[string][]string, error)

LoadKVTables Loads the list of KV tables.

func (*KeyValue) OpenKVTable 

func (kv *KeyValue) OpenKVTable(name, encryptionPassword string) error

OpenKVTable open a given key value table and loads the index.

type Manifest 

type Manifest struct {
	Name         string    `json:"name"`
	Mutable      bool      `json:"mutable"`
	PodFile      string    `json:"pod_file,omitempty"`
	IdxType      IndexType `json:"index_type"`
	CreationTime int64     `json:"creation_time"`
	Entries      []*Entry  `json:"entries,omitempty"`
	Count        uint64    `json:"count,omitempty"` // number of entries in the kv table, this should be updated on root manifest
	// contains filtered or unexported fields
}

Manifest is the structure of the manifest

func NewManifest 

func NewManifest(name string, time int64, idxType IndexType, mutable bool) *Manifest

NewManifest creates a new manifest

type ManifestState 

type ManifestState struct {
	// contains filtered or unexported fields
}

ManifestState is used to keep track of the current manifest and the current index of the manifest.

type SIndex 

type SIndex struct {
	FieldName string    `json:"name"`
	FieldType IndexType `json:"type"`
}

SIndex is a simple index

type TableKeyCount added in v0.9.1 

type TableKeyCount struct {
	Count     uint64 `json:"count"`
	TableName string `json:"tableName"`
}

TableKeyCount is the object used to store the count of keys in a table.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.