core

package
v0.5.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 25, 2024 License: Apache-2.0 Imports: 43 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	ShardIDNeedLatest int64 = -1 // get lastest shardID
	ShardIDNeedUpdate int64 = -2 // get all shardIDs
)
View Source 
const (
	RedoActionRead     = uint64(1)
	RedoActionWrite    = uint64(2)
	RedoActionTruncate = uint64(3)
)
View Source 
const MaxBatchSize = 10240

MaxBatchSize used to limit memory

Variables

View Source 
var Telemetry = newTelemetry()

Telemetry instance

View Source 
var ZINC_METRICS *ginprometheus.Metric

Functions

func CheckIndexName 

func CheckIndexName(name string) error

func DeleteIndex 

func DeleteIndex(name string) error

func DeleteTemplate 

func DeleteTemplate(name string) error

DeleteTemplate delete a template from local

func IncrMetricStatsByIndex 

func IncrMetricStatsByIndex(index, field string)

func ListTemplates 

func ListTemplates(pattern string) ([]*meta.Template, error)

ListTemplates returns all templates

func LoadTemplate 

func LoadTemplate(name string) (*meta.IndexTemplate, bool, error)

LoadTemplate load a specific template from local

func LoadZincIndexesFromMetadata 

func LoadZincIndexesFromMetadata(version string) error

func MultiSearch 

func MultiSearch(indexNames []string, query *meta.ZincQuery) (*meta.SearchResponse, error)

func NewTemplate 

func NewTemplate(name string, template *meta.IndexTemplate) error

NewTemplate create a template and store in local

func OpenIndexWriter 

func OpenIndexWriter(name string, storageType string, defaultSearchAnalyzer *analysis.Analyzer, timeRange ...int64) (*bluge.Writer, error)

LoadIndexWriter load the index writer from the storage

func SetMetricStatsByIndex 

func SetMetricStatsByIndex(index, field string, val float64)

func StoreIndex 

func StoreIndex(index *Index) error

storeIndex stores the index to metadata

func UseTemplate 

func UseTemplate(indexName string) (*meta.IndexTemplate, error)

UseTemplate use a specific template for new index

Types

type AliasList 

type AliasList struct {
	Aliases map[string][]string
	// contains filtered or unexported fields
}
var ZINC_INDEX_ALIAS_LIST AliasList

func NewAliasList 

func NewAliasList() *AliasList

func (*AliasList) AddIndexesToAlias 

func (al *AliasList) AddIndexesToAlias(alias string, indexes []string) error

func (*AliasList) GetAliasMap 

func (al *AliasList) GetAliasMap(targetIndexes, targetAliases []string) M

GetAliasMap returns an ES compatible map of indexes to their aliases In the form: 

{"gitea_issues":{"aliases":{}},"gitea_codes.v1":{"aliases":{"gitea_codes":{}}}}

func (*AliasList) GetAliasesForIndex 

func (al *AliasList) GetAliasesForIndex(indexName string) []string

func (*AliasList) GetIndexesForAlias 

func (al *AliasList) GetIndexesForAlias(aliasName string) ([]string, bool)

func (*AliasList) RemoveIndexesFromAlias 

func (al *AliasList) RemoveIndexesFromAlias(alias string, removeIndexes []string) error

type Index 

type Index struct {
	// contains filtered or unexported fields
}

func GetIndex 

func GetIndex(name string) (*Index, bool)

func GetOrCreateIndex 

func GetOrCreateIndex(name, storageType string, shardNum int64) (*Index, bool, error)

func NewIndex 

func NewIndex(name, storageType string, shardNum int64) (*Index, error)

NewIndex creates an instance of a physical zinc index that can be used to store and retrieve data.

func (*Index) CheckShards 

func (index *Index) CheckShards() error

CheckShards check all shards status if need create new second layer shard

func (*Index) Close 

func (index *Index) Close() error

func (*Index) CreateDocument 

func (index *Index) CreateDocument(docID string, doc map[string]interface{}, update bool) error

CreateDocument inserts or updates a document in the zinc index

func (*Index) DeleteDocument 

func (index *Index) DeleteDocument(docID string) error

DeleteDocument deletes a document in the zinc index

func (*Index) GetAllShardNum 

func (index *Index) GetAllShardNum() int64

func (*Index) GetAnalyzers 

func (index *Index) GetAnalyzers() map[string]*analysis.Analyzer

func (*Index) GetDocument 

func (index *Index) GetDocument(docID string) (*meta.Hit, error)

GetDocument get a document in the zinc index

func (*Index) GetIndex 

func (index *Index) GetIndex() meta.Index

func (*Index) GetMappings 

func (index *Index) GetMappings() *meta.Mappings

func (*Index) GetName 

func (index *Index) GetName() string

func (*Index) GetReaders 

func (index *Index) GetReaders(timeMin, timeMax int64) ([]*bluge.Reader, error)

GetReaders return all shard readers

func (*Index) GetSettings 

func (index *Index) GetSettings() *meta.IndexSettings

func (*Index) GetShardByDocID 

func (index *Index) GetShardByDocID(docID string) *IndexShard

GetShardByDocID return the shard by hash docID

func (*Index) GetShardNum 

func (index *Index) GetShardNum() int64

func (*Index) GetStats 

func (index *Index) GetStats() meta.IndexStat

func (*Index) GetStorageType 

func (index *Index) GetStorageType() string

func (*Index) GetWALSize 

func (index *Index) GetWALSize() uint64

func (*Index) MarshalJSON 

func (index *Index) MarshalJSON() ([]byte, error)

func (*Index) Reopen 

func (index *Index) Reopen() error

Reopen just close the index, it will open automatically by trigger Deprecated: it will be removed in the future

func (*Index) Search 

func (index *Index) Search(query *meta.ZincQuery) (*meta.SearchResponse, error)

func (*Index) SetAnalyzers 

func (index *Index) SetAnalyzers(analyzers map[string]*analysis.Analyzer) error

func (*Index) SetMappings 

func (index *Index) SetMappings(mappings *meta.Mappings) error

func (*Index) SetSettings 

func (index *Index) SetSettings(settings *meta.IndexSettings) error

func (*Index) UpdateDocument 

func (index *Index) UpdateDocument(docID string, doc map[string]interface{}, insert bool) error

UpdateDocument updates a document in the zinc index

func (*Index) UpdateMetadata 

func (index *Index) UpdateMetadata() error

UpdateMetadata update index metadata, mainly docNum and storageSize need merge from all first layer shards

func (*Index) UpdateMetadataByShard 

func (index *Index) UpdateMetadataByShard(id string)

UpdateMetadataByShard update first layer shard metadata, mainly docNum, storageSize and timeRange need merge from all second layer shards

func (*Index) UpdateStatsBySecondShard 

func (index *Index) UpdateStatsBySecondShard(id string, secondIndex int64)

UpdateStatsBySecondShard update second layer shard stats, mainly docNum and storageSize

func (*Index) UpdateWALSize 

func (index *Index) UpdateWALSize(size uint64)

func (*Index) UseTemplate 

func (index *Index) UseTemplate() error

type IndexList 

type IndexList struct {
	Indexes map[string]*Index
	// contains filtered or unexported fields
}
var ZINC_INDEX_LIST IndexList

func (*IndexList) Add 

func (t *IndexList) Add(index *Index)

func (*IndexList) Close 

func (t *IndexList) Close() error

func (*IndexList) Delete 

func (t *IndexList) Delete(name string)

func (*IndexList) GC 

func (t *IndexList) GC() error

GC auto close unused indexes what inactive for a long time (10m)

func (*IndexList) Get 

func (t *IndexList) Get(name string) (*Index, bool)

func (*IndexList) GetOrCreate 

func (t *IndexList) GetOrCreate(name, storageType string, shardNum int64) (*Index, bool, error)

func (*IndexList) Len 

func (t *IndexList) Len() int

func (*IndexList) List 

func (t *IndexList) List() []*Index

func (*IndexList) ListMap 

func (t *IndexList) ListMap() map[string]*Index

func (*IndexList) ListName 

func (t *IndexList) ListName() []string

func (*IndexList) ListStat 

func (t *IndexList) ListStat() []*Index

type IndexSecondShard 

type IndexSecondShard struct {
	// contains filtered or unexported fields
}

IndexSecondShard second layer shard by auto increate shards for index. Under first layer shards, Documents will store in this layer shards. Use a environment `config.ZINC_SHARD_MAX_SIZE` to control second layer shard max size. If the shard size over limit then will auto create a new shard for accept new documents. And we will log time range for this layer shards, when query data we can use time range filter which shards need to find data. We keep one shard size wouldn't over limit, we will fozen old shards, just write new documents to new shards and do merge in new shards this will improve shard performance.

type IndexShard 

type IndexShard struct {
	// contains filtered or unexported fields
}

IndexShard first layer shard by fixed number shards for index. Use hash algorithm distribute documents to different shards. This shards let we can concurrency write to many shards in same index. The shards num can not be modify, because if change the num hash algorithm will distribute the same docID to another shard, then we will can not found the old document, maybe cause duplicate documents. First layer shard just used for distribute not really store documents.

func (*IndexShard) BuildBlugeDocumentFromJSON 

func (s *IndexShard) BuildBlugeDocumentFromJSON(docID string, doc map[string]interface{}) (*bluge.Document, error)

BuildBlugeDocumentFromJSON returns the bluge document for the json document. It also updates the mapping for the fields if not found. If no mappings are found, it creates te mapping for all the encountered fields. If mapping for some fields is found but not for others then it creates the mapping for the missing fields.

func (*IndexShard) CheckDocument 

func (s *IndexShard) CheckDocument(docID string, doc map[string]interface{}, update bool, shard int64) ([]byte, error)

CheckDocument checks if the document is valid.

func (*IndexShard) CheckShards 

func (s *IndexShard) CheckShards() error

CheckShards check current shard is reach the maximum shard size or create a new shard

func (*IndexShard) Close 

func (s *IndexShard) Close() error

func (*IndexShard) ConsumeWAL 

func (s *IndexShard) ConsumeWAL() bool

ConsumeWAL consume WAL for index returns if there is any data updated

func (*IndexShard) FindDocumentByDocID 

func (s *IndexShard) FindDocumentByDocID(docID string) (*meta.Hit, error)

FindDocumentByDocID finds docID and returns the document

func (*IndexShard) FindShardByDocID 

func (s *IndexShard) FindShardByDocID(docID string) (int64, error)

FindShardByDocID finds docID in which shard and returns the shard id

func (*IndexShard) GetID 

func (s *IndexShard) GetID() string

func (*IndexShard) GetIndexName 

func (s *IndexShard) GetIndexName() string

func (*IndexShard) GetLatestShardID 

func (s *IndexShard) GetLatestShardID() int64

func (*IndexShard) GetReaders 

func (s *IndexShard) GetReaders(timeMin, timeMax int64) ([]*bluge.Reader, error)

GetReaders return all shard readers

func (*IndexShard) GetShardName 

func (s *IndexShard) GetShardName() string

func (*IndexShard) GetShardNum 

func (s *IndexShard) GetShardNum() int64

func (*IndexShard) GetWALSize 

func (s *IndexShard) GetWALSize() (uint64, error)

func (*IndexShard) GetWriter 

func (s *IndexShard) GetWriter(shardID ...int64) (*bluge.Writer, error)

GetWriter return the newest shard writer or special shard writer

func (*IndexShard) GetWriters 

func (s *IndexShard) GetWriters() ([]*bluge.Writer, error)

GetWriters return all shard writers

func (*IndexShard) NewShard 

func (s *IndexShard) NewShard() error

func (*IndexShard) OpenWAL 

func (s *IndexShard) OpenWAL() error

OpenWAL open WAL for index

func (*IndexShard) Rollback 

func (s *IndexShard) Rollback() error

func (*IndexShard) SetTimestamp 

func (s *IndexShard) SetTimestamp(t int64)

type IndexShardWALList 

type IndexShardWALList struct {
	Shards map[string]*IndexShard
	// contains filtered or unexported fields
}
var ZINC_INDEX_SHARD_WAL_LIST IndexShardWALList

Record opened WAL used to do consume

func (*IndexShardWALList) Add 

func (t *IndexShardWALList) Add(shard *IndexShard)

func (*IndexShardWALList) ConsumeWAL 

func (t *IndexShardWALList) ConsumeWAL()

func (*IndexShardWALList) Len 

func (t *IndexShardWALList) Len() int

func (*IndexShardWALList) List 

func (t *IndexShardWALList) List() []*IndexShard

func (*IndexShardWALList) Remove 

func (t *IndexShardWALList) Remove(name string)

type M 

type M map[string]interface{}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.