core

Documentation

Overview

Package core contains the implementation of Tatris core features

Index

• Constants
• func BuildDocuments(index *Index, docs []protocol.Document) error
• func CheckDocument(index *Index, doc protocol.Document) error
• func MergeSegmentReader(config *indexlib.Config, segments ...*Segment) (indexlib.Reader, error)
• type Index
  • func (index *Index) AddProperties(addProperties map[string]*protocol.Property)
  • func (index *Index) Destroy() error
  • func (index *Index) GetName() string
  • func (index *Index) GetReadersByTime(start, end int64) (indexlib.Reader, error)
  • func (index *Index) GetSegmentsByTime(start, end int64) []*Segment
  • func (index *Index) GetShard(idx int) *Shard
  • func (index *Index) GetShardByRouting() *Shard
  • func (index *Index) GetShardNum() int
  • func (index *Index) GetShards() []*Shard
• type Segment
  • func (segment *Segment) Destroy()
  • func (segment *Segment) GetName() string
  • func (segment *Segment) GetReader() (indexlib.Reader, error)
  • func (segment *Segment) GetWriter() (indexlib.Writer, error)
  • func (segment *Segment) IsMature() bool
  • func (segment *Segment) MatchTime(start, end int64) bool
  • func (segment *Segment) OnMature()
  • func (segment *Segment) Readonly() bool
  • func (segment *Segment) Status() uint8
  • func (segment *Segment) UpdateStat(min, max time.Time, docs int64)
• type SegmentStat
• type Shard
  • func (shard *Shard) CheckSegments()
  • func (shard *Shard) Destroy() error
  • func (shard *Shard) ForceAddSegment()
  • func (shard *Shard) GetLatestSegment() *Segment
  • func (shard *Shard) GetLatestSegmentID() int
  • func (shard *Shard) GetName() string
  • func (shard *Shard) GetSegment(idx int) *Segment
  • func (shard *Shard) GetSegmentNum() int
  • func (shard *Shard) GetSegments() []*Segment
  • func (shard *Shard) UpdateStat(min, max time.Time, docs int64, wals uint64)
• type ShardStat
• type Stat

Constants

const (
	// SegmentStatusWritable means segment is the latest segment of shard and is writable.
	// GetReaders returns a reader from the underlying writer, which can be used to search data near
	// real-time.
	SegmentStatusWritable uint8 = iota
	// SegmentStatusReadonly means segment is readonly. So the writer is always nil, and GetWriter
	// returns an error.
	// GetReader uses ReaderCache to ensure opening same only once.
	SegmentStatusReadonly
)

Functions

func BuildDocuments

func BuildDocuments(
	index *Index,
	docs []protocol.Document,
) error

func CheckDocument

func CheckDocument(index *Index, doc protocol.Document) error

func MergeSegmentReader

func MergeSegmentReader(
	config *indexlib.Config,
	segments ...*Segment,
) (indexlib.Reader, error)

MergeSegmentReader merges segment readers into one indexlib.Reader instance. Now the provided reader must be type of *bluge.BlugeReader.

Types

type Index

type Index struct {
	*protocol.Index
	Shards []*Shard `json:"shards"`
	// contains filtered or unexported fields
}

func (*Index) AddProperties

func (index *Index) AddProperties(addProperties map[string]*protocol.Property)

func (*Index) Destroy

func (index *Index) Destroy() error

func (*Index) GetName

func (index *Index) GetName() string

func (*Index) GetReadersByTime

func (index *Index) GetReadersByTime(start, end int64) (indexlib.Reader, error)

func (*Index) GetSegmentsByTime

func (index *Index) GetSegmentsByTime(start, end int64) []*Segment

func (*Index) GetShard

func (index *Index) GetShard(idx int) *Shard

func (*Index) GetShardByRouting

func (index *Index) GetShardByRouting() *Shard

GetShardByRouting TODO: build the real route, temporarily think that there is always only 1 shard

func (*Index) GetShardNum

func (index *Index) GetShardNum() int

func (*Index) GetShards

func (index *Index) GetShards() []*Shard

type Segment

type Segment struct {
	Shard         *Shard `json:"-"`
	SegmentID     int
	Stat          SegmentStat
	SegmentStatus uint8
	// contains filtered or unexported fields
}

func (*Segment) Destroy

func (segment *Segment) Destroy()

func (*Segment) GetName

func (segment *Segment) GetName() string

func (*Segment) GetReader

func (segment *Segment) GetReader() (indexlib.Reader, error)

GetReader returns a reader snapshot of current segment. So docs wrote this func returns are invisible to returned reader. Returned reader must be closed after use.

func (*Segment) GetWriter

func (segment *Segment) GetWriter() (indexlib.Writer, error)

func (*Segment) IsMature

func (segment *Segment) IsMature() bool

func (*Segment) MatchTime

func (segment *Segment) MatchTime(start, end int64) bool

func (*Segment) OnMature

func (segment *Segment) OnMature()

OnMature is called when segment becomes mature. It marks segment readonly and closes the underlying writer.

func (*Segment) Readonly

func (segment *Segment) Readonly() bool

func (*Segment) Status

func (segment *Segment) Status() uint8

func (*Segment) UpdateStat

func (segment *Segment) UpdateStat(min, max time.Time, docs int64)

type SegmentStat

type SegmentStat struct {
	Stat
	MatureTime int64
}

type Shard

type Shard struct {
	Index    *Index `json:"-"`
	ShardID  int
	Segments []*Segment
	Stat     ShardStat
	Wal      log.WalLog `json:"-"`
	// contains filtered or unexported fields
}

Shard is a logical split of the index

func (*Shard) CheckSegments

func (shard *Shard) CheckSegments()

func (*Shard) Destroy

func (shard *Shard) Destroy() error

func (*Shard) ForceAddSegment

func (shard *Shard) ForceAddSegment()

ForceAddSegment forces adding a segment to current shard

func (*Shard) GetLatestSegment

func (shard *Shard) GetLatestSegment() *Segment

func (*Shard) GetLatestSegmentID

func (shard *Shard) GetLatestSegmentID() int

func (*Shard) GetName

func (shard *Shard) GetName() string

func (*Shard) GetSegment

func (shard *Shard) GetSegment(idx int) *Segment

func (*Shard) GetSegmentNum

func (shard *Shard) GetSegmentNum() int

func (*Shard) GetSegments

func (shard *Shard) GetSegments() []*Segment

func (*Shard) UpdateStat

func (shard *Shard) UpdateStat(min, max time.Time, docs int64, wals uint64)

type ShardStat

type ShardStat struct {
	Stat
	// WalIndex is the last consumed wal index
	WalIndex uint64
}

type Stat

type Stat struct {
	CreateTime int64
	MinTime    int64
	MaxTime    int64
	DocNum     int64
}

Stat records the statistics of an index split