ctokenizer

Documentation

Index

• func HandleCStatus(status *C.CStatus, extraInfo string) error
• func NewTokenizer(param string) (tokenizerapi.Tokenizer, error)
• func ValidateTextSchema(fieldSchema *schemapb.FieldSchema, EnableBM25 bool) error
• type CMap
  • func NewCMap() *CMap
  • func (m *CMap) Destroy()
  • func (m *CMap) From(gm map[string]string)
  • func (m *CMap) GetPointer() C.CMap
  • func (m *CMap) Set(key string, value string)
• type CTokenStream
  • func NewCTokenStream(ptr C.CTokenStream) *CTokenStream
  • func (impl *CTokenStream) Advance() bool
  • func (impl *CTokenStream) Destroy()
  • func (impl *CTokenStream) Token() string
• type CTokenizer
  • func NewCTokenizer(ptr C.CTokenizer) *CTokenizer
  • func (impl *CTokenizer) Clone() (tokenizerapi.Tokenizer, error)
  • func (impl *CTokenizer) Destroy()
  • func (impl *CTokenizer) NewTokenStream(text string) tokenizerapi.TokenStream

Functions

func HandleCStatus

func HandleCStatus(status *C.CStatus, extraInfo string) error

HandleCStatus deal with the error returned from CGO

func NewTokenizer

func NewTokenizer(param string) (tokenizerapi.Tokenizer, error)

func ValidateTextSchema

func ValidateTextSchema(fieldSchema *schemapb.FieldSchema, EnableBM25 bool) error

Types

type CMap

type CMap struct {
	// contains filtered or unexported fields
}

func NewCMap

func NewCMap() *CMap

func (*CMap) Destroy

func (m *CMap) Destroy()

func (*CMap) From

func (m *CMap) From(gm map[string]string)

func (*CMap) GetPointer

func (m *CMap) GetPointer() C.CMap

func (*CMap) Set

func (m *CMap) Set(key string, value string)

type CTokenStream

type CTokenStream struct {
	// contains filtered or unexported fields
}

func NewCTokenStream

func NewCTokenStream(ptr C.CTokenStream) *CTokenStream

func (*CTokenStream) Advance

func (impl *CTokenStream) Advance() bool

func (*CTokenStream) Destroy

func (impl *CTokenStream) Destroy()

func (*CTokenStream) Token

func (impl *CTokenStream) Token() string

type CTokenizer

type CTokenizer struct {
	// contains filtered or unexported fields
}

func NewCTokenizer

func NewCTokenizer(ptr C.CTokenizer) *CTokenizer

func (*CTokenizer) Clone

func (impl *CTokenizer) Clone() (tokenizerapi.Tokenizer, error)

func (*CTokenizer) Destroy

func (impl *CTokenizer) Destroy()

func (*CTokenizer) NewTokenStream

func (impl *CTokenizer) NewTokenStream(text string) tokenizerapi.TokenStream