tok

Feb 22, 2017

Changes in this version

+ func RegisterTokenizer(t Tokenizer)

+ func SetDefault(typ types.TypeID, name string)

+ type DateTimeTokenizer struct

+ func (t DateTimeTokenizer) Name() string

+ func (t DateTimeTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t DateTimeTokenizer) Type() types.TypeID

+ type DateTokenizer struct

+ func (t DateTokenizer) Name() string

+ func (t DateTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t DateTokenizer) Type() types.TypeID

+ type ExactTokenizer struct

+ func (t ExactTokenizer) Name() string

+ func (t ExactTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t ExactTokenizer) Type() types.TypeID

+ type FloatTokenizer struct

+ func (t FloatTokenizer) Name() string

+ func (t FloatTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t FloatTokenizer) Type() types.TypeID

+ type GeoTokenizer struct

+ func (t GeoTokenizer) Name() string

+ func (t GeoTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t GeoTokenizer) Type() types.TypeID

+ type Int32Tokenizer struct

+ func (t Int32Tokenizer) Name() string

+ func (t Int32Tokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t Int32Tokenizer) Type() types.TypeID

+ type TermTokenizer struct

+ func (t TermTokenizer) Name() string

+ func (t TermTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t TermTokenizer) Type() types.TypeID

type Tokenizer

+ func Default(typ types.TypeID) Tokenizer

+ func GetTokenizer(name string) Tokenizer

+ Name func() string

+ Type func() types.TypeID

Jan 27, 2017

Changes in this version

+ func ICUDisabled() bool

Dec 8, 2016

Changes in this version

+ type Tokenizer struct

+ func NewTokenizer(s []byte) (*Tokenizer, error)

+ func (t *Tokenizer) Destroy()

+ func (t *Tokenizer) Next() []byte

+ func (t *Tokenizer) Tokens() []string