tok

Nov 14, 2017

Changes in this version

+ func BuildTokens(val interface{}, t Tokenizer) ([]string, error)

+ func LoadCustomTokenizer(soFile string)

+ type CustomTokenizer struct

+ func (CustomTokenizer) IsLossy() bool

+ func (CustomTokenizer) IsSortable() bool

+ type PluginTokenizer interface

+ Identifier func() byte

+ Name func() string

+ Tokens func(value interface{}) ([]string, error)

+ Type func() string

Sep 20, 2017

Changes in this version

+ const FTSTokenizerName

+ func EncodeGeoTokens(tokens []string)

+ func EncodeRegexTokens(tokens []string)

+ func FtsTokenizerName(lang string) string

+ func GetTextTokens(funcArgs []string, lang string) ([]string, error)

+ func GetTokens(funcArgs []string) ([]string, error)

+ type BoolTokenizer struct

+ func (t BoolTokenizer) Identifier() byte

+ func (t BoolTokenizer) IsLossy() bool

+ func (t BoolTokenizer) IsSortable() bool

+ func (t BoolTokenizer) Name() string

+ func (t BoolTokenizer) Tokens(v types.Val) ([]string, error)

+ func (t BoolTokenizer) Type() types.TypeID

+ type DayTokenizer struct

+ func (t DayTokenizer) Identifier() byte

+ func (t DayTokenizer) IsLossy() bool

+ func (t DayTokenizer) IsSortable() bool

+ func (t DayTokenizer) Name() string

+ func (t DayTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t DayTokenizer) Type() types.TypeID

type ExactTokenizer

+ func (t ExactTokenizer) Identifier() byte

+ func (t ExactTokenizer) IsLossy() bool

+ func (t ExactTokenizer) IsSortable() bool

type FloatTokenizer

+ func (t FloatTokenizer) Identifier() byte

+ func (t FloatTokenizer) IsLossy() bool

+ func (t FloatTokenizer) IsSortable() bool

+ type FullTextTokenizer struct

+ Lang string

+ func (t FullTextTokenizer) Identifier() byte

+ func (t FullTextTokenizer) IsLossy() bool

+ func (t FullTextTokenizer) IsSortable() bool

+ func (t FullTextTokenizer) Name() string

+ func (t FullTextTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t FullTextTokenizer) Type() types.TypeID

type GeoTokenizer

+ func (t GeoTokenizer) Identifier() byte

+ func (t GeoTokenizer) IsLossy() bool

+ func (t GeoTokenizer) IsSortable() bool

+ type HashTokenizer struct

+ func (t HashTokenizer) Identifier() byte

+ func (t HashTokenizer) IsLossy() bool

+ func (t HashTokenizer) IsSortable() bool

+ func (t HashTokenizer) Name() string

+ func (t HashTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t HashTokenizer) Type() types.TypeID

+ type HourTokenizer struct

+ func (t HourTokenizer) Identifier() byte

+ func (t HourTokenizer) IsLossy() bool

+ func (t HourTokenizer) IsSortable() bool

+ func (t HourTokenizer) Name() string

+ func (t HourTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t HourTokenizer) Type() types.TypeID

+ type IntTokenizer struct

+ func (t IntTokenizer) Identifier() byte

+ func (t IntTokenizer) IsLossy() bool

+ func (t IntTokenizer) IsSortable() bool

+ func (t IntTokenizer) Name() string

+ func (t IntTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t IntTokenizer) Type() types.TypeID

+ type MonthTokenizer struct

+ func (t MonthTokenizer) Identifier() byte

+ func (t MonthTokenizer) IsLossy() bool

+ func (t MonthTokenizer) IsSortable() bool

+ func (t MonthTokenizer) Name() string

+ func (t MonthTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t MonthTokenizer) Type() types.TypeID

type TermTokenizer

+ func (t TermTokenizer) Identifier() byte

+ func (t TermTokenizer) IsLossy() bool

+ func (t TermTokenizer) IsSortable() bool

type Tokenizer

+ Identifier func() byte

+ IsLossy func() bool

+ IsSortable func() bool

+ type TrigramTokenizer struct

+ func (t TrigramTokenizer) Identifier() byte

+ func (t TrigramTokenizer) IsLossy() bool

+ func (t TrigramTokenizer) IsSortable() bool

+ func (t TrigramTokenizer) Name() string

+ func (t TrigramTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t TrigramTokenizer) Type() types.TypeID

+ type YearTokenizer struct

+ func (t YearTokenizer) Identifier() byte

+ func (t YearTokenizer) IsLossy() bool

+ func (t YearTokenizer) IsSortable() bool

+ func (t YearTokenizer) Name() string

+ func (t YearTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t YearTokenizer) Type() types.TypeID

Feb 22, 2017

Changes in this version

+ func RegisterTokenizer(t Tokenizer)

+ func SetDefault(typ types.TypeID, name string)

+ type DateTimeTokenizer struct

+ func (t DateTimeTokenizer) Name() string

+ func (t DateTimeTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t DateTimeTokenizer) Type() types.TypeID

+ type DateTokenizer struct

+ func (t DateTokenizer) Name() string

+ func (t DateTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t DateTokenizer) Type() types.TypeID

+ type ExactTokenizer struct

+ func (t ExactTokenizer) Name() string

+ func (t ExactTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t ExactTokenizer) Type() types.TypeID

+ type FloatTokenizer struct

+ func (t FloatTokenizer) Name() string

+ func (t FloatTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t FloatTokenizer) Type() types.TypeID

+ type GeoTokenizer struct

+ func (t GeoTokenizer) Name() string

+ func (t GeoTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t GeoTokenizer) Type() types.TypeID

+ type Int32Tokenizer struct

+ func (t Int32Tokenizer) Name() string

+ func (t Int32Tokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t Int32Tokenizer) Type() types.TypeID

+ type TermTokenizer struct

+ func (t TermTokenizer) Name() string

+ func (t TermTokenizer) Tokens(sv types.Val) ([]string, error)

+ func (t TermTokenizer) Type() types.TypeID

type Tokenizer

+ func Default(typ types.TypeID) Tokenizer

+ func GetTokenizer(name string) Tokenizer

+ Name func() string

+ Type func() types.TypeID

Jan 27, 2017

Changes in this version

+ func ICUDisabled() bool

Dec 8, 2016

Changes in this version

+ type Tokenizer struct

+ func NewTokenizer(s []byte) (*Tokenizer, error)

+ func (t *Tokenizer) Destroy()

+ func (t *Tokenizer) Next() []byte

+ func (t *Tokenizer) Tokens() []string