tok

package
v83.0.0-...-38e0eed Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 12, 2017 License: AGPL-3.0, Apache-2.0 Imports: 16 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	FTSTokenizerName = "fulltext"
)

Variables

This section is empty.

Functions

func EncodeGeoTokens 

func EncodeGeoTokens(tokens []string)

func EncodeRegexTokens 

func EncodeRegexTokens(tokens []string)

func FtsTokenizerName 

func FtsTokenizerName(lang string) string

func GetTextTokens 

func GetTextTokens(funcArgs []string, lang string) ([]string, error)

func GetTokens 

func GetTokens(funcArgs []string) ([]string, error)

func RegisterTokenizer 

func RegisterTokenizer(t Tokenizer)

RegisterTokenizer adds your tokenizer to our list.

Types

type BoolTokenizer 

type BoolTokenizer struct{}

func (BoolTokenizer) Identifier 

func (t BoolTokenizer) Identifier() byte

func (BoolTokenizer) IsLossy 

func (t BoolTokenizer) IsLossy() bool

func (BoolTokenizer) IsSortable 

func (t BoolTokenizer) IsSortable() bool

func (BoolTokenizer) Name 

func (t BoolTokenizer) Name() string

func (BoolTokenizer) Tokens 

func (t BoolTokenizer) Tokens(v types.Val) ([]string, error)

func (BoolTokenizer) Type 

func (t BoolTokenizer) Type() types.TypeID

type DayTokenizer 

type DayTokenizer struct{}

func (DayTokenizer) Identifier 

func (t DayTokenizer) Identifier() byte

func (DayTokenizer) IsLossy 

func (t DayTokenizer) IsLossy() bool

func (DayTokenizer) IsSortable 

func (t DayTokenizer) IsSortable() bool

func (DayTokenizer) Name 

func (t DayTokenizer) Name() string

func (DayTokenizer) Tokens 

func (t DayTokenizer) Tokens(sv types.Val) ([]string, error)

func (DayTokenizer) Type 

func (t DayTokenizer) Type() types.TypeID

type ExactTokenizer 

type ExactTokenizer struct{}

func (ExactTokenizer) Identifier 

func (t ExactTokenizer) Identifier() byte

func (ExactTokenizer) IsLossy 

func (t ExactTokenizer) IsLossy() bool

func (ExactTokenizer) IsSortable 

func (t ExactTokenizer) IsSortable() bool

func (ExactTokenizer) Name 

func (t ExactTokenizer) Name() string

func (ExactTokenizer) Tokens 

func (t ExactTokenizer) Tokens(sv types.Val) ([]string, error)

func (ExactTokenizer) Type 

func (t ExactTokenizer) Type() types.TypeID

type FloatTokenizer 

type FloatTokenizer struct{}

func (FloatTokenizer) Identifier 

func (t FloatTokenizer) Identifier() byte

func (FloatTokenizer) IsLossy 

func (t FloatTokenizer) IsLossy() bool

func (FloatTokenizer) IsSortable 

func (t FloatTokenizer) IsSortable() bool

func (FloatTokenizer) Name 

func (t FloatTokenizer) Name() string

func (FloatTokenizer) Tokens 

func (t FloatTokenizer) Tokens(sv types.Val) ([]string, error)

func (FloatTokenizer) Type 

func (t FloatTokenizer) Type() types.TypeID

type FullTextTokenizer 

type FullTextTokenizer struct {
	Lang string
}

Full text tokenizer, with language support

func (FullTextTokenizer) Identifier 

func (t FullTextTokenizer) Identifier() byte

func (FullTextTokenizer) IsLossy 

func (t FullTextTokenizer) IsLossy() bool

func (FullTextTokenizer) IsSortable 

func (t FullTextTokenizer) IsSortable() bool

func (FullTextTokenizer) Name 

func (t FullTextTokenizer) Name() string

func (FullTextTokenizer) Tokens 

func (t FullTextTokenizer) Tokens(sv types.Val) ([]string, error)

func (FullTextTokenizer) Type 

func (t FullTextTokenizer) Type() types.TypeID

type GeoTokenizer 

type GeoTokenizer struct{}

func (GeoTokenizer) Identifier 

func (t GeoTokenizer) Identifier() byte

func (GeoTokenizer) IsLossy 

func (t GeoTokenizer) IsLossy() bool

func (GeoTokenizer) IsSortable 

func (t GeoTokenizer) IsSortable() bool

func (GeoTokenizer) Name 

func (t GeoTokenizer) Name() string

func (GeoTokenizer) Tokens 

func (t GeoTokenizer) Tokens(sv types.Val) ([]string, error)

func (GeoTokenizer) Type 

func (t GeoTokenizer) Type() types.TypeID

type HashTokenizer 

type HashTokenizer struct{}

func (HashTokenizer) Identifier 

func (t HashTokenizer) Identifier() byte

func (HashTokenizer) IsLossy 

func (t HashTokenizer) IsLossy() bool

func (HashTokenizer) IsSortable 

func (t HashTokenizer) IsSortable() bool

func (HashTokenizer) Name 

func (t HashTokenizer) Name() string

func (HashTokenizer) Tokens 

func (t HashTokenizer) Tokens(sv types.Val) ([]string, error)

func (HashTokenizer) Type 

func (t HashTokenizer) Type() types.TypeID

type HourTokenizer 

type HourTokenizer struct{}

func (HourTokenizer) Identifier 

func (t HourTokenizer) Identifier() byte

func (HourTokenizer) IsLossy 

func (t HourTokenizer) IsLossy() bool

func (HourTokenizer) IsSortable 

func (t HourTokenizer) IsSortable() bool

func (HourTokenizer) Name 

func (t HourTokenizer) Name() string

func (HourTokenizer) Tokens 

func (t HourTokenizer) Tokens(sv types.Val) ([]string, error)

func (HourTokenizer) Type 

func (t HourTokenizer) Type() types.TypeID

type IntTokenizer 

type IntTokenizer struct{}

func (IntTokenizer) Identifier 

func (t IntTokenizer) Identifier() byte

func (IntTokenizer) IsLossy 

func (t IntTokenizer) IsLossy() bool

func (IntTokenizer) IsSortable 

func (t IntTokenizer) IsSortable() bool

func (IntTokenizer) Name 

func (t IntTokenizer) Name() string

func (IntTokenizer) Tokens 

func (t IntTokenizer) Tokens(sv types.Val) ([]string, error)

func (IntTokenizer) Type 

func (t IntTokenizer) Type() types.TypeID

type MonthTokenizer 

type MonthTokenizer struct{}

func (MonthTokenizer) Identifier 

func (t MonthTokenizer) Identifier() byte

func (MonthTokenizer) IsLossy 

func (t MonthTokenizer) IsLossy() bool

func (MonthTokenizer) IsSortable 

func (t MonthTokenizer) IsSortable() bool

func (MonthTokenizer) Name 

func (t MonthTokenizer) Name() string

func (MonthTokenizer) Tokens 

func (t MonthTokenizer) Tokens(sv types.Val) ([]string, error)

func (MonthTokenizer) Type 

func (t MonthTokenizer) Type() types.TypeID

type TermTokenizer 

type TermTokenizer struct{}

func (TermTokenizer) Identifier 

func (t TermTokenizer) Identifier() byte

func (TermTokenizer) IsLossy 

func (t TermTokenizer) IsLossy() bool

func (TermTokenizer) IsSortable 

func (t TermTokenizer) IsSortable() bool

func (TermTokenizer) Name 

func (t TermTokenizer) Name() string

func (TermTokenizer) Tokens 

func (t TermTokenizer) Tokens(sv types.Val) ([]string, error)

func (TermTokenizer) Type 

func (t TermTokenizer) Type() types.TypeID

type Tokenizer 

type Tokenizer interface {

	// Name is name of tokenizer. This should be unique.
	Name() string

	// Type returns typeID that we care about.
	Type() types.TypeID

	// Tokens return tokens for a given value.
	Tokens(sv types.Val) ([]string, error)

	// Identifier returns the prefix byte for this token type.
	Identifier() byte

	// IsSortable returns true if the tokenizer can be used for sorting/ordering.
	IsSortable() bool

	// IsLossy() returns true if we don't store the values directly as index keys
	// during tokenization. If a predicate is tokenized using an IsLossy() tokenizer,
	// then we need to fetch the actual value and compare.
	IsLossy() bool
}

Tokenizer defines what a tokenizer must provide.

func GetTokenizer 

func GetTokenizer(name string) (Tokenizer, bool)

GetTokenizer returns tokenizer given unique name.

type TrigramTokenizer 

type TrigramTokenizer struct{}

func (TrigramTokenizer) Identifier 

func (t TrigramTokenizer) Identifier() byte

func (TrigramTokenizer) IsLossy 

func (t TrigramTokenizer) IsLossy() bool

func (TrigramTokenizer) IsSortable 

func (t TrigramTokenizer) IsSortable() bool

func (TrigramTokenizer) Name 

func (t TrigramTokenizer) Name() string

func (TrigramTokenizer) Tokens 

func (t TrigramTokenizer) Tokens(sv types.Val) ([]string, error)

func (TrigramTokenizer) Type 

func (t TrigramTokenizer) Type() types.TypeID

type YearTokenizer 

type YearTokenizer struct{}

func (YearTokenizer) Identifier 

func (t YearTokenizer) Identifier() byte

func (YearTokenizer) IsLossy 

func (t YearTokenizer) IsLossy() bool

func (YearTokenizer) IsSortable 

func (t YearTokenizer) IsSortable() bool

func (YearTokenizer) Name 

func (t YearTokenizer) Name() string

func (YearTokenizer) Tokens 

func (t YearTokenizer) Tokens(sv types.Val) ([]string, error)

func (YearTokenizer) Type 

func (t YearTokenizer) Type() types.TypeID

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.