tok

Documentation

Index

• func RegisterTokenizer(t Tokenizer)
• func SetDefault(typ types.TypeID, name string)
• type DateTimeTokenizer
  • func (t DateTimeTokenizer) Name() string
  • func (t DateTimeTokenizer) Tokens(sv types.Val) ([]string, error)
  • func (t DateTimeTokenizer) Type() types.TypeID
• type DateTokenizer
  • func (t DateTokenizer) Name() string
  • func (t DateTokenizer) Tokens(sv types.Val) ([]string, error)
  • func (t DateTokenizer) Type() types.TypeID
• type ExactTokenizer
  • func (t ExactTokenizer) Name() string
  • func (t ExactTokenizer) Tokens(sv types.Val) ([]string, error)
  • func (t ExactTokenizer) Type() types.TypeID
• type FloatTokenizer
  • func (t FloatTokenizer) Name() string
  • func (t FloatTokenizer) Tokens(sv types.Val) ([]string, error)
  • func (t FloatTokenizer) Type() types.TypeID
• type GeoTokenizer
  • func (t GeoTokenizer) Name() string
  • func (t GeoTokenizer) Tokens(sv types.Val) ([]string, error)
  • func (t GeoTokenizer) Type() types.TypeID
• type Int32Tokenizer
  • func (t Int32Tokenizer) Name() string
  • func (t Int32Tokenizer) Tokens(sv types.Val) ([]string, error)
  • func (t Int32Tokenizer) Type() types.TypeID
• type TermTokenizer
  • func (t TermTokenizer) Name() string
  • func (t TermTokenizer) Tokens(sv types.Val) ([]string, error)
  • func (t TermTokenizer) Type() types.TypeID
• type Tokenizer
  • func Default(typ types.TypeID) Tokenizer
  • func GetTokenizer(name string) Tokenizer

Functions

func RegisterTokenizer

func RegisterTokenizer(t Tokenizer)

RegisterTokenizer adds your tokenizer to our list.

func SetDefault

func SetDefault(typ types.TypeID, name string)

SetDefault sets the default tokenizer for given typeID.

Types

type DateTimeTokenizer

type DateTimeTokenizer struct{}

func (DateTimeTokenizer) Name

func (t DateTimeTokenizer) Name() string

func (DateTimeTokenizer) Tokens

func (t DateTimeTokenizer) Tokens(sv types.Val) ([]string, error)

func (DateTimeTokenizer) Type

func (t DateTimeTokenizer) Type() types.TypeID

type DateTokenizer

type DateTokenizer struct{}

func (DateTokenizer) Name

func (t DateTokenizer) Name() string

func (DateTokenizer) Tokens

func (t DateTokenizer) Tokens(sv types.Val) ([]string, error)

func (DateTokenizer) Type

func (t DateTokenizer) Type() types.TypeID

type ExactTokenizer

type ExactTokenizer struct{}

func (ExactTokenizer) Name

func (t ExactTokenizer) Name() string

func (ExactTokenizer) Tokens

func (t ExactTokenizer) Tokens(sv types.Val) ([]string, error)

func (ExactTokenizer) Type

func (t ExactTokenizer) Type() types.TypeID

type FloatTokenizer

type FloatTokenizer struct{}

func (FloatTokenizer) Name

func (t FloatTokenizer) Name() string

func (FloatTokenizer) Tokens

func (t FloatTokenizer) Tokens(sv types.Val) ([]string, error)

func (FloatTokenizer) Type

func (t FloatTokenizer) Type() types.TypeID

type GeoTokenizer

type GeoTokenizer struct{}

func (GeoTokenizer) Name

func (t GeoTokenizer) Name() string

func (GeoTokenizer) Tokens

func (t GeoTokenizer) Tokens(sv types.Val) ([]string, error)

func (GeoTokenizer) Type

func (t GeoTokenizer) Type() types.TypeID

type Int32Tokenizer

type Int32Tokenizer struct{}

func (Int32Tokenizer) Name

func (t Int32Tokenizer) Name() string

func (Int32Tokenizer) Tokens

func (t Int32Tokenizer) Tokens(sv types.Val) ([]string, error)

func (Int32Tokenizer) Type

func (t Int32Tokenizer) Type() types.TypeID

type TermTokenizer

type TermTokenizer struct{}

func (TermTokenizer) Name

func (t TermTokenizer) Name() string

func (TermTokenizer) Tokens

func (t TermTokenizer) Tokens(sv types.Val) ([]string, error)

func (TermTokenizer) Type

func (t TermTokenizer) Type() types.TypeID

type Tokenizer

type Tokenizer interface {
	// Name is name of tokenizer. This should be unique.
	Name() string

	// Type returns typeID that we care about.
	Type() types.TypeID

	// Tokens return tokens for a given value.
	Tokens(sv types.Val) ([]string, error)
}

Tokenizer defines what a tokenizer must provide.

func Default

func Default(typ types.TypeID) Tokenizer

Default returns the default tokenizer for a given type.

func GetTokenizer

func GetTokenizer(name string) Tokenizer

GetTokenizer returns tokenizer given unique name.