unicode

Documentation

Index

• Constants
• func UnicodeTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error)
• type UnicodeTokenizer
  • func NewUnicodeTokenizer() *UnicodeTokenizer
  • func (rt *UnicodeTokenizer) Tokenize(input []byte) analysis.TokenStream

Constants

const Name = "unicode"

Functions

func UnicodeTokenizerConstructor

func UnicodeTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error)

Types

type UnicodeTokenizer

type UnicodeTokenizer struct {
}

func NewUnicodeTokenizer

func NewUnicodeTokenizer() *UnicodeTokenizer

func (*UnicodeTokenizer) Tokenize

func (rt *UnicodeTokenizer) Tokenize(input []byte) analysis.TokenStream