regexp_tokenizer

Documentation

Index

• Constants
• Variables
• func RegexpTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error)
• type RegexpTokenizer
  • func NewRegexpTokenizer(r *regexp.Regexp) *RegexpTokenizer
  • func (rt *RegexpTokenizer) Tokenize(input []byte) analysis.TokenStream

Constants

const Name = "regexp"

Variables

var IdeographRegexp = regexp.MustCompile(`\p{Han}|\p{Hangul}|\p{Hiragana}|\p{Katakana}`)

Functions

func RegexpTokenizerConstructor

func RegexpTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error)

Types

type RegexpTokenizer

type RegexpTokenizer struct {
	// contains filtered or unexported fields
}

func NewRegexpTokenizer

func NewRegexpTokenizer(r *regexp.Regexp) *RegexpTokenizer

func (*RegexpTokenizer) Tokenize

func (rt *RegexpTokenizer) Tokenize(input []byte) analysis.TokenStream