Documentation ¶
Index ¶
Constants ¶
View Source
const Name = "regexp"
Variables ¶
View Source
var IdeographRegexp = regexp.MustCompile(`\p{Han}|\p{Hangul}|\p{Hiragana}|\p{Katakana}`)
Functions ¶
Types ¶
type RegexpTokenizer ¶
type RegexpTokenizer struct {
// contains filtered or unexported fields
}
func NewRegexpTokenizer ¶
func NewRegexpTokenizer(r *regexp.Regexp) *RegexpTokenizer
func (*RegexpTokenizer) Tokenize ¶
func (rt *RegexpTokenizer) Tokenize(input []byte) analysis.TokenStream
Click to show internal directories.
Click to hide internal directories.