Versions in this module Expand all Collapse all v1 v1.0.0 Aug 23, 2021 Changes in this version + func Join(a []Text) string + func SegmentsToSlice(segs []Segment, searchMode bool) (output []string) + func SegmentsToString(segs []Segment, searchMode bool) (output string) + type Dictionary struct + func NewDictionary() *Dictionary + func (dict *Dictionary) MaxTokenLength() int + func (dict *Dictionary) NumTokens() int + func (dict *Dictionary) TotalFrequency() int64 + type Segment struct + func (s *Segment) End() int + func (s *Segment) Start() int + func (s *Segment) Token() *Token + type Segmenter struct + func (seg *Segmenter) Dictionary() *Dictionary + func (seg *Segmenter) InternalSegment(bytes []byte, searchMode bool) []Segment + func (seg *Segmenter) LoadDictionary(files string) + func (seg *Segmenter) Segment(bytes []byte) []Segment + type Text []byte + type Token struct + func (token *Token) Frequency() int + func (token *Token) Pos() string + func (token *Token) Segments() []*Segment + func (token *Token) Text() string + func (token *Token) TextEquals(string string) bool