tokenize

Aug 31, 2021

Changes in this version

type Wordpiece

+ func (wp Wordpiece) CharLoop(text string, result []string) []string

+ func (wp Wordpiece) CheckIsLargeThanMaxWordChars(text string, result []string) (bool, []string)

Aug 31, 2021

Changes in this version

type Wordpiece

+ func (wp Wordpiece) SubTokenize(text string) (toks []string)

Aug 30, 2021

Changes in this version

+ const ClassToken

+ const DefaultMaxWordChars

+ const DefaultUnknownToken

+ const SeparatorToken

+ const SequenceSeparator

+ type Basic struct

+ Lower bool

+ func NewBasic() Basic

+ func (bt Basic) Tokenize(text string) (toks []string)

+ type Feature struct

+ ID int32

+ Mask []int32

+ Text string

+ TokenIDs []int32

+ Tokens []string

+ TypeIDs []int32

+ func (f Feature) Count() int

+ type FeatureFactory struct

+ SeqLen int32

+ Tokenizer VocabTokenizer

+ func (ff *FeatureFactory) Feature(text string) Feature

+ func (ff *FeatureFactory) Features(texts ...string) []Feature

+ type Full struct

+ Basic Basic

+ Wordpiece Wordpiece

+ func (f Full) Tokenize(text string) (toks []string)

+ func (f Full) Vocab() vocab.Dict

+ type Option func(tkz Full) Full

+ func WithLower(lower bool) Option

+ func WithMaxChars(wc int) Option

+ func WithUnknownToken(unk string) Option

+ type Tokenizer interface

+ Tokenize func(text string) (tokens []string)

+ type VocabTokenizer interface

+ func NewTokenizer(voc vocab.Dict, opts ...Option) VocabTokenizer

+ type Wordpiece struct

+ func NewWordpiece(voc vocab.Dict) Wordpiece

+ func (wp Wordpiece) SetMaxWordChars(c int)

+ func (wp Wordpiece) SetUnknownToken(tok string)

+ func (wp Wordpiece) Tokenize(text string) (toks []string)