wordpiecetokenizer

Dec 9, 2020

Changes in this version

+ const DefaultClassToken

+ const DefaultMaskToken

+ const DefaultMaxWordChars

+ const DefaultSequenceSeparator

+ const DefaultSplitPrefix

+ const DefaultUnknownToken

+ var DefaultNeverSplit = []string

+ func IsDefaultSpecial(word string) bool

+ func MakeOffsetPairsFromGroups(text string, tokens []tokenizers.StringOffsetsPair, groups []TokensRange) []tokenizers.StringOffsetsPair

+ type TokensRange struct

+ End int

+ Start int

+ func GroupPieces(tokens []tokenizers.StringOffsetsPair) []TokensRange

+ type WordPieceTokenizer struct

+ func New(vocabulary *vocabulary.Vocabulary) *WordPieceTokenizer

+ func (t *WordPieceTokenizer) Tokenize(text string) []tokenizers.StringOffsetsPair

+ func (t *WordPieceTokenizer) WordPieceTokenize(tokens []tokenizers.StringOffsetsPair) []tokenizers.StringOffsetsPair