Versions in this module Expand all Collapse all v1 v1.0.1 Sep 17, 2024 Changes in this version + func BertBaseUncased() *tokenizer.Tokenizer + func BertLargeCasedWholeWordMaskingSquad() *tokenizer.Tokenizer + func CreateAddedTokens(data []tokenizer.TokenConfig) (specialToks, toks []tokenizer.AddedToken) + func CreateDecoder(config map[string]interface{}) (tokenizer.Decoder, error) + func CreateModel(config *tokenizer.Config) (tokenizer.Model, error) + func CreateNormalizer(config map[string]interface{}) (normalizer.Normalizer, error) + func CreatePaddingParams(config map[string]interface{}) (*tokenizer.PaddingParams, error) + func CreatePostProcessor(config map[string]interface{}) (tokenizer.PostProcessor, error) + func CreatePreTokenizer(config map[string]interface{}) (tokenizer.PreTokenizer, error) + func CreateTruncationParams(config map[string]interface{}) (*tokenizer.TruncationParams, error) + func FromFile(file string) (*tokenizer.Tokenizer, error) + func FromReader(r io.Reader) (*tokenizer.Tokenizer, error) + func GPT2(addPrefixSpace bool, trimOffsets bool) *tokenizer.Tokenizer + func RobertaBase(addPrefixSpace, trimOffsets bool) *tokenizer.Tokenizer + func RobertaBaseSquad2(addPrefixSpace, trimOffsets bool) *tokenizer.Tokenizer
Go to main page