Documentation
¶
Index ¶
- func ConfigFromFile[T ConfigFile](file string) (config T, _ error)
- type Config
- type ConfigFile
- type Embeddings
- type Encoder
- type EncoderLayer
- type FeedForwardBlock
- type FeedForwardBlockConfig
- type Model
- type ModelForMaskedLM
- type ModelForQuestionAnswering
- type ModelForSequenceClassification
- type ModelForSequenceEncoding
- type ModelForTokenClassification
- type Pooler
- type PoolingStrategyType
- type SelfAttentionBlock
- type SelfAttentionBlockConfig
- type TokenizerConfig
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func ConfigFromFile ¶
func ConfigFromFile[T ConfigFile](file string) (config T, _ error)
ConfigFromFile loads a Bart model Config from file.
Types ¶
type Config ¶
type Config struct {
Architectures []string `json:"architectures"`
AttentionProbsDropoutProb float64 `json:"attention_probs_dropout_prob"`
GradientCheckpointing bool `json:"gradient_checkpointing"`
HiddenAct string `json:"hidden_act"`
HiddenDropoutProb float64 `json:"hidden_dropout_prob"`
HiddenSize int `json:"hidden_size"`
EmbeddingsSize int `json:"embeddings_size"`
InitializerRange float64 `json:"initializer_range"`
IntermediateSize int `json:"intermediate_size"`
LayerNormEps float64 `json:"layer_norm_eps"`
MaxPositionEmbeddings int `json:"max_position_embeddings"`
ModelType string `json:"model_type"`
NumAttentionHeads int `json:"num_attention_heads"`
NumHiddenLayers int `json:"num_hidden_layers"`
PadTokenId int `json:"pad_token_id"`
PositionEmbeddingType string `json:"position_embedding_type"`
TransformersVersion string `json:"transformers_version"`
TypeVocabSize int `json:"type_vocab_size"`
UseCache bool `json:"use_cache"`
VocabSize int `json:"vocab_size"`
ID2Label map[string]string `json:"id2label"`
Cybertron struct {
Training bool `json:"training"`
TokensStoreName string `json:"tokens_store_name"`
PositionsStoreName string `json:"positions_store_name"`
TokenTypesStoreName string `json:"token_types_store_name"`
}
}
Config contains the global configuration of the Bert model and the heads of fine-tuning tasks. The configuration coincides with that of Hugging Face to facilitate compatibility between the two architectures.
type ConfigFile ¶
type ConfigFile interface {
Config | TokenizerConfig
}
ConfigFile is the union of the configuration structures.
type Embeddings ¶
type Embeddings struct {
nn.Module
Vocab *vocabulary.Vocabulary
Tokens *emb.Model // string
Positions *emb.Model
TokenTypes *emb.Model
Norm *layernorm.Model
Projector *linear.Model
Config Config
}
Embeddings implements a Bert input embedding module.
func NewEmbeddings ¶
func NewEmbeddings[T float.DType](c Config) *Embeddings
NewEmbeddings returns a new Bert input embedding module.
func (*Embeddings) EncodeTokens ¶ added in v0.2.0
func (m *Embeddings) EncodeTokens(tokens []string) []mat.Tensor
EncodeTokens performs the Bert input encoding.
type Encoder ¶
type Encoder struct {
nn.Module
Layers nn.ModuleList[*EncoderLayer]
Config Config
}
Encoder implements a Bert encoder.
func NewEncoder ¶
NewEncoder returns a new Encoder.
type EncoderLayer ¶
type EncoderLayer struct {
nn.Module
SelfAttention *SelfAttentionBlock
FF *FeedForwardBlock
Config Config
}
EncoderLayer implements a Bert encoder layer.
func NewEncoderLayer ¶
func NewEncoderLayer[T float.DType](c Config) *EncoderLayer
NewEncoderLayer returns a new EncoderLayer.
type FeedForwardBlock ¶
type FeedForwardBlock struct {
nn.Module
MLP nn.ModuleList[nn.StandardModel]
Norm *layernorm.Model
}
FeedForwardBlock implements a Bert feed forward block.
func NewFeedForwardBlock ¶
func NewFeedForwardBlock[T float.DType](c FeedForwardBlockConfig) *FeedForwardBlock
NewFeedForwardBlock returns a new FeedForwardBlock.
type FeedForwardBlockConfig ¶
type FeedForwardBlockConfig struct {
Dim int
HiddenDim int
Activation activation.Activation
}
FeedForwardBlockConfig is the configuration for the FeedForwardBlock.
type Model ¶
type Model struct {
nn.Module
Embeddings *Embeddings
Encoder *Encoder
Pooler *Pooler
Config Config
}
Model implements a base Bert encoder model without any head on top.
type ModelForMaskedLM ¶ added in v0.1.2
type ModelForMaskedLM struct {
nn.Module
// Bart is the fine-tuned BERT model.
Bert *Model
// Layers contains the feedforward layers for masked language modeling.
Layers nn.ModuleList[nn.StandardModel]
}
ModelForMaskedLM implements a Bert model for masked language modeling.
func NewModelForMaskedLM ¶ added in v0.1.2
func NewModelForMaskedLM[T float.DType](bert *Model) *ModelForMaskedLM
NewModelForMaskedLM returns a new model for masked language model.
type ModelForQuestionAnswering ¶
type ModelForQuestionAnswering struct {
nn.Module
// Bart is the fine-tuned BERT model.
Bert *Model
// Classifier is the linear layer for span classification.
Classifier *linear.Model
}
ModelForQuestionAnswering implements span classification for extractive question-answering tasks. It uses a linear layers to compute "span start logits" and "span end logits".
func NewModelForQuestionAnswering ¶
func NewModelForQuestionAnswering[T float.DType](bert *Model) *ModelForQuestionAnswering
NewModelForQuestionAnswering returns a new model for question-answering.
type ModelForSequenceClassification ¶
type ModelForSequenceClassification struct {
nn.Module
// Bart is the fine-tuned BERT model.
Bert *Model
// Classifier is the linear layer for sequence classification.
Classifier *linear.Model
}
ModelForSequenceClassification implements a Bert model for sequence classification.
func NewModelForSequenceClassification ¶
func NewModelForSequenceClassification[T float.DType](bert *Model) *ModelForSequenceClassification
NewModelForSequenceClassification returns a new model for sequence classification.
type ModelForSequenceEncoding ¶
type ModelForSequenceEncoding struct {
nn.Module
// Bart is the fine-tuned BERT model.
Bert *Model
}
ModelForSequenceEncoding implements a Bert model for sequence encoding.
func NewModelForSequenceEncoding ¶
func NewModelForSequenceEncoding(bert *Model) *ModelForSequenceEncoding
NewModelForSequenceEncoding returns a new model for sequence encoding.
func (*ModelForSequenceEncoding) Encode ¶
func (m *ModelForSequenceEncoding) Encode(tokens []string, poolingStrategy PoolingStrategyType) (mat.Tensor, error)
Encode returns the vector representation for the input sequence.
type ModelForTokenClassification ¶
type ModelForTokenClassification struct {
nn.Module
// Bart is the fine-tuned BERT model.
Bert *Model
// Classifier is the linear layer for sequence classification.
Classifier *linear.Model
}
ModelForTokenClassification implements a Bert model for token classification.
func NewModelForTokenClassification ¶
func NewModelForTokenClassification[T float.DType](bert *Model) *ModelForTokenClassification
NewModelForTokenClassification returns a new model for token classification.
type Pooler ¶
type Pooler struct {
nn.Module
Model nn.ModuleList[nn.StandardModel]
}
Pooler implements a Bert pooler.
type PoolingStrategyType ¶
type PoolingStrategyType int
PoolingStrategyType defines the strategy to obtain the dense sequence representation
const ( // ClsTokenPooling gets the last encoding state corresponding to [CLS], i.e. the first token (default) ClsTokenPooling PoolingStrategyType = iota // MeanPooling takes the average of the last encoding states MeanPooling // MaxPooling takes the maximum of the last encoding states MaxPooling // MeanMaxPooling does MeanPooling and MaxPooling separately and then concat them together MeanMaxPooling )
type SelfAttentionBlock ¶
type SelfAttentionBlock struct {
nn.Module
// Attention is the multi-head attention module.
Attention *multiheadattention.Model
// Norm is the layer normalization module.
Norm *layernorm.Model
}
SelfAttentionBlock implements the self-attention block of a BERT model.
func NewSelfAttentionBlock ¶
func NewSelfAttentionBlock[T float.DType](c SelfAttentionBlockConfig) *SelfAttentionBlock
NewSelfAttentionBlock creates a new SelfAttentionBlock.
type SelfAttentionBlockConfig ¶
SelfAttentionBlockConfig is the configuration of a SelfAttentionBlock.
type TokenizerConfig ¶
type TokenizerConfig struct {
DoLowerCase bool `json:"do_lower_case"`
UnkToken string `json:"unk_token"`
SepToken string `json:"sep_token"`
PadToken string `json:"pad_token"`
ClsToken string `json:"cls_token"`
MaskToken string `json:"mask_token"`
TokenizeChineseChars bool `json:"tokenize_chinese_chars"`
StripAccents interface{} `json:"strip_accents"`
ModelMaxLength int `json:"model_max_length"`
}
TokenizerConfig contains the configuration of the tokenizer. The configuration coincides with that of Hugging Face to facilitate compatibility between the two architectures.
Source Files