charlm

Documentation

Overview

CharLM implements a character-level language model that uses a recurrent neural network as its backbone. A fully connected softmax layer (a.k.a decoder) is placed on top of each recurrent hidden state to predict the next character.

Index

• Constants
• func CalculatePerplexity(m *Model, text string) float64
• func Initialize(m *Model, rndGen *rand.LockedRand)
• type Config
• type Generator
  • func NewGenerator(model *Model, config GeneratorConfig) *Generator
  • func (m *Generator) GenerateText(prefix string) (text string, logProb float64)
• type GeneratorConfig
• type Model
  • func New(config Config) *Model
  • func (m *Model) NewProc(ctx nn.Context) nn.Processor
• type Processor
  • func (p *Processor) Forward(_ ...ag.Node) []ag.Node
  • func (p *Processor) GetEmbeddings(xs []string) []ag.Node
  • func (p *Processor) Predict(xs ...string) []ag.Node
  • func (p *Processor) UseProjection(xs ...ag.Node) []ag.Node
• type Trainer
  • func NewTrainer(config TrainingConfig, corpus corpora.TextCorpusIterator, model *Model) *Trainer
  • func (t *Trainer) Train()
• type TrainingConfig

Constants

const (
	DefaultSequenceSeparator = "[SEP]"
	DefaultUnknownToken      = "[UNK]"
)

Functions

func CalculatePerplexity

func CalculatePerplexity(m *Model, text string) float64

CalculatePerplexity returns the perplexity for the text calculated as Exp(CrossEntropyLoss). The output of the language model is directly compared to the expected targets extracted from the input itself.

func Initialize

func Initialize(m *Model, rndGen *rand.LockedRand)

Types

type Config

type Config struct {
	VocabularySize    int
	EmbeddingSize     int
	HiddenSize        int
	OutputSize        int    // use the projection layer when the output size is > 0
	SequenceSeparator string // empty string is replaced with DefaultSequenceSeparator
	UnknownToken      string // empty string is replaced with DefaultUnknownToken
}

TODO: add dropout

type Generator

type Generator struct {
	GeneratorConfig
	// contains filtered or unexported fields
}

func NewGenerator

func NewGenerator(model *Model, config GeneratorConfig) *Generator

func (*Generator) GenerateText

func (m *Generator) GenerateText(prefix string) (text string, logProb float64)

GenerateText returns a new sequences of text generated by the learned language model, and its log probability. The output text has a maximum length defined in the generator configuration. The text is incrementally constructed character by character, using all previously sampled characters as input to predict the next one.

type GeneratorConfig

type GeneratorConfig struct {
	MaxCharacters int
	StopAtEOS     bool
	Temperature   float64
}

type Model

type Model struct {
	Config
	Decoder    *linear.Model
	Projection *linear.Model
	RNN        nn.Model
	Embeddings []*nn.Param `type:"weights"`
	Vocabulary *vocabulary.Vocabulary
}

func New

func New(config Config) *Model

func (*Model) NewProc

func (m *Model) NewProc(ctx nn.Context) nn.Processor

type Processor

type Processor struct {
	nn.BaseProcessor
	Decoder    *linear.Processor
	Projection *linear.Processor
	RNN        nn.Processor

	UnknownEmbedding ag.Node
	// contains filtered or unexported fields
}

func (*Processor) Forward

func (p *Processor) Forward(_ ...ag.Node) []ag.Node

func (*Processor) GetEmbeddings

func (p *Processor) GetEmbeddings(xs []string) []ag.Node

func (*Processor) Predict

func (p *Processor) Predict(xs ...string) []ag.Node

func (*Processor) UseProjection

func (p *Processor) UseProjection(xs ...ag.Node) []ag.Node

type Trainer

type Trainer struct {
	TrainingConfig
	// contains filtered or unexported fields
}

func NewTrainer

func NewTrainer(config TrainingConfig, corpus corpora.TextCorpusIterator, model *Model) *Trainer

func (*Trainer) Train

func (t *Trainer) Train()

type TrainingConfig

type TrainingConfig struct {
	Seed                  uint64
	BatchSize             int
	BackStep              int
	GradientClipping      float64
	SerializationInterval int
	UpdateMethod          gd.MethodConfig
	ModelPath             string
}

TODO: add dropout