model

package
v0.0.0-...-25ab4ef Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 27, 2024 License: MIT Imports: 11 Imported by: 0

Documentation

Index

Examples

Constants

This section is empty.

Variables

This section is empty.

Functions

func Load 

func Load(filename string) ([][]matrix.Matrix, bool)

Load loads the params from a file.

Example (Invaliddir) 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
)

func main() {
	if _, ok := model.Load("invalid_dir"); !ok {
		fmt.Println("failed to save params")
		return
	}

}

Output:

failed to save params
Example (Invalidfile) 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
)

func main() {
	if _, ok := model.Load("../testdata/.gitkeep"); !ok {
		fmt.Println("failed to save params")
		return
	}

}

Output:

failed to save params

func Save 

func Save(filename string, params [][]matrix.Matrix) error

Save saves the params to a file.

Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	s := rand.Const(1)
	m := model.NewSeq2Seq(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	}, s)

	if err := model.Save("../testdata/example_save.gob", m.Params()); err != nil {
		fmt.Println("failed to save params:", err)
		return
	}

	params, ok := model.Load("../testdata/example_save.gob")
	if !ok {
		fmt.Println("failed to load params")
		return
	}

	if len(m.Params()) != len(params) {
		fmt.Println("invalid length")
		return
	}

	for i, p := range m.Params() {
		if len(p) != len(params[i]) {
			fmt.Println("invalid length")
			return
		}

		for j := range p {
			if p[j].Sub(params[i][j]).Abs().Sum() > 1e-13 {
				fmt.Println("invalid value")
				return
			}
		}
	}

}

Output:
Example (Nosuchdir) 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
)

func main() {
	if err := model.Save("../nosuchdir/hoge.gob", nil); err != nil {
		fmt.Println("failed to save params:", err)
		return
	}

}

Output:

failed to save params: create file: open ../nosuchdir/hoge.gob: no such file or directory

Types

type AttentionDecoder 

type AttentionDecoder struct {
	TimeEmbedding *layer.TimeEmbedding
	TimeLSTM      *layer.TimeLSTM
	TimeAttention *layer.TimeAttention
	TimeAffine    *layer.TimeAffine
	Source        randv2.Source
}

func NewAttentionDecoder 

func NewAttentionDecoder(c *RNNLMConfig, s ...randv2.Source) *AttentionDecoder

func (*AttentionDecoder) Backward 

func (m *AttentionDecoder) Backward(dscore []matrix.Matrix) []matrix.Matrix

func (*AttentionDecoder) Forward 

func (m *AttentionDecoder) Forward(xs, enchs []matrix.Matrix) []matrix.Matrix

func (*AttentionDecoder) Generate 

func (m *AttentionDecoder) Generate(enchs []matrix.Matrix, startID, length int) []int

func (*AttentionDecoder) Grads 

func (l *AttentionDecoder) Grads() []matrix.Matrix

func (*AttentionDecoder) Layers 

func (m *AttentionDecoder) Layers() []TimeLayer
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewAttentionDecoder(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	fmt.Printf("%T\n", m)
	for i, l := range m.Layers() {
		fmt.Printf("%2d: %v\n", i, l)
	}
	fmt.Println()

}

Output:

*model.AttentionDecoder
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 2: *layer.TimeAffine: W(6, 3), B(1, 3): 21

func (*AttentionDecoder) Params 

func (l *AttentionDecoder) Params() []matrix.Matrix
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewAttentionDecoder(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	m.SetParams(make([]matrix.Matrix, 6)...)
	fmt.Println(m.Params())
	fmt.Println(m.Grads())

}

Output:

[[] [] [] [] [] []]
[[] [] [] [] [] []]

func (*AttentionDecoder) SetParams 

func (l *AttentionDecoder) SetParams(p ...matrix.Matrix)

func (*AttentionDecoder) Summary 

func (m *AttentionDecoder) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewAttentionDecoder(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.AttentionDecoder
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 2: *layer.TimeAttention
 3: *layer.TimeAffine: W(6, 3), B(1, 3): 21

type AttentionEncoder 

type AttentionEncoder struct {
	Encoder
}

func NewAttentionEncoder 

func NewAttentionEncoder(c *RNNLMConfig, s ...randv2.Source) *AttentionEncoder

func (*AttentionEncoder) Backward 

func (m *AttentionEncoder) Backward(dhs []matrix.Matrix)

func (*AttentionEncoder) Forward 

func (m *AttentionEncoder) Forward(xs []matrix.Matrix) []matrix.Matrix

func (*AttentionEncoder) Summary 

func (m *AttentionEncoder) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewAttentionEncoder(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.AttentionEncoder
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84

type AttentionLayer 

type AttentionLayer interface {
	Forward(hs []matrix.Matrix, a matrix.Matrix) matrix.Matrix
	Backward(da matrix.Matrix) ([]matrix.Matrix, matrix.Matrix)
	Params() []matrix.Matrix
	Grads() []matrix.Matrix
	SetParams(p ...matrix.Matrix)
	String() string
}

AttentionLayer is an interface that represents an attention layer.

type AttentionSeq2Seq 

type AttentionSeq2Seq struct {
	Encoder *AttentionEncoder
	Decoder *AttentionDecoder
	Softmax *layer.TimeSoftmaxWithLoss
	Source  randv2.Source
}
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	s := rand.Const(1)
	m := model.NewAttentionSeq2Seq(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	}, s)

	xs := []matrix.Matrix{{{0, 1, 2}}, {{0, 1, 2}}, {{0, 1, 2}}}
	ts := []matrix.Matrix{{{0, 1, 2}}, {{0, 1, 2}}, {{0, 1, 2}}}

	loss := m.Forward(xs, ts)
	m.Backward()

	fmt.Printf("%.4f\n", loss)
	fmt.Println(m.Generate(xs, 1, 10))

}

Output:

[[[1.0958]]]
[1 1 1 1 1 1 1 1 1 1]

func NewAttentionSeq2Seq 

func NewAttentionSeq2Seq(c *RNNLMConfig, s ...randv2.Source) *AttentionSeq2Seq

func (*AttentionSeq2Seq) Backward 

func (m *AttentionSeq2Seq) Backward()

func (*AttentionSeq2Seq) Forward 

func (m *AttentionSeq2Seq) Forward(xs, ts []matrix.Matrix) []matrix.Matrix

func (*AttentionSeq2Seq) Generate 

func (m *AttentionSeq2Seq) Generate(xs []matrix.Matrix, startID, length int) []int

func (*AttentionSeq2Seq) Grads 

func (m *AttentionSeq2Seq) Grads() [][]matrix.Matrix

func (*AttentionSeq2Seq) Layers 

func (m *AttentionSeq2Seq) Layers() []TimeLayer
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewAttentionSeq2Seq(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	fmt.Printf("%T\n", m)
	for i, l := range m.Layers() {
		fmt.Printf("%2d: %v\n", i, l)
	}
	fmt.Println()

}

Output:

*model.AttentionSeq2Seq
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 2: *layer.TimeEmbedding: W(3, 3): 9
 3: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 4: *layer.TimeAffine: W(6, 3), B(1, 3): 21
 5: *layer.TimeSoftmaxWithLoss

func (*AttentionSeq2Seq) Params 

func (m *AttentionSeq2Seq) Params() [][]matrix.Matrix
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewAttentionSeq2Seq(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	m.SetParams(m.Grads())
	fmt.Println(m.Params())
	fmt.Println(m.Grads())

}

Output:

[[[] [] [] []] [[] [] [] [] [] []]]
[[[] [] [] []] [[] [] [] [] [] []]]

func (*AttentionSeq2Seq) SetParams 

func (m *AttentionSeq2Seq) SetParams(p [][]matrix.Matrix)

func (*AttentionSeq2Seq) Summary 

func (m *AttentionSeq2Seq) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewAttentionSeq2Seq(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.AttentionSeq2Seq
 0: *model.AttentionEncoder
 1: *layer.TimeEmbedding: W(3, 3): 9
 2: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 3: *model.AttentionDecoder
 4: *layer.TimeEmbedding: W(3, 3): 9
 5: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 6: *layer.TimeAttention
 7: *layer.TimeAffine: W(6, 3), B(1, 3): 21
 8: *layer.TimeSoftmaxWithLoss

type CBOW 

type CBOW struct {
	Win0, Win1, Wout Layer
	Loss             Layer
	Source           randv2.Source
}
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
)

func main() {
	// you, say, goodbye, and, I, hello, .

	// data
	contexts := []matrix.Matrix{
		{
			{1, 0, 0, 0, 0, 0, 0}, // you
			{0, 0, 1, 0, 0, 0, 0}, // goodbye
		},
	}
	targets := matrix.Matrix{
		[]float64{0, 1, 0, 0, 0, 0, 0}, // say
	}

	// model
	s := rand.Const(1)
	m := model.NewCBOW(&model.CBOWConfig{
		VocabSize:  7,
		HiddenSize: 5,
	}, s)

	loss := m.Forward(contexts, targets)
	m.Backward()
	fmt.Println(loss)

	score := m.Predict([]matrix.Matrix{
		{
			{1, 0, 0, 0, 0, 0, 0}, // you
			{0, 0, 1, 0, 0, 0, 0}, // goodbye
		},
	})
	fmt.Println(score)

}

Output:

[[1.945926338366903]]
[[0.0002949561336916829 1.6764012522628663e-05 0.00014512664854319168 -0.00013693775475907596 -0.0001516469737667451 7.902275520347142e-05 -1.1785748579958962e-05]]

func NewCBOW 

func NewCBOW(c *CBOWConfig, s ...randv2.Source) *CBOW

func (*CBOW) Backward 

func (m *CBOW) Backward()

func (*CBOW) Forward 

func (m *CBOW) Forward(contexts []matrix.Matrix, target matrix.Matrix) matrix.Matrix

func (*CBOW) Grads 

func (m *CBOW) Grads() [][]matrix.Matrix

func (*CBOW) Layers 

func (m *CBOW) Layers() []Layer
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
)

func main() {
	m := model.NewCBOW(&model.CBOWConfig{
		VocabSize:  7,
		HiddenSize: 5,
	})

	fmt.Printf("%T\n", m)
	for i, l := range m.Layers() {
		fmt.Printf("%2d: %v\n", i, l)
	}

}

Output:

*model.CBOW
 0: *layer.Dot: W(7, 5): 35
 1: *layer.Dot: W(7, 5): 35
 2: *layer.Dot: W(5, 7): 35
 3: *layer.SoftmaxWithLoss

func (*CBOW) Params 

func (m *CBOW) Params() [][]matrix.Matrix
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
)

func main() {
	m := model.NewCBOW(&model.CBOWConfig{
		VocabSize:  7,
		HiddenSize: 5,
	})

	m.SetParams(m.Grads())
	fmt.Println(m.Params())
	fmt.Println(m.Grads())

}

Output:

[[[]] [[]] [[]]]
[[[]] [[]] [[]]]

func (*CBOW) Predict 

func (m *CBOW) Predict(xs []matrix.Matrix) matrix.Matrix

func (*CBOW) SetParams 

func (m *CBOW) SetParams(p [][]matrix.Matrix)

func (*CBOW) Summary 

func (m *CBOW) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
)

func main() {
	m := model.NewCBOW(&model.CBOWConfig{
		VocabSize:  7,
		HiddenSize: 5,
	})

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.CBOW
 0: *layer.Dot: W(7, 5): 35
 1: *layer.Dot: W(7, 5): 35
 2: *layer.Dot: W(5, 7): 35
 3: *layer.SoftmaxWithLoss

type CBOWConfig 

type CBOWConfig struct {
	VocabSize  int
	HiddenSize int
}

type CBOWNegativeSampling 

type CBOWNegativeSampling struct {
	Embedding []Layer
	Loss      Layer
	// contains filtered or unexported fields
}
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
)

func main() {
	s := rand.Const(1)
	m := model.NewCBOWNegativeSampling(model.CBOWNegativeSamplingConfig{
		CBOWConfig: model.CBOWConfig{
			VocabSize:  7,
			HiddenSize: 5,
		},
		Corpus:     []int{0, 1, 2, 3, 4, 1, 5, 6},
		WindowSize: 1,
		SampleSize: 2,
		Power:      0.75,
	}, s)

	contexts := matrix.New(
		[]float64{0, 2}, // you, goodbye
		[]float64{1, 3}, // say, and
		[]float64{2, 4}, // goodbye, i
		[]float64{3, 1}, // and, say
		[]float64{4, 5}, // i, hello
		[]float64{1, 6}, // say, .
	)

	target := matrix.New(
		[]float64{1}, // say
		[]float64{2}, // goodbye
		[]float64{3}, // and
		[]float64{4}, // i
		[]float64{1}, // say
		[]float64{5}, // hello
	)

	loss := m.Forward(contexts, target)
	m.Backward()
	fmt.Println(loss)

}

Output:

[[12.47659755078791]]

func NewCBOWNegativeSampling 

func NewCBOWNegativeSampling(c CBOWNegativeSamplingConfig, s ...randv2.Source) *CBOWNegativeSampling

func (*CBOWNegativeSampling) Backward 

func (m *CBOWNegativeSampling) Backward() matrix.Matrix

func (*CBOWNegativeSampling) Forward 

func (m *CBOWNegativeSampling) Forward(contexts, target matrix.Matrix) matrix.Matrix

func (*CBOWNegativeSampling) Grads 

func (m *CBOWNegativeSampling) Grads() [][]matrix.Matrix

func (*CBOWNegativeSampling) Layers 

func (m *CBOWNegativeSampling) Layers() []Layer
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
)

func main() {
	m := model.NewCBOWNegativeSampling(model.CBOWNegativeSamplingConfig{
		CBOWConfig: model.CBOWConfig{
			VocabSize:  7,
			HiddenSize: 5,
		},
		Corpus:     []int{0, 1, 2, 3, 4, 1, 5, 6},
		WindowSize: 1,
		SampleSize: 2,
		Power:      0.75,
	})

	fmt.Printf("%T\n", m)
	for i, l := range m.Layers() {
		fmt.Printf("%2d: %v\n", i, l)
	}

}

Output:

*model.CBOWNegativeSampling
 0: *layer.Embedding: W(7, 5): 35
 1: *layer.Embedding: W(7, 5): 35
 2: *layer.NegativeSamplingLoss: W(7, 5)*3: 105

func (*CBOWNegativeSampling) Params 

func (m *CBOWNegativeSampling) Params() [][]matrix.Matrix
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
)

func main() {
	m := model.NewCBOWNegativeSampling(model.CBOWNegativeSamplingConfig{
		CBOWConfig: model.CBOWConfig{
			VocabSize:  7,
			HiddenSize: 5,
		},
		Corpus:     []int{0, 1, 2, 3, 4, 1, 5, 6},
		WindowSize: 1,
		SampleSize: 2,
		Power:      0.75,
	})

	for _, p := range m.Params() {
		for _, m := range p {
			fmt.Println(m.Dim())
		}
	}

	m.SetParams(m.Grads())
	fmt.Println(m.Params())
	fmt.Println(m.Grads())

}

Output:

7 5
7 5
7 5
7 5
7 5
[[[]] [[]] [[] [] []]]
[[[]] [[]] [[] [] []]]

func (*CBOWNegativeSampling) Predict 

func (m *CBOWNegativeSampling) Predict(x matrix.Matrix, _ ...layer.Opts) matrix.Matrix

func (*CBOWNegativeSampling) SetParams 

func (m *CBOWNegativeSampling) SetParams(p [][]matrix.Matrix)

func (*CBOWNegativeSampling) Summary 

func (m *CBOWNegativeSampling) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
)

func main() {
	m := model.NewCBOWNegativeSampling(model.CBOWNegativeSamplingConfig{
		CBOWConfig: model.CBOWConfig{
			VocabSize:  7,
			HiddenSize: 5,
		},
		Corpus:     []int{0, 1, 2, 3, 4, 1, 5, 6},
		WindowSize: 1,
		SampleSize: 2,
		Power:      0.75,
	})

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.CBOWNegativeSampling
 0: *layer.Embedding: W(7, 5): 35
 1: *layer.Embedding: W(7, 5): 35
 2: *layer.NegativeSamplingLoss: W(7, 5)*3: 105

type CBOWNegativeSamplingConfig 

type CBOWNegativeSamplingConfig struct {
	CBOWConfig
	Corpus     []int
	WindowSize int
	SampleSize int
	Power      float64
}

type Decoder 

type Decoder struct {
	TimeEmbedding *layer.TimeEmbedding
	TimeLSTM      *layer.TimeLSTM
	TimeAffine    *layer.TimeAffine
	Source        randv2.Source
}
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	s := rand.Const(1)
	m := model.NewDecoder(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	}, s)

	// forward
	xs := []matrix.Matrix{
		// (T, N, 1) = (2, 3, 1)
		{{0.1}, {0.2}, {0.3}}, // (N, 1) = (3, 1)
		{{0.1}, {0.2}, {0.3}}, // (N, 1) = (3, 1)
	}

	// (N, H) = (3, 3)
	h := matrix.New(
		[]float64{0.1, 0.2, 0.3},
		[]float64{0.3, 0.4, 0.5},
		[]float64{0.3, 0.4, 0.5},
	)

	score := m.Forward(xs, h)
	fmt.Println(len(score))
	for _, s := range score {
		fmt.Println(s.Dim())
	}

	// backward
	dout := []matrix.Matrix{
		{{0.1}, {0.1}, {0.1}},
		{{0.1}, {0.1}, {0.1}},
	}
	dh := m.Backward(dout)
	fmt.Println(dh.Dim())

	// generate
	sampeld := m.Generate(h, 1, 10)
	fmt.Println(sampeld)

}

Output:

2
3 3
3 3
3 3
[2 2 1 1 1 1 1 1 1 1]

func NewDecoder 

func NewDecoder(c *RNNLMConfig, s ...randv2.Source) *Decoder

func (*Decoder) Backward 

func (m *Decoder) Backward(dscore []matrix.Matrix) matrix.Matrix

func (*Decoder) Forward 

func (m *Decoder) Forward(xs []matrix.Matrix, h matrix.Matrix) []matrix.Matrix

func (*Decoder) Generate 

func (m *Decoder) Generate(h matrix.Matrix, startID, length int) []int

func (*Decoder) Grads 

func (l *Decoder) Grads() []matrix.Matrix

func (*Decoder) Layers 

func (m *Decoder) Layers() []TimeLayer
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewDecoder(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	fmt.Printf("%T\n", m)
	for i, l := range m.Layers() {
		fmt.Printf("%2d: %v\n", i, l)
	}
	fmt.Println()

}

Output:

*model.Decoder
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 2: *layer.TimeAffine: W(3, 3), B(1, 3): 12

func (*Decoder) Params 

func (l *Decoder) Params() []matrix.Matrix
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewDecoder(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	m.SetParams(make([]matrix.Matrix, 6)...)
	fmt.Println(m.Params())
	fmt.Println(m.Grads())

}

Output:

[[] [] [] [] [] []]
[[] [] [] [] [] []]

func (*Decoder) SetParams 

func (l *Decoder) SetParams(p ...matrix.Matrix)

func (*Decoder) Summary 

func (m *Decoder) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewDecoder(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.Decoder
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 2: *layer.TimeAffine: W(3, 3), B(1, 3): 12

type Encoder 

type Encoder struct {
	TimeEmbedding *layer.TimeEmbedding
	TimeLSTM      *layer.TimeLSTM
	Source        randv2.Source
	// contains filtered or unexported fields
}
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	s := rand.Const(1)
	m := model.NewEncoder(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	}, s)

	// forward
	xs := []matrix.Matrix{
		// (T, N, 1) = (2, 3, 1)
		{{0.1}, {0.2}, {0.3}}, // (N, 1) = (3, 1)
		{{0.1}, {0.2}, {0.3}}, // (N, 1) = (3, 1)
	}

	hs := m.Forward(xs)
	fmt.Println(hs.Dim()) // (N, H) = (3, 3)

	// backward
	// (N, H) = (3, 3)
	dh := matrix.New(
		[]float64{0.1, 0.2, 0.3},
		[]float64{0.3, 0.4, 0.5},
		[]float64{0.3, 0.4, 0.5},
	)

	m.Backward(dh)

}

Output:

3 3

func NewEncoder 

func NewEncoder(c *RNNLMConfig, s ...randv2.Source) *Encoder

func (*Encoder) Backward 

func (m *Encoder) Backward(dh matrix.Matrix)

func (*Encoder) Forward 

func (m *Encoder) Forward(xs []matrix.Matrix) matrix.Matrix

func (*Encoder) Grads 

func (l *Encoder) Grads() []matrix.Matrix

func (*Encoder) Layers 

func (m *Encoder) Layers() []TimeLayer
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewEncoder(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	fmt.Printf("%T\n", m)
	for i, l := range m.Layers() {
		fmt.Printf("%2d: %v\n", i, l)
	}
	fmt.Println()

}

Output:

*model.Encoder
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84

func (*Encoder) Params 

func (l *Encoder) Params() []matrix.Matrix
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewEncoder(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	m.SetParams(make([]matrix.Matrix, 4)...)
	fmt.Println(m.Params())
	fmt.Println(m.Grads())

}

Output:

[[] [] [] []]
[[] [] [] []]

func (*Encoder) SetParams 

func (l *Encoder) SetParams(p ...matrix.Matrix)

func (*Encoder) Summary 

func (m *Encoder) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewEncoder(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.Encoder
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84

type GRULM 

type GRULM struct {
	RNNLM
}
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	// model
	s := rand.Const(1)
	m := model.NewGRULM(&model.LSTMLMConfig{
		RNNLMConfig: model.RNNLMConfig{
			VocabSize:   3,
			WordVecSize: 3,
			HiddenSize:  3,
			WeightInit:  weight.Xavier,
		},
		DropoutRatio: 0.5,
	}, s)

	fmt.Printf("%T\n", m)
	for i, l := range m.Layers() {
		fmt.Printf("%2d: %v\n", i, l)
	}
	fmt.Println()

	// data
	xs := []matrix.Matrix{{{0, 1, 2}}}
	ts := []matrix.Matrix{{{0, 1, 2}}}

	loss := m.Forward(xs, ts)
	dout := m.Backward()

	fmt.Printf("%.4f\n", loss)
	fmt.Println(dout)

}

Output:

*model.GRULM
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeDropout: Ratio(0.5)
 2: *layer.TimeGRU: Wx(3, 9), Wh(3, 9), B(1, 9): 63
 3: *layer.TimeDropout: Ratio(0.5)
 4: *layer.TimeGRU: Wx(3, 9), Wh(3, 9), B(1, 9): 63
 5: *layer.TimeDropout: Ratio(0.5)
 6: *layer.TimeAffine: W(3, 3), B(1, 3): 12
 7: *layer.TimeSoftmaxWithLoss

[[[1.0986]]]
[]

func NewGRULM 

func NewGRULM(c *LSTMLMConfig, s ...randv2.Source) *GRULM

func (*GRULM) Summary 

func (m *GRULM) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewGRULM(&model.LSTMLMConfig{
		RNNLMConfig: model.RNNLMConfig{
			VocabSize:   3,
			WordVecSize: 3,
			HiddenSize:  3,
			WeightInit:  weight.Xavier,
		},
		DropoutRatio: 0.5,
	})

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.GRULM
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeDropout: Ratio(0.5)
 2: *layer.TimeGRU: Wx(3, 9), Wh(3, 9), B(1, 9): 63
 3: *layer.TimeDropout: Ratio(0.5)
 4: *layer.TimeGRU: Wx(3, 9), Wh(3, 9), B(1, 9): 63
 5: *layer.TimeDropout: Ratio(0.5)
 6: *layer.TimeAffine: W(3, 3), B(1, 3): 12
 7: *layer.TimeSoftmaxWithLoss

type LSTMLM 

type LSTMLM struct {
	RNNLM
}
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	// model
	s := rand.Const(1)
	m := model.NewLSTMLM(&model.LSTMLMConfig{
		RNNLMConfig: model.RNNLMConfig{
			VocabSize:   3,
			WordVecSize: 3,
			HiddenSize:  3,
			WeightInit:  weight.Xavier,
		},
		DropoutRatio: 0.5,
	}, s)

	fmt.Printf("%T\n", m)
	for i, l := range m.Layers() {
		fmt.Printf("%2d: %v\n", i, l)
	}
	fmt.Println()

	// data
	xs := []matrix.Matrix{{{0, 1, 2}}}
	ts := []matrix.Matrix{{{0, 1, 2}}}

	loss := m.Forward(xs, ts)
	dout := m.Backward()

	fmt.Printf("%.4f\n", loss)
	fmt.Println(dout)

}

Output:

*model.LSTMLM
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeDropout: Ratio(0.5)
 2: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 3: *layer.TimeDropout: Ratio(0.5)
 4: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 5: *layer.TimeDropout: Ratio(0.5)
 6: *layer.TimeAffine: W(3, 3), B(1, 3): 12
 7: *layer.TimeSoftmaxWithLoss

[[[1.0980]]]
[]

func NewLSTMLM 

func NewLSTMLM(c *LSTMLMConfig, s ...randv2.Source) *LSTMLM

func (*LSTMLM) Summary 

func (m *LSTMLM) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewLSTMLM(&model.LSTMLMConfig{
		RNNLMConfig: model.RNNLMConfig{
			VocabSize:   3,
			WordVecSize: 3,
			HiddenSize:  3,
			WeightInit:  weight.Xavier,
		},
		DropoutRatio: 0.5,
	})

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.LSTMLM
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeDropout: Ratio(0.5)
 2: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 3: *layer.TimeDropout: Ratio(0.5)
 4: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 5: *layer.TimeDropout: Ratio(0.5)
 6: *layer.TimeAffine: W(3, 3), B(1, 3): 12
 7: *layer.TimeSoftmaxWithLoss

type LSTMLMConfig 

type LSTMLMConfig struct {
	RNNLMConfig
	DropoutRatio float64
}

type Layer 

type Layer interface {
	Forward(x, y matrix.Matrix, opts ...layer.Opts) matrix.Matrix
	Backward(dout matrix.Matrix) (matrix.Matrix, matrix.Matrix)
	Params() []matrix.Matrix
	Grads() []matrix.Matrix
	SetParams(p ...matrix.Matrix)
	String() string
}

Layer is an interface that represents a layer.

type MLP 

type MLP struct {
	Sequential
}
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	s := rand.Const(1)
	m := model.NewMLP(&model.MLPConfig{
		InputSize:         2,
		OutputSize:        2,
		HiddenSize:        []int{3},
		WeightInit:        weight.Std(0.01),
		BatchNormMomentum: 0.9,
	}, s)

	x := matrix.New([]float64{0.5, 0.5}, []float64{1, 0}, []float64{0, 1})
	t := matrix.New([]float64{1, 0}, []float64{0, 1}, []float64{0, 1})

	loss := m.Forward(x, t)
	m.Backward()
	fmt.Printf("%.4f %v\n", loss, m.Predict(x).Argmax())

}

Output:

[[0.6975]] [0 0 0]
Example (GradientCheck) 
// data
x := matrix.New([]float64{0.5, 0.5}, []float64{1, 0}, []float64{0, 1})
t := matrix.New([]float64{1, 0}, []float64{0, 1}, []float64{0, 1})

// model
s := rand.Const(1)
m := model.NewMLP(&model.MLPConfig{
	InputSize:         2,
	OutputSize:        2,
	HiddenSize:        []int{3},
	WeightInit:        weight.Std(0.01),
	BatchNormMomentum: 0.9,
}, s)

fmt.Printf("%T\n", m)
for i, l := range m.Layers() {
	fmt.Printf("%2d: %v\n", i, l)
}
fmt.Println()

// gradients
m.Forward(x, t)
m.Backward()
grads := m.Grads()
gradsn := numericalGrads(m, x, t)

// check
for i := range gradsn {
	// 20, 21 is ReLU. empty
	for j := range gradsn[i] {
		eps := gradsn[i][j].Sub(grads[i][j]).Abs().Mean() // mean(| A - B |)
		fmt.Printf("%v%v: %v\n", i, j, eps)
	}
}

Output:

*model.MLP
 0: *layer.Affine: W(2, 3), B(1, 3): 9
 1: *layer.BatchNorm: G(1, 3), B(1, 3): 6
 2: *layer.ReLU
 3: *layer.Affine: W(3, 2), B(1, 2): 8
 4: *layer.SoftmaxWithLoss

00: 0.0018878976165139967
01: 9.251858538542972e-17
10: 3.416015634604153e-10
11: 0.0008399733189571372
30: 3.21054273907014e-08
31: 3.421842263706676e-08

func NewMLP 

func NewMLP(c *MLPConfig, s ...randv2.Source) *MLP

func (*MLP) Summary 

func (m *MLP) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewMLP(&model.MLPConfig{
		InputSize:         2,
		OutputSize:        2,
		HiddenSize:        []int{3},
		WeightInit:        weight.Std(0.01),
		BatchNormMomentum: 0.9,
	})

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.MLP
 0: *layer.Affine: W(2, 3), B(1, 3): 9
 1: *layer.BatchNorm: G(1, 3), B(1, 3): 6
 2: *layer.ReLU
 3: *layer.Affine: W(3, 2), B(1, 2): 8
 4: *layer.SoftmaxWithLoss

type MLPConfig 

type MLPConfig struct {
	InputSize         int
	OutputSize        int
	HiddenSize        []int
	WeightInit        WeightInit
	BatchNormMomentum float64
}

type PeekyDecoder 

type PeekyDecoder struct {
	Decoder
	H int
}
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	s := rand.Const(1)
	m := model.NewPeekyDecoder(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	}, s)

	// forward
	xs := []matrix.Matrix{
		// (T, N, 1) = (2, 3, 1)
		{{0.1}, {0.2}, {0.3}}, // (N, 1) = (3, 1)
		{{0.1}, {0.2}, {0.3}}, // (N, 1) = (3, 1)
	}

	// (N, H) = (3, 3)
	h := matrix.New(
		[]float64{0.1, 0.2, 0.3},
		[]float64{0.3, 0.4, 0.5},
		[]float64{0.3, 0.4, 0.5},
	)

	score := m.Forward(xs, h)
	fmt.Println(len(score))
	for _, s := range score {
		fmt.Println(s.Dim())
	}

	// backward
	dout := []matrix.Matrix{
		{{0.1}, {0.1}, {0.1}},
		{{0.1}, {0.1}, {0.1}},
	}
	dh := m.Backward(dout)
	fmt.Println(dh.Dim())

	// generate
	sampeld := m.Generate(h, 1, 10)
	fmt.Println(sampeld)

}

Output:

2
3 3
3 3
3 3
[2 2 2 2 2 2 2 2 2 2]

func NewPeekyDecoder 

func NewPeekyDecoder(c *RNNLMConfig, s ...randv2.Source) *PeekyDecoder

func (*PeekyDecoder) Backward 

func (m *PeekyDecoder) Backward(dscore []matrix.Matrix) matrix.Matrix

func (*PeekyDecoder) Forward 

func (m *PeekyDecoder) Forward(xs []matrix.Matrix, h matrix.Matrix) []matrix.Matrix

func (*PeekyDecoder) Generate 

func (m *PeekyDecoder) Generate(h matrix.Matrix, startID, length int) []int

func (*PeekyDecoder) Summary 

func (m *PeekyDecoder) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewPeekyDecoder(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.PeekyDecoder
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeLSTM: Wx(6, 12), Wh(3, 12), B(1, 12): 120
 2: *layer.TimeAffine: W(6, 3), B(1, 3): 21

type PeekySeq2Seq 

type PeekySeq2Seq struct {
	Seq2Seq
}
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	s := rand.Const(1)
	m := model.NewPeekySeq2Seq(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	}, s)

	fmt.Printf("%T\n", m)
	for i, l := range m.Layers() {
		fmt.Printf("%2d: %v\n", i, l)
	}
	fmt.Println()

}

Output:

*model.PeekySeq2Seq
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 2: *layer.TimeEmbedding: W(3, 3): 9
 3: *layer.TimeLSTM: Wx(6, 12), Wh(3, 12), B(1, 12): 120
 4: *layer.TimeAffine: W(6, 3), B(1, 3): 21
 5: *layer.TimeSoftmaxWithLoss

func NewPeekySeq2Seq 

func NewPeekySeq2Seq(c *RNNLMConfig, s ...randv2.Source) *PeekySeq2Seq

func (*PeekySeq2Seq) Summary 

func (m *PeekySeq2Seq) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewPeekySeq2Seq(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.PeekySeq2Seq
 0: *model.Encoder
 1: *layer.TimeEmbedding: W(3, 3): 9
 2: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 3: *model.PeekyDecoder
 4: *layer.TimeEmbedding: W(3, 3): 9
 5: *layer.TimeLSTM: Wx(6, 12), Wh(3, 12), B(1, 12): 120
 6: *layer.TimeAffine: W(6, 3), B(1, 3): 21
 7: *layer.TimeSoftmaxWithLoss

type QNet 

type QNet struct {
	Sequential
}
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/math/vector"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	s := rand.Const(1)
	m := model.NewQNet(&model.QNetConfig{
		InputSize:  12,
		OutputSize: 4,
		HiddenSize: []int{100},
		WeightInit: weight.Std(0.01),
	}, s)

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

	target := matrix.New([]float64{1})
	qs := m.Predict(matrix.New([]float64{0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0}))
	q := matrix.New([]float64{vector.Max(qs[0])})
	fmt.Printf("%.8f, %0.8f\n", qs, q)
	fmt.Printf("%.8f\n", m.Loss(target, q))
	fmt.Printf("%.8f\n", m.Backward())

	params, grads := m.Params(), m.Grads()
	for i, p := range params {
		for j := range p {
			a, b := params[i][j].Dim()
			c, d := grads[i][j].Dim()
			fmt.Printf("p(%3v, %3v), g(%3v,%3v)\n", a, b, c, d)
		}
	}

}

Output:

*model.QNet
 0: *layer.Affine: W(12, 100), B(1, 100): 1300
 1: *layer.ReLU
 2: *layer.Affine: W(100, 4), B(1, 4): 404
 3: *layer.MeanSquaredError
[[-0.00007664 -0.00098626 -0.00063908 -0.00086191]], [[-0.00007664]]
[[1.00015329]]
[[-0.00108430 0.00035194 0.00033155 -0.00098515 0.00012841 -0.00121666 -0.00180430 0.00153819 -0.00015329 0.00126458 -0.00082200 0.00016637]]
p( 12, 100), g( 12,100)
p(  1, 100), g(  1,100)
p(100,   4), g(100,  1)
p(  1,   4), g(  1,  1)

func NewQNet 

func NewQNet(c *QNetConfig, s ...randv2.Source) *QNet

func (*QNet) Loss 

func (m *QNet) Loss(target, q matrix.Matrix) matrix.Matrix

func (*QNet) Summary 

func (m *QNet) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewQNet(&model.QNetConfig{
		InputSize:  12,
		OutputSize: 4,
		HiddenSize: []int{100},
		WeightInit: weight.Std(0.01),
	})

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.QNet
 0: *layer.Affine: W(12, 100), B(1, 100): 1300
 1: *layer.ReLU
 2: *layer.Affine: W(100, 4), B(1, 4): 404
 3: *layer.MeanSquaredError

func (*QNet) Sync 

func (m *QNet) Sync(q *QNet)
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	diff := func(m, t *model.QNet) []float64 {
		out := make([]float64, 0)
		for i := range m.Params() {
			for j := range m.Params()[i] {
				out = append(out, m.Params()[i][j].Sub(t.Params()[i][j]).Abs().Sum())
			}
		}

		return out
	}

	s := rand.Const(1)
	c := &model.QNetConfig{
		InputSize:  12,
		OutputSize: 4,
		HiddenSize: []int{100},
		WeightInit: weight.Std(0.01),
	}

	m := model.NewQNet(c, s)
	t := model.NewQNet(c, s)
	fmt.Println("new:", diff(m, t))

	t.Sync(m)
	fmt.Println("sync:", diff(m, t))

	m.SetParams(model.NewQNet(c, s).Params())
	fmt.Println("set:", diff(m, t))

	t.Sync(m)
	fmt.Println("sync:", diff(m, t))

}

Output:

new: [13.36619293822008 0 4.599586087033022 0]
sync: [0 0 0 0]
set: [13.515551906185843 0 4.263141626774541 0]
sync: [0 0 0 0]

type QNetConfig 

type QNetConfig struct {
	InputSize  int
	OutputSize int
	HiddenSize []int
	WeightInit WeightInit
}

type RNNLM 

type RNNLM struct {
	Layer  []TimeLayer
	Source randv2.Source
}
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	// model
	s := rand.Const(1)
	m := model.NewRNNLM(&model.RNNLMConfig{
		VocabSize:   3,
		WordVecSize: 3,
		HiddenSize:  3,
		WeightInit:  weight.Xavier,
	}, s)

	fmt.Printf("%T\n", m)
	for i, l := range m.Layers() {
		fmt.Printf("%2d: %v\n", i, l)
	}
	fmt.Println()

	// data
	xs := []matrix.Matrix{{{0, 1, 2}}}
	ts := []matrix.Matrix{{{0, 1, 2}}}

	loss := m.Forward(xs, ts)
	dout := m.Backward()

	fmt.Printf("%.4f\n", loss)
	fmt.Println(dout)

}

Output:

*model.RNNLM
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeRNN: Wx(3, 3), Wh(3, 3), B(1, 3): 21
 2: *layer.TimeAffine: W(3, 3), B(1, 3): 12
 3: *layer.TimeSoftmaxWithLoss

[[[1.1069]]]
[]

func NewRNNLM 

func NewRNNLM(c *RNNLMConfig, s ...randv2.Source) *RNNLM

func (*RNNLM) Backward 

func (m *RNNLM) Backward() []matrix.Matrix

func (*RNNLM) Forward 

func (m *RNNLM) Forward(xs, ts []matrix.Matrix) []matrix.Matrix

func (*RNNLM) Grads 

func (m *RNNLM) Grads() [][]matrix.Matrix

func (*RNNLM) Layers 

func (m *RNNLM) Layers() []TimeLayer
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewRNNLM(&model.RNNLMConfig{
		VocabSize:   3,
		WordVecSize: 3,
		HiddenSize:  3,
		WeightInit:  weight.Xavier,
	})

	fmt.Printf("%T\n", m)
	for i, l := range m.Layers() {
		fmt.Printf("%2d: %v\n", i, l)
	}
	fmt.Println()

}

Output:

*model.RNNLM
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeRNN: Wx(3, 3), Wh(3, 3), B(1, 3): 21
 2: *layer.TimeAffine: W(3, 3), B(1, 3): 12
 3: *layer.TimeSoftmaxWithLoss

func (*RNNLM) Params 

func (m *RNNLM) Params() [][]matrix.Matrix
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	s := rand.Const(1)
	m := model.NewRNNLM(&model.RNNLMConfig{
		VocabSize:   3,
		WordVecSize: 3,
		HiddenSize:  3,
		WeightInit:  weight.Xavier,
	}, s)

	// params
	for _, p := range m.Params() {
		fmt.Println(p)
	}
	fmt.Println()

	// grads
	for _, g := range m.Grads() {
		fmt.Println(g) // empty
	}
	fmt.Println()

	// set params
	m.SetParams(m.Grads())
	for _, p := range m.Params() {
		fmt.Println(p)
	}
	fmt.Println()

}

Output:

[[[-0.008024826241110656 0.00424707052949676 -0.004985070978632815] [-0.009872764577745819 0.004770185009670911 -0.0037300956589935985] [0.01182810122110346 -0.008066822642915392 0.0010337870485847512]]]
[[[-0.4272036569812764 -0.051439283460071226 0.0588887761030214] [-0.11157067538737257 0.42479877444693387 -1.6415745494204534] [-0.969465128610357 -0.4261080517614557 1.0999236840248148]] [[0.18248391466570424 0.5477994083834968 -0.1579822276645198] [-0.48305059627014674 -0.4277824240418346 0.9395249505867087] [0.2701827393831761 0.49870245028370974 -0.1766569492870927]] [[0 0 0]]]
[[[-0.5584779273923454 0.2927783198117813 0.5136963587943321] [-1.208969787150421 -0.6633000273730952 0.19743387804132795] [-0.33298687925268927 -0.6371079759680973 -0.14198751865229117]] [[0 0 0]]]
[]

[[]]
[[] [] []]
[[] []]
[]

[[]]
[[] [] []]
[[] []]
[]

func (*RNNLM) Predict 

func (m *RNNLM) Predict(xs []matrix.Matrix, opts ...layer.Opts) []matrix.Matrix

func (*RNNLM) ResetState 

func (m *RNNLM) ResetState()
Example 
package main

import (
	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	s := rand.Const(1)
	m := model.NewRNNLM(&model.RNNLMConfig{
		VocabSize:   3,
		WordVecSize: 3,
		HiddenSize:  3,
		WeightInit:  weight.Xavier,
	}, s)

	m.ResetState()

}

Output:

func (*RNNLM) SetParams 

func (m *RNNLM) SetParams(p [][]matrix.Matrix)

func (*RNNLM) Summary 

func (m *RNNLM) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewRNNLM(&model.RNNLMConfig{
		VocabSize:   3,
		WordVecSize: 3,
		HiddenSize:  3,
		WeightInit:  weight.Xavier,
	})

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.RNNLM
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeRNN: Wx(3, 3), Wh(3, 3), B(1, 3): 21
 2: *layer.TimeAffine: W(3, 3), B(1, 3): 12
 3: *layer.TimeSoftmaxWithLoss

type RNNLMConfig 

type RNNLMConfig struct {
	VocabSize   int
	WordVecSize int
	HiddenSize  int
	WeightInit  WeightInit
}

type RNNLMGen 

type RNNLMGen struct {
	GRULM
}
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	// model
	s := rand.Const(1)
	m := model.NewRNNLMGen(&model.LSTMLMConfig{
		RNNLMConfig: model.RNNLMConfig{
			VocabSize:   100,
			WordVecSize: 100,
			HiddenSize:  100,
			WeightInit:  weight.Xavier,
		},
		DropoutRatio: 0.5,
	}, s)

	fmt.Println(m.Generate(0, []int{}, 10))
	fmt.Println(m.Generate(0, []int{6, 99, 42}, 10))

}

Output:

[49 20 18 15 24 96 48 31 94 79]
[58 61 2 35 17 90 71 52 40 29]

func NewRNNLMGen 

func NewRNNLMGen(c *LSTMLMConfig, s ...randv2.Source) *RNNLMGen

func (*RNNLMGen) Generate 

func (g *RNNLMGen) Generate(startID int, skipIDs []int, length int) []int

func (*RNNLMGen) Summary 

func (m *RNNLMGen) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewRNNLMGen(&model.LSTMLMConfig{
		RNNLMConfig: model.RNNLMConfig{
			VocabSize:   100,
			WordVecSize: 100,
			HiddenSize:  100,
			WeightInit:  weight.Xavier,
		},
		DropoutRatio: 0.5,
	})

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.RNNLMGen
 0: *layer.TimeEmbedding: W(100, 100): 10000
 1: *layer.TimeDropout: Ratio(0.5)
 2: *layer.TimeGRU: Wx(100, 300), Wh(100, 300), B(1, 300): 60300
 3: *layer.TimeDropout: Ratio(0.5)
 4: *layer.TimeGRU: Wx(100, 300), Wh(100, 300), B(1, 300): 60300
 5: *layer.TimeDropout: Ratio(0.5)
 6: *layer.TimeAffine: W(100, 100), B(1, 100): 10100
 7: *layer.TimeSoftmaxWithLoss

type Seq2Seq 

type Seq2Seq struct {
	Encoder *Encoder
	Decoder decoder
	Softmax *layer.TimeSoftmaxWithLoss
	Source  randv2.Source
}
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	s := rand.Const(1)
	m := model.NewSeq2Seq(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	}, s)

	xs := []matrix.Matrix{{{0, 1, 2}}, {{0, 1, 2}}, {{0, 1, 2}}}
	ts := []matrix.Matrix{{{0, 1, 2}}, {{0, 1, 2}}, {{0, 1, 2}}}

	loss := m.Forward(xs, ts)
	m.Backward()

	fmt.Printf("%.4f\n", loss)
	fmt.Println(m.Generate(xs, 1, 10))

}

Output:

[[[1.0987]]]
[1 1 1 1 1 1 1 1 1 1]

func NewSeq2Seq 

func NewSeq2Seq(c *RNNLMConfig, s ...randv2.Source) *Seq2Seq

func (*Seq2Seq) Backward 

func (m *Seq2Seq) Backward()

func (*Seq2Seq) Forward 

func (m *Seq2Seq) Forward(xs, ts []matrix.Matrix) []matrix.Matrix

func (*Seq2Seq) Generate 

func (m *Seq2Seq) Generate(xs []matrix.Matrix, startID, length int) []int

func (*Seq2Seq) Grads 

func (m *Seq2Seq) Grads() [][]matrix.Matrix

func (*Seq2Seq) Layers 

func (m *Seq2Seq) Layers() []TimeLayer
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewSeq2Seq(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	fmt.Printf("%T\n", m)
	for i, l := range m.Layers() {
		fmt.Printf("%2d: %v\n", i, l)
	}
	fmt.Println()

}

Output:

*model.Seq2Seq
 0: *layer.TimeEmbedding: W(3, 3): 9
 1: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 2: *layer.TimeEmbedding: W(3, 3): 9
 3: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 4: *layer.TimeAffine: W(3, 3), B(1, 3): 12
 5: *layer.TimeSoftmaxWithLoss

func (*Seq2Seq) Params 

func (m *Seq2Seq) Params() [][]matrix.Matrix
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewSeq2Seq(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	m.SetParams(m.Grads())
	fmt.Println(m.Params())
	fmt.Println(m.Grads())

}

Output:

[[[] [] [] []] [[] [] [] [] [] []]]
[[[] [] [] []] [[] [] [] [] [] []]]

func (*Seq2Seq) SetParams 

func (m *Seq2Seq) SetParams(p [][]matrix.Matrix)

func (*Seq2Seq) Summary 

func (m *Seq2Seq) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

func main() {
	m := model.NewSeq2Seq(&model.RNNLMConfig{
		VocabSize:   3, // V
		WordVecSize: 3, // D
		HiddenSize:  3, // H
		WeightInit:  weight.Xavier,
	})

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.Seq2Seq
 0: *model.Encoder
 1: *layer.TimeEmbedding: W(3, 3): 9
 2: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 3: *model.Decoder
 4: *layer.TimeEmbedding: W(3, 3): 9
 5: *layer.TimeLSTM: Wx(3, 12), Wh(3, 12), B(1, 12): 84
 6: *layer.TimeAffine: W(3, 3), B(1, 3): 12
 7: *layer.TimeSoftmaxWithLoss

type Sequential 

type Sequential struct {
	Layer  []Layer
	Source randv2.Source
}
Example (GradientCheck) 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/layer"
	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/math/numerical"
	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
	"github.com/itsubaki/neu/weight"
)

type Model interface {
	Forward(x, t matrix.Matrix) matrix.Matrix
	Layers() []model.Layer
}

func numericalGrads(m Model, x, t matrix.Matrix) [][]matrix.Matrix {
	lossW := func(w ...float64) float64 {
		return m.Forward(x, t)[0][0]
	}

	grad := func(f func(x ...float64) float64, x matrix.Matrix) matrix.Matrix {
		out := make(matrix.Matrix, 0)
		for _, r := range x {
			out = append(out, numerical.Gradient(f, r))
		}

		return out
	}

	grads := make([][]matrix.Matrix, 0)
	for _, l := range m.Layers() {
		g := make([]matrix.Matrix, 0)
		for _, p := range l.Params() {
			g = append(g, grad(lossW, p))
		}

		grads = append(grads, g)
	}

	return grads
}

func main() {
	// weight
	s := rand.Const(1)
	W1 := matrix.Randn(2, 3, s).MulC(weight.Std(0.01)(2))
	B1 := matrix.Zero(1, 3)
	W2 := matrix.Randn(3, 2, s).MulC(weight.Std(0.01)(3))
	B2 := matrix.Zero(1, 2)

	// model
	m := model.NewSequential(
		[]model.Layer{
			&layer.Affine{W: W1, B: B1},
			&layer.ReLU{},
			&layer.Affine{W: W2, B: B2},
			&layer.SoftmaxWithLoss{},
		},
		s,
	)

	// gradients
	x := matrix.New([]float64{0.5, 0.5}, []float64{1, 0}, []float64{0, 1})
	t := matrix.New([]float64{1, 0}, []float64{0, 1}, []float64{0, 1})

	m.Forward(x, t)
	m.Backward()
	grads := m.Grads()
	gradsn := numericalGrads(m, x, t)

	// check
	for i := range gradsn {
		// 10, 11 is ReLU. empty
		for j := range gradsn[i] {
			eps := gradsn[i][j].Sub(grads[i][j]).Abs().Mean() // mean(| A - B |)
			fmt.Printf("%v%v: %v\n", i, j, eps)
		}
	}

}

Output:

00: 4.6971204522838296e-11
01: 9.375737187490574e-11
20: 5.0088649301722556e-11
21: 3.333775858149757e-08

func NewSequential 

func NewSequential(layer []Layer, s randv2.Source) *Sequential

func (*Sequential) Backward 

func (m *Sequential) Backward() matrix.Matrix

func (*Sequential) Forward 

func (m *Sequential) Forward(x, t matrix.Matrix) matrix.Matrix

func (*Sequential) Grads 

func (m *Sequential) Grads() [][]matrix.Matrix

func (*Sequential) Layers 

func (m *Sequential) Layers() []Layer

func (*Sequential) Params 

func (m *Sequential) Params() [][]matrix.Matrix

func (*Sequential) Predict 

func (m *Sequential) Predict(x matrix.Matrix, opts ...layer.Opts) matrix.Matrix

func (*Sequential) SetParams 

func (m *Sequential) SetParams(p [][]matrix.Matrix)

func (*Sequential) Summary 

func (m *Sequential) Summary() []string
Example 
package main

import (
	"fmt"

	"github.com/itsubaki/neu/layer"
	"github.com/itsubaki/neu/math/matrix"
	"github.com/itsubaki/neu/math/rand"
	"github.com/itsubaki/neu/model"
)

func main() {
	m := model.NewSequential(
		[]model.Layer{
			&layer.Affine{
				W: matrix.New([]float64{0.1, 0.2, 0.3}, []float64{0.4, 0.5, 0.6}),
				B: matrix.New([]float64{0.1, 0.2, 0.3}),
			},
			&layer.ReLU{},
			&layer.Affine{
				W: matrix.New([]float64{0.1, 0.2, 0.3}, []float64{0.4, 0.5, 0.6}),
				B: matrix.New([]float64{0.1, 0.2, 0.3}),
			},
			&layer.SoftmaxWithLoss{},
		},
		rand.Const(1),
	)

	fmt.Println(m.Summary()[0])
	for i, s := range m.Summary()[1:] {
		fmt.Printf("%2d: %v\n", i, s)
	}

}

Output:

*model.Sequential
 0: *layer.Affine: W(2, 3), B(1, 3): 9
 1: *layer.ReLU
 2: *layer.Affine: W(2, 3), B(1, 3): 9
 3: *layer.SoftmaxWithLoss

type TimeLayer 

type TimeLayer interface {
	Forward(xs, ys []matrix.Matrix, opts ...layer.Opts) []matrix.Matrix
	Backward(dout []matrix.Matrix) []matrix.Matrix
	Params() []matrix.Matrix
	Grads() []matrix.Matrix
	SetParams(p ...matrix.Matrix)
	SetState(h ...matrix.Matrix)
	ResetState()
	String() string
}

TimeLayer is an interface that represents a time layer.

type WeightInit 

type WeightInit func(prevNodeNum int) float64

WeightInit is an interface that represents a weight initializer.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.