Rework teacher interface

- Add Neural ntework copy

neuralnetwork/main.go

@@ -34,9 +34,9 @@ func main() {
 
 	failCount := 0
 	teacher.Reset()
-	for teacher.Next() {
-		index, _ := nn.Predict(teacher.GetData())
-		expect := teacher.GetExpect()
+	for teacher.NextData() {
+		dataSet, expect := teacher.GetData()
+		index, _ := nn.Predict(dataSet)
 		if expect.At(index, 0) != 1.0 {
 			failCount++
 			fmt.Printf("Fail: %v, %v\n\n", teacher.Index(), expect.At(index, 0))

neuralnetwork/neuralnetworkbase/neuralnetwork.go

@@ -41,47 +41,56 @@ import (
 //
 // Matrix: A
 // Description: A is set of calculated neuron activations after sigmoid correction
-// Format:    0          n           N
+// Format:    0          l           L
 //         ⎡A[0] ⎤ ... ⎡A[0] ⎤ ... ⎡A[0] ⎤
 //         ⎢A[1] ⎥ ... ⎢A[1] ⎥ ... ⎢A[1] ⎥
 //         ⎢ ... ⎥ ... ⎢ ... ⎥ ... ⎢ ... ⎥
 //         ⎢A[i] ⎥ ... ⎢A[i] ⎥ ... ⎢A[i] ⎥
 //         ⎢ ... ⎥ ... ⎢ ... ⎥ ... ⎢ ... ⎥
 //         ⎣A[s] ⎦ ... ⎣A[s] ⎦ ... ⎣A[s] ⎦
-// Where s = Sizes[n], N = len(Sizes)
+// Where s = Sizes[l] - Neural network layer size
+//       L = len(Sizes) - Number of neural network layers
 //
 // Matrix: Z
 // Description: Z is set of calculated raw neuron activations
-// Format:    0          n           N
+// Format:    0          l           L
 //         ⎡Z[0] ⎤ ... ⎡Z[0] ⎤ ... ⎡Z[0] ⎤
 //         ⎢Z[1] ⎥ ... ⎢Z[1] ⎥ ... ⎢Z[1] ⎥
 //         ⎢ ... ⎥ ... ⎢ ... ⎥ ... ⎢ ... ⎥
 //         ⎢Z[i] ⎥ ... ⎢Z[i] ⎥ ... ⎢Z[i] ⎥
 //         ⎢ ... ⎥ ... ⎢ ... ⎥ ... ⎢ ... ⎥
 //         ⎣Z[s] ⎦ ... ⎣Z[s] ⎦ ... ⎣Z[s] ⎦
-// Where s = Sizes[n], N = len(Sizes)
+// Where s = Sizes[l] - Neural network layer size
+//       L = len(Sizes) - Number of neural network layers
 //
 // Matrix: Biases
-// Description: Biases is set of biases per layer except L0
-// Format:
-//         ⎡b[0] ⎤
-//         ⎢b[1] ⎥
-//         ⎢ ... ⎥
-//         ⎢b[i] ⎥
-//         ⎢ ... ⎥
-//         ⎣b[s] ⎦
-// Where s = Sizes[n]
+// Description: Biases is set of biases per layer except l0
+//              NOTE: l0 is always empty Dense because first layer
+//              doesn't have connections to previous layer
+// Format:    1          l           L
+//         ⎡b[0] ⎤ ... ⎡b[0] ⎤ ... ⎡b[0] ⎤
+//         ⎢b[1] ⎥ ... ⎢b[1] ⎥ ... ⎢b[1] ⎥
+//         ⎢ ... ⎥ ... ⎢ ... ⎥ ... ⎢ ... ⎥
+//         ⎢b[i] ⎥ ... ⎢b[i] ⎥ ... ⎢b[i] ⎥
+//         ⎢ ... ⎥ ... ⎢ ... ⎥ ... ⎢ ... ⎥
+//         ⎣b[s] ⎦ ... ⎣b[s] ⎦ ... ⎣b[s] ⎦
+// Where s = Sizes[l] - Neural network layer size
+//       L = len(Sizes) - Number of neural network layers
 //
 // Matrix: Weights
-// Description: Weights is set of weights per layer except L0
-// Format:
-//         ⎡w[0,0] ... w[0,j] ... w[0,s']⎤
-//         ⎢w[1,0] ... w[1,j] ... w[1,s']⎥
-//         ⎢              ...            ⎥
-//         ⎢w[i,0] ... w[i,j] ... w[i,s']⎥
-//         ⎢              ...            ⎥
-//         ⎣w[s,0] ... w[s,j] ... w[s,s']⎦
-// Where s = Sizes[n], s' = Sizes[n-1]
+// Description: Weights is set of weights per layer except l0
+//              NOTE: l0 is always empty Dense because first layer
+//              doesn't have connections to previous layer
+// Format:               1                                   l                                   L
+//         ⎡w[0,0] ... w[0,j] ... w[0,s']⎤ ... ⎡w[0,0] ... w[0,j] ... w[0,s']⎤ ... ⎡w[0,0] ... w[0,j] ... w[0,s']⎤
+//         ⎢w[1,0] ... w[1,j] ... w[1,s']⎥ ... ⎢w[1,0] ... w[1,j] ... w[1,s']⎥ ... ⎢w[1,0] ... w[1,j] ... w[1,s']⎥
+//         ⎢              ...            ⎥ ... ⎢              ...            ⎥ ... ⎢              ...            ⎥
+//         ⎢w[i,0] ... w[i,j] ... w[i,s']⎥ ... ⎢w[i,0] ... w[i,j] ... w[i,s']⎥ ... ⎢w[i,0] ... w[i,j] ... w[i,s']⎥
+//         ⎢              ...            ⎥ ... ⎢              ...            ⎥ ... ⎢              ...            ⎥
+//         ⎣w[s,0] ... w[s,j] ... w[s,s']⎦ ... ⎣w[s,0] ... w[s,j] ... w[s,s']⎦ ... ⎣w[s,0] ... w[s,j] ... w[s,s']⎦
+// Where s = Sizes[l] - Neural network layer size
+//       s' = Sizes[l-1] - Previous neural network layer size
+//       L = len(Sizes) - Number of neural network layers
 
 type NeuralNetwork struct {
 	Count          int
@@ -139,6 +148,24 @@ func NewNeuralNetwork(sizes []int, nu float64, trainingCycles int) (nn *NeuralNe
 	return
 }
 
+func (nn *NeuralNetwork) Copy() (out *NeuralNetwork) {
+	out = &NeuralNetwork{}
+	out.Sizes = nn.Sizes
+	out.Count = nn.Count
+	out.Weights = make([]*mat.Dense, nn.Count)
+	out.Biases = make([]*mat.Dense, nn.Count)
+	out.A = make([]*mat.Dense, nn.Count)
+	out.Z = make([]*mat.Dense, nn.Count)
+	out.alpha = nn.alpha
+	out.trainingCycles = nn.trainingCycles
+
+	for i := 1; i < out.Count; i++ {
+		nn.Weights[i] = mat.DenseCopyOf(out.Weights[i])
+		nn.Biases[i] = mat.DenseCopyOf(out.Biases[i])
+	}
+	return
+}
+
 func (nn *NeuralNetwork) Predict(aIn mat.Matrix) (maxIndex int, max float64) {
 	r, _ := aIn.Dims()
 	if r != nn.Sizes[0] {
@@ -162,8 +189,8 @@ func (nn *NeuralNetwork) Predict(aIn mat.Matrix) (maxIndex int, max float64) {
 
 func (nn *NeuralNetwork) Teach(teacher teach.Teacher) {
 	for i := 0; i < nn.trainingCycles; i++ {
-		for teacher.Next() {
-			nn.backward(teacher.GetData(), teacher.GetExpect())
+		for teacher.NextData() {
+			nn.backward(teacher.GetData())
 		}
 	}
 }
@@ -317,3 +344,7 @@ func (nn *NeuralNetwork) backward(aIn, aOut mat.Matrix) {
 func (nn *NeuralNetwork) result() *mat.Dense {
 	return nn.A[nn.Count-1]
 }
+
+func (nn *NeuralNetwork) TeachResilient(aIn, aOut mat.Matrix, nuP, nuM float64, deltaMin, deltaMax float64) {
+
+}

neuralnetwork/teach/teacher.go

@@ -30,8 +30,11 @@ import (
 )
 
 type Teacher interface {
-	GetData() *mat.Dense
-	GetExpect() *mat.Dense
-	Next() bool
+	GetData() (*mat.Dense, *mat.Dense)
+	NextData() bool
+
+	GetValidator() (*mat.Dense, *mat.Dense)
+	NextValidator() bool
+
 	Reset()
 }

neuralnetwork/teach/textdatareader.go

@@ -29,22 +29,27 @@ import (
 	"bufio"
 	"fmt"
 	"log"
+	"math/rand"
 	"os"
 	"strconv"
 	"strings"
+	"time"
 
 	mat "gonum.org/v1/gonum/mat"
 )
 
 type TextDataReader struct {
-	dataSet []*mat.Dense
-	result  []*mat.Dense
-	index   int
+	dataSet         []*mat.Dense
+	result          []*mat.Dense
+	index           int
+	validationIndex int
+	validationCount int
 }
 
 func NewTextDataReader(filename string) *TextDataReader {
 	r := &TextDataReader{
-		index: 0,
+		index:           0,
+		validationIndex: 0,
 	}
 	r.readData(filename)
 
@@ -127,28 +132,50 @@ func (r *TextDataReader) readData(filename string) {
 			return val / max[r]
 		}, r.dataSet[i])
 	}
-}
 
-func (r *TextDataReader) GetData() *mat.Dense {
-	return r.dataSet[r.index]
+	rand.Seed(time.Now().UnixNano())
+	for k := 0; k < 5; k++ {
+		rand.Shuffle(len(r.dataSet), func(i, j int) {
+			r.result[i], r.result[j] = r.result[j], r.result[i]
+			r.dataSet[i], r.dataSet[j] = r.dataSet[j], r.dataSet[i]
+		})
+	}
+
+	r.validationCount = 0 //len(r.dataSet) / 5
+	r.validationIndex = len(r.dataSet) - r.validationCount
 }
 
-func (r *TextDataReader) GetExpect() *mat.Dense {
-	return r.result[r.index]
+func (r *TextDataReader) GetData() (*mat.Dense, *mat.Dense) {
+	return r.dataSet[r.index], r.result[r.index]
 }
 
-func (r *TextDataReader) Next() bool {
-	r.index++
-	if r.index >= len(r.result) {
+func (r *TextDataReader) NextData() bool {
+	if (r.index + 1) >= len(r.result)-r.validationCount {
 		r.index = 0
 		return false
 	}
+	r.index++
+
+	return true
+}
+
+func (r *TextDataReader) GetValidator() (*mat.Dense, *mat.Dense) {
+	return r.dataSet[r.validationIndex], r.result[r.validationIndex]
+}
+
+func (r *TextDataReader) NextValidator() bool {
+	if (r.validationIndex + 1) >= len(r.dataSet) {
+		r.validationIndex = len(r.dataSet) - r.validationCount
+		return false
+	}
+	r.validationIndex++
 
 	return true
 }
 
 func (r *TextDataReader) Reset() {
 	r.index = 0
+	r.validationIndex = len(r.dataSet) - r.validationCount
 }
 
 func (r *TextDataReader) Index() int {