Complete network

iris.data

@@ -0,0 +1,151 @@
+5.1,3.5,1.4,0.2,Iris-setosa
+4.9,3.0,1.4,0.2,Iris-setosa
+4.7,3.2,1.3,0.2,Iris-setosa
+4.6,3.1,1.5,0.2,Iris-setosa
+5.0,3.6,1.4,0.2,Iris-setosa
+5.4,3.9,1.7,0.4,Iris-setosa
+4.6,3.4,1.4,0.3,Iris-setosa
+5.0,3.4,1.5,0.2,Iris-setosa
+4.4,2.9,1.4,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+5.4,3.7,1.5,0.2,Iris-setosa
+4.8,3.4,1.6,0.2,Iris-setosa
+4.8,3.0,1.4,0.1,Iris-setosa
+4.3,3.0,1.1,0.1,Iris-setosa
+5.8,4.0,1.2,0.2,Iris-setosa
+5.7,4.4,1.5,0.4,Iris-setosa
+5.4,3.9,1.3,0.4,Iris-setosa
+5.1,3.5,1.4,0.3,Iris-setosa
+5.7,3.8,1.7,0.3,Iris-setosa
+5.1,3.8,1.5,0.3,Iris-setosa
+5.4,3.4,1.7,0.2,Iris-setosa
+5.1,3.7,1.5,0.4,Iris-setosa
+4.6,3.6,1.0,0.2,Iris-setosa
+5.1,3.3,1.7,0.5,Iris-setosa
+4.8,3.4,1.9,0.2,Iris-setosa
+5.0,3.0,1.6,0.2,Iris-setosa
+5.0,3.4,1.6,0.4,Iris-setosa
+5.2,3.5,1.5,0.2,Iris-setosa
+5.2,3.4,1.4,0.2,Iris-setosa
+4.7,3.2,1.6,0.2,Iris-setosa
+4.8,3.1,1.6,0.2,Iris-setosa
+5.4,3.4,1.5,0.4,Iris-setosa
+5.2,4.1,1.5,0.1,Iris-setosa
+5.5,4.2,1.4,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+5.0,3.2,1.2,0.2,Iris-setosa
+5.5,3.5,1.3,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+4.4,3.0,1.3,0.2,Iris-setosa
+5.1,3.4,1.5,0.2,Iris-setosa
+5.0,3.5,1.3,0.3,Iris-setosa
+4.5,2.3,1.3,0.3,Iris-setosa
+4.4,3.2,1.3,0.2,Iris-setosa
+5.0,3.5,1.6,0.6,Iris-setosa
+5.1,3.8,1.9,0.4,Iris-setosa
+4.8,3.0,1.4,0.3,Iris-setosa
+5.1,3.8,1.6,0.2,Iris-setosa
+4.6,3.2,1.4,0.2,Iris-setosa
+5.3,3.7,1.5,0.2,Iris-setosa
+5.0,3.3,1.4,0.2,Iris-setosa
+7.0,3.2,4.7,1.4,Iris-versicolor
+6.4,3.2,4.5,1.5,Iris-versicolor
+6.9,3.1,4.9,1.5,Iris-versicolor
+5.5,2.3,4.0,1.3,Iris-versicolor
+6.5,2.8,4.6,1.5,Iris-versicolor
+5.7,2.8,4.5,1.3,Iris-versicolor
+6.3,3.3,4.7,1.6,Iris-versicolor
+4.9,2.4,3.3,1.0,Iris-versicolor
+6.6,2.9,4.6,1.3,Iris-versicolor
+5.2,2.7,3.9,1.4,Iris-versicolor
+5.0,2.0,3.5,1.0,Iris-versicolor
+5.9,3.0,4.2,1.5,Iris-versicolor
+6.0,2.2,4.0,1.0,Iris-versicolor
+6.1,2.9,4.7,1.4,Iris-versicolor
+5.6,2.9,3.6,1.3,Iris-versicolor
+6.7,3.1,4.4,1.4,Iris-versicolor
+5.6,3.0,4.5,1.5,Iris-versicolor
+5.8,2.7,4.1,1.0,Iris-versicolor
+6.2,2.2,4.5,1.5,Iris-versicolor
+5.6,2.5,3.9,1.1,Iris-versicolor
+5.9,3.2,4.8,1.8,Iris-versicolor
+6.1,2.8,4.0,1.3,Iris-versicolor
+6.3,2.5,4.9,1.5,Iris-versicolor
+6.1,2.8,4.7,1.2,Iris-versicolor
+6.4,2.9,4.3,1.3,Iris-versicolor
+6.6,3.0,4.4,1.4,Iris-versicolor
+6.8,2.8,4.8,1.4,Iris-versicolor
+6.7,3.0,5.0,1.7,Iris-versicolor
+6.0,2.9,4.5,1.5,Iris-versicolor
+5.7,2.6,3.5,1.0,Iris-versicolor
+5.5,2.4,3.8,1.1,Iris-versicolor
+5.5,2.4,3.7,1.0,Iris-versicolor
+5.8,2.7,3.9,1.2,Iris-versicolor
+6.0,2.7,5.1,1.6,Iris-versicolor
+5.4,3.0,4.5,1.5,Iris-versicolor
+6.0,3.4,4.5,1.6,Iris-versicolor
+6.7,3.1,4.7,1.5,Iris-versicolor
+6.3,2.3,4.4,1.3,Iris-versicolor
+5.6,3.0,4.1,1.3,Iris-versicolor
+5.5,2.5,4.0,1.3,Iris-versicolor
+5.5,2.6,4.4,1.2,Iris-versicolor
+6.1,3.0,4.6,1.4,Iris-versicolor
+5.8,2.6,4.0,1.2,Iris-versicolor
+5.0,2.3,3.3,1.0,Iris-versicolor
+5.6,2.7,4.2,1.3,Iris-versicolor
+5.7,3.0,4.2,1.2,Iris-versicolor
+5.7,2.9,4.2,1.3,Iris-versicolor
+6.2,2.9,4.3,1.3,Iris-versicolor
+5.1,2.5,3.0,1.1,Iris-versicolor
+5.7,2.8,4.1,1.3,Iris-versicolor
+6.3,3.3,6.0,2.5,Iris-virginica
+5.8,2.7,5.1,1.9,Iris-virginica
+7.1,3.0,5.9,2.1,Iris-virginica
+6.3,2.9,5.6,1.8,Iris-virginica
+6.5,3.0,5.8,2.2,Iris-virginica
+7.6,3.0,6.6,2.1,Iris-virginica
+4.9,2.5,4.5,1.7,Iris-virginica
+7.3,2.9,6.3,1.8,Iris-virginica
+6.7,2.5,5.8,1.8,Iris-virginica
+7.2,3.6,6.1,2.5,Iris-virginica
+6.5,3.2,5.1,2.0,Iris-virginica
+6.4,2.7,5.3,1.9,Iris-virginica
+6.8,3.0,5.5,2.1,Iris-virginica
+5.7,2.5,5.0,2.0,Iris-virginica
+5.8,2.8,5.1,2.4,Iris-virginica
+6.4,3.2,5.3,2.3,Iris-virginica
+6.5,3.0,5.5,1.8,Iris-virginica
+7.7,3.8,6.7,2.2,Iris-virginica
+7.7,2.6,6.9,2.3,Iris-virginica
+6.0,2.2,5.0,1.5,Iris-virginica
+6.9,3.2,5.7,2.3,Iris-virginica
+5.6,2.8,4.9,2.0,Iris-virginica
+7.7,2.8,6.7,2.0,Iris-virginica
+6.3,2.7,4.9,1.8,Iris-virginica
+6.7,3.3,5.7,2.1,Iris-virginica
+7.2,3.2,6.0,1.8,Iris-virginica
+6.2,2.8,4.8,1.8,Iris-virginica
+6.1,3.0,4.9,1.8,Iris-virginica
+6.4,2.8,5.6,2.1,Iris-virginica
+7.2,3.0,5.8,1.6,Iris-virginica
+7.4,2.8,6.1,1.9,Iris-virginica
+7.9,3.8,6.4,2.0,Iris-virginica
+6.4,2.8,5.6,2.2,Iris-virginica
+6.3,2.8,5.1,1.5,Iris-virginica
+6.1,2.6,5.6,1.4,Iris-virginica
+7.7,3.0,6.1,2.3,Iris-virginica
+6.3,3.4,5.6,2.4,Iris-virginica
+6.4,3.1,5.5,1.8,Iris-virginica
+6.0,3.0,4.8,1.8,Iris-virginica
+6.9,3.1,5.4,2.1,Iris-virginica
+6.7,3.1,5.6,2.4,Iris-virginica
+6.9,3.1,5.1,2.3,Iris-virginica
+5.8,2.7,5.1,1.9,Iris-virginica
+6.8,3.2,5.9,2.3,Iris-virginica
+6.7,3.3,5.7,2.5,Iris-virginica
+6.7,3.0,5.2,2.3,Iris-virginica
+6.3,2.5,5.0,1.9,Iris-virginica
+6.5,3.0,5.2,2.0,Iris-virginica
+6.2,3.4,5.4,2.3,Iris-virginica
+5.9,3.0,5.1,1.8,Iris-virginica
+

neuralnetwork/main.go

@@ -11,31 +11,48 @@ func main() {
 
 	dataSet, result := readData("./iris.data")
 
-	sizes := []int{4, 2, 2, 3}
+	sizes := []int{4, 8, 3}
 	nn := neuralnetwork.NewNeuralNetwork(sizes)
 
-	for i := 0; i < len(dataSet); i++ {
-		fmt.Printf("Dataset[%d]:\n%v\n\n", i, mat.Formatted(dataSet[i], mat.Prefix(""), mat.Excerpt(0)))
-		fmt.Printf("Result[%d]:\n%v\n\n", i, mat.Formatted(result[i], mat.Prefix(""), mat.Excerpt(0)))
-		nn.Backward(dataSet[i], result[i])
+	for i := 0; i < nn.Count; i++ {
+		if i > 0 {
+			fmt.Printf("Weights before:\n%v\n\n", mat.Formatted(nn.Weights[i], mat.Prefix(""), mat.Excerpt(0)))
+			fmt.Printf("Biases before:\n%v\n\n", mat.Formatted(nn.Biases[i], mat.Prefix(""), mat.Excerpt(0)))
+			fmt.Printf("Z before:\n%v\n\n", mat.Formatted(nn.Z[i], mat.Prefix(""), mat.Excerpt(0)))
+		}
+		fmt.Printf("A before:\n%v\n\n", mat.Formatted(nn.A[i], mat.Prefix(""), mat.Excerpt(0)))
 	}
 
+	for j := 0; j < 150; j++ {
+		for i := 0; i < len(dataSet); i++ {
+			// 	fmt.Printf("Dataset[%d]:\n%v\n\n", i, mat.Formatted(dataSet[i], mat.Prefix(""), mat.Excerpt(0)))
+			// 	fmt.Printf("Result[%d]:\n%v\n\n", i, mat.Formatted(result[i], mat.Prefix(""), mat.Excerpt(0)))
+			nn.Backward(dataSet[i], result[i])
+		}
+	}
+
+	for i := 0; i < nn.Count; i++ {
+		if i > 0 {
+			fmt.Printf("Weights after:\n%v\n\n", mat.Formatted(nn.Weights[i], mat.Prefix(""), mat.Excerpt(0)))
+			fmt.Printf("Biases after:\n%v\n\n", mat.Formatted(nn.Biases[i], mat.Prefix(""), mat.Excerpt(0)))
+			fmt.Printf("Z after:\n%v\n\n", mat.Formatted(nn.Z[i], mat.Prefix(""), mat.Excerpt(0)))
+		}
+		fmt.Printf("A after:\n%v\n\n", mat.Formatted(nn.A[i], mat.Prefix(""), mat.Excerpt(0)))
+	}
 	// data := make([]float64, sizes[0])
 	// for i := range data {
 	// 	data[i] = rand.Float64()
 	// }
 	// aIn := mat.NewDense(sizes[0], 1, data)
 
-	max, index := nn.Predict(dataSet[0])
-
-	for i := 0; i < nn.Count; i++ {
-		if i > 0 {
-			fmt.Printf("Weights:\n%v\n\n", mat.Formatted(nn.Weights[i], mat.Prefix(""), mat.Excerpt(0)))
-			fmt.Printf("Biases:\n%v\n\n", mat.Formatted(nn.Biases[i], mat.Prefix(""), mat.Excerpt(0)))
-			fmt.Printf("Z:\n%v\n\n", mat.Formatted(nn.Z[i], mat.Prefix(""), mat.Excerpt(0)))
+	failCount := 0
+	for i := 0; i < len(dataSet); i++ {
+		index, _ := nn.Predict(dataSet[i])
+		if result[i].At(index, 0) != 1.0 {
+			failCount++
+			fmt.Printf("Fail: %v, %v\n\n", i, result[i].At(index, 0))
 		}
-		fmt.Printf("A:\n%v\n\n", mat.Formatted(nn.A[i], mat.Prefix(""), mat.Excerpt(0)))
 	}
 
-	fmt.Printf("Resul: %v, %v\n\n", index, max)
+	fmt.Printf("Fail count: %v\n\n", failCount)
 }

neuralnetwork/neuralnetworkbase/neuralnetwork.go

@@ -1,6 +1,8 @@
 package neuralnetworkbase
 
 import (
+	"fmt"
+
 	mat "gonum.org/v1/gonum/mat"
 )
 
@@ -26,7 +28,7 @@ func NewNeuralNetwork(Sizes []int) (nn *NeuralNetwork) {
 	nn.Biases = make([]*mat.Dense, nn.Count)
 	nn.A = make([]*mat.Dense, nn.Count)
 	nn.Z = make([]*mat.Dense, nn.Count)
-	nn.alpha = 0.2 / float64(nn.Sizes[0])
+	nn.alpha = 0.3 / float64(nn.Sizes[0])
 
 	for i := 1; i < nn.Count; i++ {
 		nn.Weights[i] = generateRandomDense(nn.Sizes[i], nn.Sizes[i-1])
@@ -58,6 +60,12 @@ func (nn *NeuralNetwork) Forward(aIn mat.Matrix) {
 		aSrc := nn.A[i-1]
 		aDst := nn.A[i]
 
+		// r, c := nn.Weights[i].Dims()
+		// fmt.Printf("r: %v,c: %v\n", r, c)
+
+		// r, c = aSrc.Dims()
+		// fmt.Printf("src r: %v,c: %v\n\n\n", r, c)
+
 		aDst.Mul(nn.Weights[i], aSrc)
 		aDst.Add(aDst, nn.Biases[i])
 		nn.Z[i] = mat.DenseCopyOf(aDst)
@@ -66,28 +74,56 @@ func (nn *NeuralNetwork) Forward(aIn mat.Matrix) {
 }
 
 func (nn *NeuralNetwork) Backward(aIn, aOut mat.Matrix) {
-	nn.Forward(aOut)
+	nn.Forward(aIn)
+
+	fmt.Printf("Backward\n")
+	lastLayerNum := nn.Count - 1
 
 	//Initial error
 	err := &mat.Dense{}
-	err.Sub(aOut, nn.Result())
+	err.Sub(nn.Result(), aOut)
+
+	sigmoidsPrime := &mat.Dense{}
+	sigmoidsPrime.Apply(applySigmoidPrime, nn.Z[lastLayerNum])
+
+	delta := &mat.Dense{}
+	delta.MulElem(err, sigmoidsPrime)
 
-	for i := nn.Count - 1; i > 0; i-- {
+	biases := mat.DenseCopyOf(delta)
+
+	weights := &mat.Dense{}
+	weights.Mul(delta, nn.A[lastLayerNum-1].T())
+
+	newBiases := []*mat.Dense{makeBackGradien(biases, nn.Biases[lastLayerNum], nn.alpha)}
+	newWeights := []*mat.Dense{makeBackGradien(weights, nn.Weights[lastLayerNum], nn.alpha)}
+
+	err = delta
+	for i := nn.Count - 2; i > 0; i-- {
 		sigmoidsPrime := &mat.Dense{}
 		sigmoidsPrime.Apply(applySigmoidPrime, nn.Z[i])
 
 		delta := &mat.Dense{}
-		delta.MulElem(err, sigmoidsPrime)
+		wdelta := &mat.Dense{}
+		wdelta.Mul(nn.Weights[i+1].T(), err)
+
+		delta.MulElem(wdelta, sigmoidsPrime)
 		err = delta
 
 		biases := mat.DenseCopyOf(delta)
+
 		weights := &mat.Dense{}
-		weights.Mul(nn.A[i-1], delta)
+		weights.Mul(delta, nn.A[i-1].T())
 
 		// Scale down
-		nn.Weights[i] = makeBackGradien(weights, nn.Weights[i], nn.alpha)
-		nn.Biases[i] = makeBackGradien(biases, nn.Biases[i], nn.alpha)
+		newBiases = append([]*mat.Dense{makeBackGradien(biases, nn.Biases[i], nn.alpha)}, newBiases...)
+		newWeights = append([]*mat.Dense{makeBackGradien(weights, nn.Weights[i], nn.alpha)}, newWeights...)
 	}
+
+	newBiases = append([]*mat.Dense{&mat.Dense{}}, newBiases...)
+	newWeights = append([]*mat.Dense{&mat.Dense{}}, newWeights...)
+
+	nn.Biases = newBiases
+	nn.Weights = newWeights
 }
 
 func makeBackGradien(in mat.Matrix, actual mat.Matrix, alpha float64) *mat.Dense {