Implement basic genetic

neuralnetwork/genetic/genetic.go

@@ -3,18 +3,28 @@ package genetic
 import (
 	"log"
 
+	"sort"
+
 	neuralnetwork "../neuralnetwork"
 )
 
 type Population struct {
-	Networks []*neuralnetwork.NeuralNetwork
-	verifier PopulationVerifier
+	populationSize int
+	Networks       []*neuralnetwork.NeuralNetwork
+	verifier       PopulationVerifier
+	mutagen        Mutagen
 }
 
-func NewPopulation(verifier PopulationVerifier, populationSize int, sizes []int) (p *Population) {
+func NewPopulation(verifier PopulationVerifier, mutagen Mutagen, populationSize int, sizes []int) (p *Population) {
+	if populationSize%2 != 0 {
+		return nil
+	}
+
 	p = &Population{
-		verifier: verifier,
-		Networks: make([]*neuralnetwork.NeuralNetwork, populationSize),
+		populationSize: populationSize,
+		Networks:       make([]*neuralnetwork.NeuralNetwork, populationSize),
+		verifier:       verifier,
+		mutagen:        mutagen,
 	}
 
 	for i := 0; i < populationSize; i++ {
@@ -29,5 +39,43 @@ func NewPopulation(verifier PopulationVerifier, populationSize int, sizes []int)
 }
 
 func (p *Population) NaturalSelection(generationCount int) {
+	for g := 0; g < generationCount; g++ {
+		p.crossbreedPopulation(p.verifier.Verify(p))
+	}
+}
+
+func (p *Population) crossbreedPopulation(results []*IndividalResult) {
+	sort.Slice(results, func(i, j int) bool {
+		return results[i].result < results[j].result
+	})
 
+	for i := 1; i < p.populationSize; i += 2 {
+		firstParent := results[i].index
+		secondParent := results[i-1].index
+		crossbreed(p.Networks[firstParent], p.Networks[secondParent])
+		p.mutagen.Mutate(p.Networks[firstParent])
+		p.mutagen.Mutate(p.Networks[secondParent])
+	}
+}
+
+func crossbreed(firstParent, secondParent *neuralnetwork.NeuralNetwork) {
+	for l := 1; l < firstParent.LayerCount; l++ {
+		firstParentWeights := firstParent.Weights[l]
+		secondParentWeights := secondParent.Weights[l]
+		firstParentBiases := firstParent.Biases[l]
+		secondParentBiases := secondParent.Biases[l]
+		r, c := firstParentWeights.Dims()
+		for i := 0; i < r/2; i++ {
+			for j := 0; j < c; j++ {
+				// Swap first half of weights
+				w := firstParentWeights.At(i, j)
+				firstParentWeights.Set(i, j, secondParentWeights.At(i, j))
+				secondParentWeights.Set(i, j, w)
+			}
+			// Swap first half of biases
+			b := firstParentBiases.At(i, 0)
+			firstParentBiases.Set(i, 0, secondParentBiases.At(i, 0))
+			secondParentBiases.Set(i, 0, b)
+		}
+	}
 }

neuralnetwork/genetic/interface.go

@@ -1,9 +1,16 @@
 package genetic
 
-import (
-	"gonum.org/v1/gonum/mat"
-)
+import neuralnetwork "../neuralnetwork"
+
+type IndividalResult struct {
+	result float64
+	index  int
+}
 
 type PopulationVerifier interface {
-	Verify(Population) *mat.Dense
+	Verify(*Population) []*IndividalResult
+}
+
+type Mutagen interface {
+	Mutate(network *neuralnetwork.NeuralNetwork)
 }

neuralnetwork/genetic/mutagen/dummymutagen.go

@@ -0,0 +1,33 @@
+package mutagen
+
+import (
+	"math/rand"
+	"time"
+
+	neuralnetwork "../../neuralnetwork"
+)
+
+type DummyMutagen struct {
+	chance uint32
+}
+
+func NewDummyMutagen(chance uint32) (rm *DummyMutagen) {
+	rm = &DummyMutagen{
+		chance: chance,
+	}
+	return
+}
+
+func (rm *DummyMutagen) Mutate(network *neuralnetwork.NeuralNetwork) {
+	rand.Seed(time.Now().UnixNano())
+	for l := 1; l < network.LayerCount; l++ {
+		randomized := rand.Float64()
+		if randomized < (float64(rm.chance) / 100.0) {
+			r, c := network.Weights[l].Dims()
+			mutationRow := int(rand.Uint32()) % r
+			mutationColumn := int(rand.Uint32()) % c
+			weight := rand.Float64()
+			network.Weights[l].Set(mutationRow, mutationColumn, weight)
+		}
+	}
+}

neuralnetwork/main.go

@@ -6,6 +6,7 @@ import (
 )
 
 func main() {
+	// genetic.NewPopulation(nil, mutagen.NewDummyMutagen(50), 200, []int{13, 8, 12, 3})
 	sizes := []int{13, 8, 12, 3}
 	nn, _ := neuralnetwork.NewNeuralNetwork(sizes, neuralnetwork.NewRPropInitializer(neuralnetwork.RPropConfig{
 		NuPlus:   1.2,

neuralnetwork/neuralnetwork/batchworker.go

@@ -40,11 +40,11 @@ type batchWorker struct {
 func newBatchWorker(nn *NeuralNetwork) (bw *batchWorker) {
 	bw = &batchWorker{
 		network:   nn,
-		BGradient: make([]BatchGradientDescent, nn.layerCount),
-		WGradient: make([]BatchGradientDescent, nn.layerCount),
+		BGradient: make([]BatchGradientDescent, nn.LayerCount),
+		WGradient: make([]BatchGradientDescent, nn.LayerCount),
 	}
 
-	for l := 1; l < nn.layerCount; l++ {
+	for l := 1; l < nn.LayerCount; l++ {
 		bw.BGradient[l] = nn.gradientDescentInitializer(nn, l, BiasGradient).(BatchGradientDescent)
 		bw.WGradient[l] = nn.gradientDescentInitializer(nn, l, WeightGradient).(BatchGradientDescent)
 	}
@@ -54,7 +54,7 @@ func newBatchWorker(nn *NeuralNetwork) (bw *batchWorker) {
 func (bw *batchWorker) Run(trainer training.Trainer, startIndex, endIndex int) {
 	for i := startIndex; i < endIndex; i++ {
 		dB, dW := bw.network.backward(trainer.GetDataByIndex(i))
-		for l := 1; l < bw.network.layerCount; l++ {
+		for l := 1; l < bw.network.LayerCount; l++ {
 			bw.BGradient[l].AccumGradients(dB[l])
 			bw.WGradient[l].AccumGradients(dW[l])
 		}

neuralnetwork/neuralnetwork/neuralnetwork.go

@@ -99,7 +99,7 @@ import (
 //       L = len(Sizes) - Number of neural network layers
 
 type NeuralNetwork struct {
-	layerCount                 int
+	LayerCount                 int
 	Sizes                      []int
 	Biases                     []*mat.Dense
 	Weights                    []*mat.Dense
@@ -125,15 +125,15 @@ func NewNeuralNetwork(sizes []int, gradientDescentInitializer GradientDescentIni
 
 	nn = &NeuralNetwork{}
 	nn.Sizes = sizes
-	nn.layerCount = len(sizes)
-	nn.Biases = make([]*mat.Dense, nn.layerCount)
-	nn.Weights = make([]*mat.Dense, nn.layerCount)
-	nn.BGradient = make([]interface{}, nn.layerCount)
-	nn.WGradient = make([]interface{}, nn.layerCount)
+	nn.LayerCount = len(sizes)
+	nn.Biases = make([]*mat.Dense, nn.LayerCount)
+	nn.Weights = make([]*mat.Dense, nn.LayerCount)
+	nn.BGradient = make([]interface{}, nn.LayerCount)
+	nn.WGradient = make([]interface{}, nn.LayerCount)
 
 	nn.gradientDescentInitializer = gradientDescentInitializer
 
-	for l := 1; l < nn.layerCount; l++ {
+	for l := 1; l < nn.LayerCount; l++ {
 		nn.Biases[l] = generateRandomDense(nn.Sizes[l], 1)
 		nn.Weights[l] = generateRandomDense(nn.Sizes[l], nn.Sizes[l-1])
 		nn.BGradient[l] = nn.gradientDescentInitializer(nn, l, BiasGradient)
@@ -162,7 +162,7 @@ func (nn *NeuralNetwork) Predict(aIn mat.Matrix) (maxIndex int, max float64) {
 	}
 
 	A, _ := nn.forward(aIn)
-	result := A[nn.layerCount-1]
+	result := A[nn.LayerCount-1]
 	r, _ = result.Dims()
 	max = 0.0
 	maxIndex = 0
@@ -180,9 +180,9 @@ func (nn *NeuralNetwork) Train(trainer training.Trainer, epocs int) {
 		nn.watcher.UpdateState(StateLearning)
 		defer nn.watcher.UpdateState(StateIdle)
 	}
-	if _, ok := nn.WGradient[nn.layerCount-1].(OnlineGradientDescent); ok {
+	if _, ok := nn.WGradient[nn.LayerCount-1].(OnlineGradientDescent); ok {
 		nn.TrainOnline(trainer, epocs)
-	} else if _, ok := nn.WGradient[nn.layerCount-1].(BatchGradientDescent); ok {
+	} else if _, ok := nn.WGradient[nn.LayerCount-1].(BatchGradientDescent); ok {
 		nn.TrainBatch(trainer, epocs)
 	} else {
 		panic("Invalid gradient descent type")
@@ -193,7 +193,7 @@ func (nn *NeuralNetwork) TrainOnline(trainer training.Trainer, epocs int) {
 	for t := 0; t < epocs; t++ {
 		for trainer.NextData() {
 			dB, dW := nn.backward(trainer.GetData())
-			for l := 1; l < nn.layerCount; l++ {
+			for l := 1; l < nn.LayerCount; l++ {
 				bGradient, ok := nn.BGradient[l].(OnlineGradientDescent)
 				if !ok {
 					panic("bGradient is not a OnlineGradientDescent")
@@ -218,7 +218,7 @@ func (nn *NeuralNetwork) TrainBatch(trainer training.Trainer, epocs int) {
 	for t := 0; t < epocs; t++ {
 		batchWorkers := nn.runBatchWorkers(runtime.NumCPU(), trainer)
 
-		for l := 1; l < nn.layerCount; l++ {
+		for l := 1; l < nn.LayerCount; l++ {
 			bGradient, ok := nn.BGradient[l].(BatchGradientDescent)
 			if !ok {
 				panic("bGradient is not a BatchGradientDescent")
@@ -264,15 +264,15 @@ func (nn *NeuralNetwork) runBatchWorkers(threadCount int, trainer training.Train
 func (nn *NeuralNetwork) SaveState(writer io.Writer) {
 	//save input array count
 	bufferSize := make([]byte, 4)
-	binary.LittleEndian.PutUint32(bufferSize[0:], uint32(nn.layerCount))
+	binary.LittleEndian.PutUint32(bufferSize[0:], uint32(nn.LayerCount))
 	_, err := writer.Write(bufferSize)
 
 	check(err)
-	fmt.Printf("wrote value %d\n", uint32(nn.layerCount))
+	fmt.Printf("wrote value %d\n", uint32(nn.LayerCount))
 
 	// save an input array
-	buffer := make([]byte, nn.layerCount*4)
-	for i := 0; i < nn.layerCount; i++ {
+	buffer := make([]byte, nn.LayerCount*4)
+	for i := 0; i < nn.LayerCount; i++ {
 		binary.LittleEndian.PutUint32(buffer[i*4:], uint32(nn.Sizes[i]))
 	}
 
@@ -282,26 +282,26 @@ func (nn *NeuralNetwork) SaveState(writer io.Writer) {
 
 	//save biases
 	////////////////////////
-	for i := 1; i < nn.layerCount; i++ {
+	for i := 1; i < nn.LayerCount; i++ {
 		saveDense(writer, nn.Biases[i])
 	}
 
 	//save weights
 	////////////////////////
-	for i := 1; i < nn.layerCount; i++ {
+	for i := 1; i < nn.LayerCount; i++ {
 		saveDense(writer, nn.Weights[i])
 	}
 }
 
 func (nn *NeuralNetwork) LoadState(reader io.Reader) {
 	// Reade count
-	nn.layerCount = readInt(reader)
+	nn.LayerCount = readInt(reader)
 
 	// Read an input array
-	sizeBuffer := readByteArray(reader, nn.layerCount*4)
-	nn.Sizes = make([]int, nn.layerCount)
+	sizeBuffer := readByteArray(reader, nn.LayerCount*4)
+	nn.Sizes = make([]int, nn.LayerCount)
 
-	for i := 0; i < nn.layerCount; i++ {
+	for i := 0; i < nn.LayerCount; i++ {
 		nn.Sizes[i] = int(binary.LittleEndian.Uint32(sizeBuffer[i*4:]))
 		// fmt.Printf("LoadState: nn.Sizes[%d] %d \n", i, nn.Sizes[i])
 	}
@@ -311,14 +311,14 @@ func (nn *NeuralNetwork) LoadState(reader io.Reader) {
 
 	// read Biases
 	nn.Biases[0] = &mat.Dense{}
-	for i := 1; i < nn.layerCount; i++ {
+	for i := 1; i < nn.LayerCount; i++ {
 		nn.Biases = append(nn.Biases, &mat.Dense{})
 		nn.Biases[i] = readDense(reader, nn.Biases[i])
 	}
 
 	// read Weights
 	nn.Weights[0] = &mat.Dense{}
-	for i := 1; i < nn.layerCount; i++ {
+	for i := 1; i < nn.LayerCount; i++ {
 		nn.Weights = append(nn.Weights, &mat.Dense{})
 		nn.Weights[i] = readDense(reader, nn.Weights[i])
 	}
@@ -327,8 +327,8 @@ func (nn *NeuralNetwork) LoadState(reader io.Reader) {
 }
 
 func (nn NeuralNetwork) forward(aIn mat.Matrix) (A, Z []*mat.Dense) {
-	A = make([]*mat.Dense, nn.layerCount)
-	Z = make([]*mat.Dense, nn.layerCount)
+	A = make([]*mat.Dense, nn.LayerCount)
+	Z = make([]*mat.Dense, nn.LayerCount)
 
 	A[0] = mat.DenseCopyOf(aIn)
 
@@ -336,7 +336,7 @@ func (nn NeuralNetwork) forward(aIn mat.Matrix) (A, Z []*mat.Dense) {
 		nn.watcher.UpdateActivations(0, A[0])
 	}
 
-	for l := 1; l < nn.layerCount; l++ {
+	for l := 1; l < nn.LayerCount; l++ {
 		A[l] = mat.NewDense(nn.Sizes[l], 1, nil)
 		aSrc := A[l-1]
 		aDst := A[l]
@@ -367,9 +367,9 @@ func (nn NeuralNetwork) forward(aIn mat.Matrix) (A, Z []*mat.Dense) {
 func (nn NeuralNetwork) backward(aIn, aOut mat.Matrix) (dB, dW []*mat.Dense) {
 	A, Z := nn.forward(aIn)
 
-	lastLayerNum := nn.layerCount - 1
-	dB = make([]*mat.Dense, nn.layerCount)
-	dW = make([]*mat.Dense, nn.layerCount)
+	lastLayerNum := nn.LayerCount - 1
+	dB = make([]*mat.Dense, nn.LayerCount)
+	dW = make([]*mat.Dense, nn.LayerCount)
 
 	// To calculate new values of weights and biases
 	// following formulas are used:
@@ -383,7 +383,7 @@ func (nn NeuralNetwork) backward(aIn, aOut mat.Matrix) (dB, dW []*mat.Dense) {
 	// error = A[L]-y
 	// Where y is expected activations set
 	err := &mat.Dense{}
-	err.Sub(A[nn.layerCount-1], aOut)
+	err.Sub(A[nn.LayerCount-1], aOut)
 
 	// Calculate sigmoids prime σ'(Z[L]) for last layer L
 	sigmoidsPrime := &mat.Dense{}
@@ -412,7 +412,7 @@ func (nn NeuralNetwork) backward(aIn, aOut mat.Matrix) (dB, dW []*mat.Dense) {
 	// δ[l] = ((Wt[l+1])*δ[l+1])⊙σ'(Z[l])
 	// Where Wt[l+1] is transposed matrix of actual Weights from
 	// forward step
-	for l := nn.layerCount - 2; l > 0; l-- {
+	for l := nn.LayerCount - 2; l > 0; l-- {
 		// Calculate sigmoids prime σ'(Z[l]) for last layer l
 		sigmoidsPrime := &mat.Dense{}
 		sigmoidsPrime.Apply(applySigmoidPrime, Z[l])