Isolate multithread training and prediction

Add initialization of gradient descent for LoadState

neuralnetwork/neuralnetwork/neuralnetwork.go

@@ -107,6 +107,7 @@ type NeuralNetwork struct {
 	WGradient                  []interface{}
 	gradientDescentInitializer GradientDescentInitializer
 	watcher                    StateWatcher
+	syncMutex                  sync.Mutex
 }
 
 func NewNeuralNetwork(sizes []int, gradientDescentInitializer GradientDescentInitializer) (nn *NeuralNetwork, err error) {
@@ -176,6 +177,8 @@ func (nn *NeuralNetwork) SetStateWatcher(watcher StateWatcher) {
 }
 
 func (nn *NeuralNetwork) Predict(aIn mat.Matrix) (maxIndex int, max float64) {
+	nn.syncMutex.Lock()
+	defer nn.syncMutex.Unlock()
 	if nn.watcher != nil {
 		nn.watcher.UpdateState(StatePredict)
 		defer nn.watcher.UpdateState(StateIdle)
@@ -217,6 +220,7 @@ func (nn *NeuralNetwork) Train(trainer training.Trainer, epocs int) {
 func (nn *NeuralNetwork) TrainOnline(trainer training.Trainer, epocs int) {
 	for t := 0; t < epocs; t++ {
 		for trainer.NextData() {
+			nn.syncMutex.Lock()
 			dB, dW := nn.backward(trainer.GetData())
 			for l := 1; l < nn.LayerCount; l++ {
 				bGradient, ok := nn.BGradient[l].(OnlineGradientDescent)
@@ -234,6 +238,7 @@ func (nn *NeuralNetwork) TrainOnline(trainer training.Trainer, epocs int) {
 					nn.watcher.UpdateWeights(l, nn.Weights[l])
 				}
 			}
+			nn.syncMutex.Unlock()
 		}
 		trainer.Reset()
 	}
@@ -243,7 +248,7 @@ func (nn *NeuralNetwork) TrainBatch(trainer training.Trainer, epocs int) {
 	fmt.Printf("Start training in %v threads\n", runtime.NumCPU())
 	for t := 0; t < epocs; t++ {
 		batchWorkers := nn.runBatchWorkers(runtime.NumCPU(), trainer)
-
+		nn.syncMutex.Lock()
 		for l := 1; l < nn.LayerCount; l++ {
 			bGradient, ok := nn.BGradient[l].(BatchGradientDescent)
 			if !ok {
@@ -265,6 +270,7 @@ func (nn *NeuralNetwork) TrainBatch(trainer training.Trainer, epocs int) {
 				nn.watcher.UpdateWeights(l, nn.Weights[l])
 			}
 		}
+		nn.syncMutex.Unlock()
 		//TODO: remove this is not used for visualization
 		time.Sleep(100 * time.Millisecond)
 	}
@@ -327,9 +333,9 @@ func (nn *NeuralNetwork) LoadState(reader io.Reader) {
 	sizeBuffer := readByteArray(reader, nn.LayerCount*4)
 	nn.Sizes = make([]int, nn.LayerCount)
 
-	for i := 0; i < nn.LayerCount; i++ {
-		nn.Sizes[i] = int(binary.LittleEndian.Uint32(sizeBuffer[i*4:]))
-		fmt.Printf("LoadState: nn.Sizes[%d] %d \n", i, nn.Sizes[i])
+	for l := 0; l < nn.LayerCount; l++ {
+		nn.Sizes[l] = int(binary.LittleEndian.Uint32(sizeBuffer[l*4:]))
+		fmt.Printf("LoadState: nn.Sizes[%d] %d \n", l, nn.Sizes[l])
 	}
 
 	nn.Weights = []*mat.Dense{&mat.Dense{}}
@@ -337,16 +343,22 @@ func (nn *NeuralNetwork) LoadState(reader io.Reader) {
 
 	// read Biases
 	nn.Biases[0] = &mat.Dense{}
-	for i := 1; i < nn.LayerCount; i++ {
+	for l := 1; l < nn.LayerCount; l++ {
 		nn.Biases = append(nn.Biases, &mat.Dense{})
-		nn.Biases[i] = readDense(reader, nn.Biases[i])
+		nn.Biases[l] = readDense(reader, nn.Biases[l])
 	}
 
-	// read Weights
+	nn.BGradient = make([]interface{}, nn.LayerCount)
+	nn.WGradient = make([]interface{}, nn.LayerCount)
+	// read Weights and initialize gradient descents
 	nn.Weights[0] = &mat.Dense{}
-	for i := 1; i < nn.LayerCount; i++ {
+	for l := 1; l < nn.LayerCount; l++ {
 		nn.Weights = append(nn.Weights, &mat.Dense{})
-		nn.Weights[i] = readDense(reader, nn.Weights[i])
+		nn.Weights[l] = readDense(reader, nn.Weights[l])
+		if nn.gradientDescentInitializer != nil {
+			nn.BGradient[l] = nn.gradientDescentInitializer(nn, l, BiasGradient)
+			nn.WGradient[l] = nn.gradientDescentInitializer(nn, l, WeightGradient)
+		}
 	}
 
 	// fmt.Printf("\nLoadState end\n")