|
@@ -148,19 +148,19 @@ func (nn *NeuralNetwork) forward(aIn mat.Matrix) {
|
|
|
aSrc := nn.A[i-1]
|
|
|
aDst := nn.A[i]
|
|
|
|
|
|
- //Each iteration implements formula bellow for neuron activation values
|
|
|
- //A[l]=σ(W[l]*A[l−1]+B[l])
|
|
|
+ // Each iteration implements formula bellow for neuron activation values
|
|
|
+ // A[l]=σ(W[l]*A[l−1]+B[l])
|
|
|
|
|
|
- //W[l]*A[l−1]
|
|
|
+ // W[l]*A[l−1]
|
|
|
aDst.Mul(nn.Weights[i], aSrc)
|
|
|
|
|
|
- //W[l]*A[l−1]+B[l]
|
|
|
+ // W[l]*A[l−1]+B[l]
|
|
|
aDst.Add(aDst, nn.Biases[i])
|
|
|
|
|
|
- //Save raw activation value for back propagation
|
|
|
+ // Save raw activation value for back propagation
|
|
|
nn.Z[i] = mat.DenseCopyOf(aDst)
|
|
|
|
|
|
- //σ(W[l]*A[l−1]+B[l])
|
|
|
+ // σ(W[l]*A[l−1]+B[l])
|
|
|
aDst.Apply(applySigmoid, aDst)
|
|
|
}
|
|
|
}
|
|
@@ -170,75 +170,75 @@ func (nn *NeuralNetwork) backward(aIn, aOut mat.Matrix) {
|
|
|
|
|
|
lastLayerNum := nn.Count - 1
|
|
|
|
|
|
- //To calculate new values of weights and biases
|
|
|
- //following formulas are used:
|
|
|
- //W[l] = A[l−1]*δ[l]
|
|
|
- //B[l] = δ[l]
|
|
|
+ // To calculate new values of weights and biases
|
|
|
+ // following formulas are used:
|
|
|
+ // W[l] = A[l−1]*δ[l]
|
|
|
+ // B[l] = δ[l]
|
|
|
|
|
|
- //For last layer δ value is calculated by following:
|
|
|
- //δ = (A[L]−y)⊙σ'(Z[L])
|
|
|
+ // For last layer δ value is calculated by following:
|
|
|
+ // δ = (A[L]−y)⊙σ'(Z[L])
|
|
|
|
|
|
- //Calculate initial error for last layer L
|
|
|
- //error = A[L]-y
|
|
|
- //Where y is expected activations set
|
|
|
+ // Calculate initial error for last layer L
|
|
|
+ // error = A[L]-y
|
|
|
+ // Where y is expected activations set
|
|
|
err := &mat.Dense{}
|
|
|
err.Sub(nn.result(), aOut)
|
|
|
|
|
|
- //Calculate sigmoids prime σ'(Z[L]) for last layer L
|
|
|
+ // Calculate sigmoids prime σ'(Z[L]) for last layer L
|
|
|
sigmoidsPrime := &mat.Dense{}
|
|
|
sigmoidsPrime.Apply(applySigmoidPrime, nn.Z[lastLayerNum])
|
|
|
|
|
|
- //(A[L]−y)⊙σ'(Z[L])
|
|
|
+ // (A[L]−y)⊙σ'(Z[L])
|
|
|
delta := &mat.Dense{}
|
|
|
delta.MulElem(err, sigmoidsPrime)
|
|
|
|
|
|
- //B[L] = δ[L]
|
|
|
+ // B[L] = δ[L]
|
|
|
biases := mat.DenseCopyOf(delta)
|
|
|
|
|
|
- //W[L] = A[L−1]*δ[L]
|
|
|
+ // W[L] = A[L−1]*δ[L]
|
|
|
weights := &mat.Dense{}
|
|
|
weights.Mul(delta, nn.A[lastLayerNum-1].T())
|
|
|
|
|
|
- //Initialize new weights and biases values with last layer values
|
|
|
- newBiases := []*mat.Dense{makeBackGradien(biases, nn.Biases[lastLayerNum], nn.alpha)}
|
|
|
- newWeights := []*mat.Dense{makeBackGradien(weights, nn.Weights[lastLayerNum], nn.alpha)}
|
|
|
+ // Initialize new weights and biases values with last layer values
|
|
|
+ newBiases := []*mat.Dense{makeBackGradient(biases, nn.Biases[lastLayerNum], nn.alpha)}
|
|
|
+ newWeights := []*mat.Dense{makeBackGradient(weights, nn.Weights[lastLayerNum], nn.alpha)}
|
|
|
|
|
|
- //Save calculated delta value temporary error variable
|
|
|
+ // Save calculated delta value temporary error variable
|
|
|
err = delta
|
|
|
|
|
|
- //Next layer Weights and Biases are calculated using same formulas:
|
|
|
- //W[l] = A[l−1]*δ[l]
|
|
|
- //B[l] = δ[l]
|
|
|
+ // Next layer Weights and Biases are calculated using same formulas:
|
|
|
+ // W[l] = A[l−1]*δ[l]
|
|
|
+ // B[l] = δ[l]
|
|
|
|
|
|
- //But δ[l] is calculated using different formula:
|
|
|
- //δ[l] = ((Wt[l+1])*δ[l+1])⊙σ'(Z[l])
|
|
|
- //Where Wt[l+1] is transponded matrix of actual Weights from
|
|
|
- //forward step
|
|
|
+ // But δ[l] is calculated using different formula:
|
|
|
+ // δ[l] = ((Wt[l+1])*δ[l+1])⊙σ'(Z[l])
|
|
|
+ // Where Wt[l+1] is transposed matrix of actual Weights from
|
|
|
+ // forward step
|
|
|
for l := nn.Count - 2; l > 0; l-- {
|
|
|
- //Calculate sigmoids prime σ'(Z[l]) for last layer l
|
|
|
+ // Calculate sigmoids prime σ'(Z[l]) for last layer l
|
|
|
sigmoidsPrime := &mat.Dense{}
|
|
|
sigmoidsPrime.Apply(applySigmoidPrime, nn.Z[l])
|
|
|
|
|
|
- //(Wt[l+1])*δ[l+1]
|
|
|
- //err bellow is delta from previous step(l+1)
|
|
|
+ // (Wt[l+1])*δ[l+1]
|
|
|
+ // err bellow is delta from previous step(l+1)
|
|
|
delta := &mat.Dense{}
|
|
|
wdelta := &mat.Dense{}
|
|
|
wdelta.Mul(nn.Weights[l+1].T(), err)
|
|
|
|
|
|
- //Calculate new delta and store it to temporary variable err
|
|
|
- //δ[l] = ((Wt[l+1])*δ[l+1])⊙σ'(Z[l])
|
|
|
+ // Calculate new delta and store it to temporary variable err
|
|
|
+ // δ[l] = ((Wt[l+1])*δ[l+1])⊙σ'(Z[l])
|
|
|
delta.MulElem(wdelta, sigmoidsPrime)
|
|
|
err = delta
|
|
|
|
|
|
- //B[l] = δ[l]
|
|
|
+ // B[l] = δ[l]
|
|
|
biases := mat.DenseCopyOf(delta)
|
|
|
|
|
|
- //W[l] = A[l−1]*δ[l]
|
|
|
- //At this point it's required to give explanation for inaccuracy
|
|
|
- //in the formula
|
|
|
+ // W[l] = A[l−1]*δ[l]
|
|
|
+ // At this point it's required to give explanation for inaccuracy
|
|
|
+ // in the formula
|
|
|
|
|
|
- //Multiplying of activations matrix for layer l-1 and δ[l] is imposible
|
|
|
- //because view of matrices are following:
|
|
|
+ // Multiplying of activations matrix for layer l-1 and δ[l] is imposible
|
|
|
+ // because view of matrices are following:
|
|
|
// A[l-1] δ[l]
|
|
|
// ⎡A[0] ⎤ ⎡δ[0] ⎤
|
|
|
// ⎢A[1] ⎥ ⎢δ[1] ⎥
|
|
@@ -246,16 +246,16 @@ func (nn *NeuralNetwork) backward(aIn, aOut mat.Matrix) {
|
|
|
// ⎢A[i] ⎥ X ⎢δ[i] ⎥
|
|
|
// ⎢ ... ⎥ ⎢ ... ⎥
|
|
|
// ⎣A[s'] ⎦ ⎣δ[s] ⎦
|
|
|
- //So we need to modify these matrices to apply mutiplications and got Weights matrix
|
|
|
- //of following view:
|
|
|
+ // So we need to modify these matrices to apply mutiplications and got
|
|
|
+ // Weights matrix of following view:
|
|
|
// ⎡w[0,0] ... w[0,j] ... w[0,s']⎤
|
|
|
// ⎢w[1,0] ... w[1,j] ... w[1,s']⎥
|
|
|
// ⎢ ... ⎥
|
|
|
// ⎢w[i,0] ... w[i,j] ... w[i,s']⎥
|
|
|
// ⎢ ... ⎥
|
|
|
// ⎣w[s,0] ... w[s,j] ... w[s,s']⎦
|
|
|
- //So we substitude matrices and transposes A[l-1] to get valid multiplication
|
|
|
- //if following view:
|
|
|
+ // So we swap matrices and transpose A[l-1] to get valid multiplication
|
|
|
+ // of following view:
|
|
|
// δ[l] A[l-1]
|
|
|
// ⎡δ[0] ⎤ x [A[0] A[1] ... A[i] ... A[s']]
|
|
|
// ⎢δ[1] ⎥
|
|
@@ -266,10 +266,9 @@ func (nn *NeuralNetwork) backward(aIn, aOut mat.Matrix) {
|
|
|
weights := &mat.Dense{}
|
|
|
weights.Mul(delta, nn.A[l-1].T())
|
|
|
|
|
|
- //!Prepend! new Biases and Weights
|
|
|
- // Scale down
|
|
|
- newBiases = append([]*mat.Dense{makeBackGradien(biases, nn.Biases[l], nn.alpha)}, newBiases...)
|
|
|
- newWeights = append([]*mat.Dense{makeBackGradien(weights, nn.Weights[l], nn.alpha)}, newWeights...)
|
|
|
+ // !Prepend! new Biases and Weights
|
|
|
+ newBiases = append([]*mat.Dense{makeBackGradient(biases, nn.Biases[l], nn.alpha)}, newBiases...)
|
|
|
+ newWeights = append([]*mat.Dense{makeBackGradient(weights, nn.Weights[l], nn.alpha)}, newWeights...)
|
|
|
}
|
|
|
|
|
|
newBiases = append([]*mat.Dense{&mat.Dense{}}, newBiases...)
|