NeuralNetwork/neuralnetwork/neuralnetworkbase/backpropgradient.go

/*
 * MIT License
 *
 * Copyright (c) 2019 Alexey Edelev <semlanik@gmail.com>
 *
 * This file is part of NeuralNetwork project https://git.semlanik.org/semlanik/NeuralNetwork
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this
 * software and associated documentation files (the "Software"), to deal in the Software
 * without restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software, and
 * to permit persons to whom the Software is furnished to do so, subject to the following
 * conditions:
 *
 * The above copyright notice and this permission notice shall be included in all copies
 * or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
 * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
 * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
 * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

package neuralnetworkbase

import (
	mat "gonum.org/v1/gonum/mat"
)

// Simple backpropagation with constant value η
type backPropGradient struct {
	alpha float64
}

func NewBackPropInitializer(nu float64) GradientDescentInitializer {
	return func(nn *NeuralNetwork, layer, gradientType int) interface{} {
		return newBackPropGradient(nu / float64(nn.Sizes[0]))
	}
}

func newBackPropGradient(a float64) (g *backPropGradient) {
	g = &backPropGradient{alpha: a}
	return
}

func (g *backPropGradient) ApplyDelta(m mat.Matrix, gradient mat.Matrix) (result *mat.Dense) {
	// Gradient change of actual matrix using:
	// m[l]′ = m[l] − η * ∂C/∂m
	// Where ∂E/∂m is `in` matrix
	scaled := &mat.Dense{}
	result = &mat.Dense{}

	// η * ∂E/∂m
	scaled.Scale(g.alpha, gradient)
	// m[l] − η * ∂E/∂m
	result.Sub(m, scaled)
	return result
}