NeuralNetwork/neuralnetwork/neuralnetworkbase/gradients.go

/*
 * MIT License
 *
 * Copyright (c) 2019 Alexey Edelev <semlanik@gmail.com>
 *
 * This file is part of NeuralNetwork project https://git.semlanik.org/semlanik/NeuralNetwork
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this
 * software and associated documentation files (the "Software"), to deal in the Software
 * without restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software, and
 * to permit persons to whom the Software is furnished to do so, subject to the following
 * conditions:
 *
 * The above copyright notice and this permission notice shall be included in all copies
 * or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
 * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
 * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
 * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

package neuralnetworkbase

import (
	"math"

	mat "gonum.org/v1/gonum/mat"
)

type RPropGradient struct {
	GradientsPrev *mat.Dense
	Gradients     *mat.Dense
	Deltas        *mat.Dense
	batchSize     int
}

func NewRPropGradient(r, c int) (g *RPropGradient) {
	g = &RPropGradient{}

	deltas := make([]float64, r*c)

	for j, _ := range deltas {
		deltas[j] = 0.1
	}

	g.Gradients = mat.NewDense(r, c, nil)
	g.GradientsPrev = mat.NewDense(r, c, nil)
	g.Deltas = mat.NewDense(r, c, deltas)
	return
}

func (g *RPropGradient) ApplyDelta(m mat.Matrix, _ mat.Matrix) (result *mat.Dense) {
	//TODO: move this hardcoded parameters to separate config for gradient
	nuPlus := 1.2
	nuMinus := 0.5

	deltaMax := 50.0
	deltaMin := 0.000001

	result = &mat.Dense{}

	gradient := g.Gradients
	r, c := gradient.Dims()
	dividers := make([]float64, r*c)
	for i := range dividers {
		dividers[i] = float64(g.batchSize)
	}
	gradientDivider := mat.NewDense(r, c, dividers)
	gradient.DivElem(gradient, gradientDivider)

	result.Apply(func(i, j int, v float64) (outV float64) {
		gradientSign := g.GradientsPrev.At(i, j) * gradient.At(i, j)
		if gradientSign > 0 {
			g.Deltas.Set(i, j, math.Min(nuPlus*g.Deltas.At(i, j), deltaMax))
			outV = v - sign(gradient.At(i, j))*g.Deltas.At(i, j)

			g.GradientsPrev.Set(i, j, gradient.At(i, j))
		} else if gradientSign < 0 {
			outV = v + sign(g.GradientsPrev.At(i, j))*g.Deltas.At(i, j)
			g.Deltas.Set(i, j, math.Max(nuMinus*g.Deltas.At(i, j), deltaMin))

			g.GradientsPrev.Set(i, j, 0.0)
		} else {
			outV = v - sign(gradient.At(i, j))*g.Deltas.At(i, j)

			g.GradientsPrev.Set(i, j, gradient.At(i, j))
		}
		return
	}, m)

	g.batchSize = 0
	return result
}

func (g *RPropGradient) AccumGradients(gradient mat.Matrix) {
	g.Gradients.Apply(func(i, j int, v float64) float64 {
		v += gradient.At(i, j)
		return v
	}, g.Gradients)
	g.batchSize++
}

//Simple backpropagation with constant value η
type BackPropGradient struct {
	alpha float64
}

func (g *BackPropGradient) ApplyDelta(m mat.Matrix, gradient mat.Matrix) (result *mat.Dense) {
	// Gradient change of actual matrix using:
	// m[l]′ = m[l] − η * ∂C/∂m
	// Where ∂E/∂m is `in` matrix
	scaled := &mat.Dense{}
	result = &mat.Dense{}

	// η * ∂E/∂m
	scaled.Scale(g.alpha, gradient)
	// m[l] − η * ∂E/∂m
	result.Sub(m, scaled)
	return result
}

func (g *BackPropGradient) AccumGradients(gradient mat.Matrix) {

}