NeuralNetwork/neuralnetwork/teach/mnistreader.go

/*
 * MIT License
 *
 * Copyright (c) 2019 Alexey Edelev <semlanik@gmail.com>
 *
 * This file is part of NeuralNetwork project https://git.semlanik.org/semlanik/NeuralNetwork
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this
 * software and associated documentation files (the "Software"), to deal in the Software
 * without restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software, and
 * to permit persons to whom the Software is furnished to do so, subject to the following
 * conditions:
 *
 * The above copyright notice and this permission notice shall be included in all copies
 * or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
 * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
 * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
 * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

package teach

import (
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"os"

	mat "gonum.org/v1/gonum/mat"
)

type MNISTReader struct {
	file            *os.File
	resultsFile     *os.File
	size            int
	imageSize       int
	buffered        *mat.Dense
	resultsBuffered *mat.Dense
	window          MNISTBatchWindow
	currentIndex    int64
}

type MNISTBatchWindow struct {
	from int64
	to   int64
}

func NewMNISTReader(dataFilename string, resultsFilename string, window MNISTBatchWindow) (r *MNISTReader) {
	r = &MNISTReader{
		window: window,
	}

	var err error
	r.file, err = os.Open(dataFilename)
	if err != nil {
		return nil
	}

	r.resultsFile, err = os.Open(resultsFilename)
	if err != nil {
		return nil
	}

	buffer := make([]byte, 16)
	r.file.Read(buffer)
	header := binary.BigEndian.Uint32(buffer[:4])
	if header != 0x00000803 {
		return nil
	}
	r.size = int(binary.BigEndian.Uint32(buffer[4:8]))
	r.imageSize = int(binary.BigEndian.Uint32(buffer[8:12])) * int(binary.BigEndian.Uint32(buffer[12:16]))
	fmt.Printf("Image size: %v\n", r.imageSize)
	buffer = make([]byte, 8)
	r.resultsFile.Read(buffer)
	header = binary.BigEndian.Uint32(buffer[0:4])
	if header != 0x00000801 {
		return nil
	}
	resultsSize := int(binary.BigEndian.Uint32(buffer[4:8]))
	if resultsSize != r.size {
		return nil
	}

	r.Reset()
	return
}

func (r *MNISTReader) GetData() *mat.Dense {
	return r.buffered
}

func (r *MNISTReader) GetExpect() *mat.Dense {
	return r.resultsBuffered
}

func (r *MNISTReader) Next() bool {
	buffer := make([]byte, r.imageSize)
	_, err := r.file.Read(buffer)

	if err == io.EOF || r.currentIndex >= r.window.to {
		r.Reset()
		return false
	} else if err != nil {
		log.Fatal("File read error\n")
	}

	values := make([]float64, r.imageSize)
	for i, v := range buffer {
		values[i] = float64(v) / 255.0
	}

	r.buffered = mat.NewDense(r.imageSize, 1, values)

	// values = make([]float64, len(values))
	// for i, v := range buffer {
	// 	if v > 0 {
	// 		values[i] = 1
	// 	} else {
	// 		values[i] = 0
	// 	}
	// }

	// squareDense := mat.NewDense(28, 28, values)
	// fmt.Printf("r.buffered:\n%v\n\n", mat.Formatted(squareDense, mat.Prefix(""), mat.Excerpt(0), mat.Squeeze()))

	buffer = make([]byte, 1)
	_, err = r.resultsFile.Read(buffer)
	if err != nil {
		log.Fatal("Result file read error\n")
	}

	num := int(buffer[0])

	r.resultsBuffered = mat.NewDense(10, 1, nil)
	r.resultsBuffered.Set(num, 0, 1.0)

	// fmt.Printf("r.resultsBuffered:\n%v\n\n", mat.Formatted(r.resultsBuffered, mat.Prefix(""), mat.Excerpt(0)))
	r.currentIndex++

	return true
}

func (r *MNISTReader) Reset() {
	r.file.Seek(16+r.window.from*int64(r.imageSize), 0)
	r.resultsFile.Seek(8+r.window.from*int64(r.imageSize), 0)
}