Complete UI part of handwriting

handwriting/handwriting.proto

@@ -28,12 +28,20 @@ syntax="proto3";
 package handwriting;
 
 message Matrix {
-    bytes matrix = 1;
+    repeated double data = 1;
 }
 
 message Result {
     uint32 resultCharacter = 1;
 }
-service SnakeSimulator {
+
+message NeuralNetworkRaw {
+    bytes data = 1;
+}
+
+message None {}
+
+service Handwriting {
     rpc recognize(Matrix) returns (Result) {}
+    rpc setNeuralNetworkData(NeuralNetworkRaw) returns (None) {}
 }

handwriting/handwritingui/CMakeLists.txt

@@ -19,5 +19,5 @@ generate_qtprotobuf(TARGET HandwritingUi PROTO_FILES ${PROTO_FILES} EXCLUDE_HEAD
 set(CMAKE_AUTOMOC ON)
 set(CMAKE_AUTORCC ON)
 
-add_executable(HandwritingUi main.cpp qml.qrc)
+add_executable(HandwritingUi main.cpp qml.qrc handwritingengine.cpp)
 target_link_libraries(HandwritingUi Qt5::Core Qt5::Gui Qt5::Qml Qt5::Quick QtProtobufProject::QtProtobuf QtProtobufProject::QtGrpc ${QtProtobuf_GENERATED})

handwriting/handwritingui/handwritingengine.cpp

@@ -0,0 +1,79 @@
+#include "handwritingengine.h"
+
+#include <QFile>
+#include <QDebug>
+
+#include <QtProtobufTypes>
+#include <QGrpcHttp2Channel>
+#include <InsecureCredentials>
+
+#include "handwritingclient.h"
+#include "matrix.h"
+#include "neuralnetworkraw.h"
+#include "none.h"
+#include "result.h"
+
+class NoneCredencials : public QtProtobuf::CallCredentials
+{
+public:
+    NoneCredencials() : QtProtobuf::CallCredentials(QtProtobuf::AbstractCredentials::CredentialMap{}) {}
+};
+
+static const QtProtobuf::DoubleList emptyData{0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                              0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,
+                                             };
+
+HandwritingEngine::HandwritingEngine(QObject *parent) : QObject(parent)
+  , m_client(new handwriting::HandwritingClient)
+{
+
+    auto chan = std::shared_ptr<QtProtobuf::QGrpcHttp2Channel>(new QtProtobuf::QGrpcHttp2Channel(QUrl("http://localhost:65002"), QtProtobuf::InsecureCredentials()|NoneCredencials()));
+    m_client->attachChannel(chan);
+
+    m_matrix.setData(emptyData);
+}
+
+
+void HandwritingEngine::recognize()
+{
+    m_client->recognize(m_matrix, this, [](QtProtobuf::QGrpcAsyncReply *reply) {
+        qDebug() << "Recognition result" << reply->read<handwriting::Result>().resultCharacter();
+    });
+}
+
+void HandwritingEngine::setNeuralNetworkData(const QString &networkDataPath)
+{
+    QFile dataFile(QUrl(networkDataPath).toLocalFile());
+    if (!dataFile.open(QFile::ReadOnly)) {
+        qCritical() << "Could not open" << QUrl(networkDataPath).toLocalFile();
+        return;
+    }
+
+    m_client->setNeuralNetworkData({dataFile.readAll()});
+}

handwriting/handwritingui/handwritingengine.h

@@ -0,0 +1,58 @@
+#ifndef HANDWRITINGENGINE_H
+#define HANDWRITINGENGINE_H
+
+#include <QObject>
+#include <QUrl>
+#include <QByteArray>
+#include <QtProtobufTypes>
+
+#include "matrix.h"
+
+#include <memory>
+
+namespace handwriting {
+    class HandwritingClient;
+}
+class HandwritingEngine final : public QObject
+{
+    Q_OBJECT
+    Q_PROPERTY(int result READ result WRITE setResult NOTIFY resultChanged)
+    Q_PROPERTY(handwriting::Matrix *matrix READ matrix CONSTANT)
+public:
+    explicit HandwritingEngine(QObject *parent = nullptr);
+    virtual ~HandwritingEngine() = default;
+
+    Q_INVOKABLE void recognize();
+    Q_INVOKABLE void setNeuralNetworkData(const QString &networkDataPath);
+
+    int result() const
+    {
+        return m_result;
+    }
+
+    handwriting::Matrix *matrix()
+    {
+        return &m_matrix;
+    }
+
+public slots:
+    void setResult(int result)
+    {
+        if (m_result == result)
+            return;
+
+        m_result = result;
+        emit resultChanged(m_result);
+    }
+
+signals:
+    void resultChanged(int result);
+
+private:
+    std::shared_ptr<handwriting::HandwritingClient> m_client;
+    int m_result;
+    QtProtobuf::DoubleList m_data;
+    handwriting::Matrix m_matrix;
+};
+
+#endif // HANDWRITINGENGINE_H

handwriting/handwritingui/main.cpp

@@ -27,16 +27,29 @@
 #include <QQmlApplicationEngine>
 #include <QQmlContext>
 #include <QDebug>
+
 #include <QtProtobufTypes>
 
-#include "qgrpchttp2channel.h"
-#include "insecurecredentials.h"
+#include "handwritingclient.h"
+#include "matrix.h"
+#include "neuralnetworkraw.h"
+#include "none.h"
+#include "result.h"
+
+#include "handwritingengine.h"
 
 int main(int argc, char *argv[])
 {
+    qRegisterProtobufType<handwriting::Matrix>();
+    qRegisterProtobufType<handwriting::None>();
+    qRegisterProtobufType<handwriting::Result>();
+    qRegisterProtobufType<handwriting::NeuralNetworkRaw>();
+
     QGuiApplication app(argc, argv);
+    HandwritingEngine hwengine;
     QtProtobuf::registerProtoTypes();
     QQmlApplicationEngine engine;
+    engine.rootContext()->setContextProperty("hwengine", &hwengine);
     engine.load(QUrl(QStringLiteral("qrc:/main.qml")));
     if (engine.rootObjects().isEmpty())
         return -1;

handwriting/handwritingui/main.qml

@@ -26,61 +26,114 @@
 import QtQuick 2.11
 import QtQuick.Window 2.11
 import QtQuick.Controls 1.4
+import QtQuick.Dialogs 1.2
 
 ApplicationWindow {
     id: root
     visible: true
-    width: 1024
-    height: 768
+    width: 1120
+    height: 560
+    maximumWidth: 1120
+    maximumHeight: 560
+    minimumWidth: 1120
+    minimumHeight: 560
+
+    property int tileSize: 20
+    property int tileCount: 28
+    property color resetColor: "#333333"
+
     MouseArea {
         id: drawingArea
-        width: 28*20
-        height: 28*20
-        onPositionChanged: {
-            root.contentItem.child
-            mouse.x
-            mouse.y
-        }
-    }
-    Repeater {
-        model: 784
-        Rectangle {
-            id: tile
-            width: 20
-            height: 20
-            color: "black"
-            x: 20*Math.floor(model.index/28)
-            y: 20*(model.index%28)
-            border.width: 1
-            border.color: "white"
-            Connections {
-                target: drawingArea
-                onPositionChanged: {
-                    var centerX = tile.x + tile.width/2
-                    var centerY = tile.y + tile.height/2
-                    var diffX = mouse.x - centerX
-                    var diffY = mouse.y - centerY
-                    var dense = Math.sqrt(diffX * diffX + diffY * diffY)
-                    if(dense < 20) {
-                        tile.color = Qt.rgba(color.r + (20-dense)/20, color.g + (20-dense)/20, color.b + (20-dense)/20, 1.0)
+        width: tileCount*tileSize
+        height: tileCount*tileSize
+        Repeater {
+            model: 784
+            Rectangle {
+                id: tile
+                width: tileSize
+                height: tileSize
+                color: resetColor
+                x: tileSize*Math.floor(model.index/tileCount)
+                y: tileSize*(model.index%tileCount)
+                Connections {
+                    target: drawingArea
+                    onPositionChanged: {
+                        var centerX = tile.x + tile.width/2
+                        var centerY = tile.y + tile.height/2
+                        var diffX = mouse.x - centerX
+                        var diffY = mouse.y - centerY
+                        var dense = Math.sqrt(diffX * diffX + diffY * diffY)
+                        if (dense < tileSize) {
+                            var newColor = color.r + (tileSize - dense)/tileSize
+                            var newValue = newColor - 0.2
+                            if (newColor > 1.0) {
+                                newColor = 1.0
+                                newValue = 1.0
+                            }
+
+                            tile.color = Qt.rgba(newColor, newColor, newColor, 1.0)
+                            hwengine.matrix[model.index] = newColor
+                        }
                     }
                 }
-            }
 
-            Connections {
-                target: clearButton
-                onClicked: {
-                    tile.color = "black"
+                Connections {
+                    target: clearButton
+                    onClicked: {
+                        tile.color = resetColor
+                        hwengine.matrix[model.index] = 0.0
+                    }
                 }
             }
         }
     }
 
     Column {
-        anchors.right: parent.right
+        anchors.left: drawingArea.right
+        anchors.top: parent.top
+        anchors.margins: 10
+        spacing: 10
+
+        Text {
+            text: "Select neural network data"
+        }
+
+        Row {
+            TextField {
+                id: pathToNeuralNetwork
+                enabled: false
+                width: 300
+            }
+            Button {
+                id: browseButton
+                text: "Browse"
+                onClicked: {
+                    neuralNetworkImport.open()
+                }
+            }
+
+            Button {
+                id: uploadButton
+                text: "Upload"
+            }
+        }
         Button {
             id: clearButton
             text: "Clear"
         }
+        Button {
+            id: recognizeButton
+            text: "Recognize"
+        }
+    }
+
+    FileDialog {
+        id: neuralNetworkImport
+        selectMultiple: false
+        selectFolder: false
+        nameFilters: ["*.nnd"]
+        onAccepted: {
+            pathToNeuralNetwork.text = neuralNetworkImport.fileUrl
+        }
     }
 }