Reorg

Author: FabianSchuetze

Diff for: CMakeLists.txt

@@ -58,6 +58,7 @@ set(CMAKE_CXX_FLAGS_RELEASE "-g -O3")
 add_subdirectory(test)
 add_subdirectory(examples/cifar)
 add_subdirectory(examples/rnn)
+add_subdirectory(examples/mnist)
 
 #add_subdirectory(third_party/pybind11)
 #pybind11_add_module(NeuralNetwork python/bindings.cpp)

Diff for: README.md

@@ -19,7 +19,10 @@ Strenghts
 
 Examples
 --------
-####A deep neural network for MNIST
+The repo contains examples for deep neural networks, CNNs and RNNs. The
+examples can be found in the folder `examples`
+
+#### A deep neural network for MNIST
 ----------------------------------------
 The standard MNIST example is developed in [test/mnist.cpp](test/mnist.cpp). Running this file, prints the following information to `stdout`
 ```shell
@@ -122,3 +125,4 @@ of the results is checked with dedicated tests. Similary, I test that the GPU
 implementation is never slower than the CPU version.
 
 
+

Diff for: examples/mnist/CMakeLists.txt

@@ -0,0 +1,12 @@
+set(${LIBRARY_TARGET_NAME}_TEST
+    mnist_lenet.cpp
+    mnist.cpp
+)
+foreach( testsourcefile ${${LIBRARY_TARGET_NAME}_TEST} )
+    #string( REPLACE "test/cifar/" "" testname ${testsourcefile})
+    string( REPLACE ".cpp" "" testname ${testsourcefile} )
+    add_executable( ${testname} ${testsourcefile} )
+    target_link_libraries(${testname} GPU_NN ${CUDA_LIBRARIES})
+    target_link_libraries(${testname} GPU_NN cuda_obj)
+    add_test(${testname} ${testname})
+endforeach( testsourcefile ${${LIBRARY_TARGET_NAME}_TEST})

Diff for: test/mnist.cpp renamed to examples/mnist/mnist.cpp

@@ -1,9 +1,9 @@
 #include <stdlib.h>
 //#include <type_traits>
 #include <memory>
-#include "../include/neural_network.h"
-#include "../include/metrics/missclassified.hpp"
-#include "../third_party/mnist/include/mnist/get_data.h"
+#include "../../include/neural_network.h"
+#include "../../include/metrics/missclassified.hpp"
+#include "../../third_party/mnist/include/mnist/get_data.h"
 
 typedef std::shared_ptr<Layer> s_Layer;
 using std::make_shared;

Diff for: test/mnist_lenet.cpp renamed to examples/mnist/mnist_lenet.cpp

@@ -1,7 +1,8 @@
 #include <stdlib.h>
 //#include <type_traits>
-#include "../include/neural_network.h"
-#include "../third_party/mnist/include/mnist/get_data.h"
+#include "../../include/neural_network.h"
+#include "../../third_party/mnist/include/mnist/get_data.h"
+#include "../../include/metrics/missclassified.hpp"
 
 typedef std::shared_ptr<Layer> s_Layer;
 using std::make_shared;
@@ -26,6 +27,9 @@ int main(int argc, char** argv) {
     std::shared_ptr<GradientDescent> sgd =
         std::make_shared<Momentum>(LearningRate(0.001), MomentumRate(0.90),
                                     WeightDecay(0.004));
+    std::vector<Metric*> metrics;
+    Metric* val = new Missclassified(&n1);
+    metrics.push_back(val);
     n1.train(data.get_x_train(), data.get_y_train(), sgd, Epochs(30),
-             Patience(10), BatchSize(32));
+             Patience(10), BatchSize(32), metrics);
 }

Diff for: src/initalization/glorot.cpp

@@ -1,8 +1,11 @@
 #include "../../include/initalization/glorot.hpp"
+#include <random>
 
 Matrix Glorot::weights(int rows, int cols) const {
-    srand((unsigned int)0);
-    Matrix mat = Matrix::Random(rows, cols);
+    std::mt19937 gen;
+    gen.seed(0);
+    std::uniform_real_distribution<dtype> dis(-1.0, 1.0);
+    Matrix mat = Matrix::NullaryExpr(rows, cols, [&]() { return dis(gen); });
     dtype glorot_scale = std::sqrt(6.) / std::sqrt(rows + cols);
     mat *= glorot_scale;
     return mat;

Diff for: src/layer/convolution.cpp

@@ -103,15 +103,16 @@ void Convolution::initialize_grad() {
 }
 
 void Convolution::initialize_weight(Init* init) {
+    // this is the same init as keras uses for conv, overwrite init, not good
     int receptive_field = _kernel.first() * _kernel.second();
     int fan_in = receptive_field * _channels.get();
     int fan_out = receptive_field * _filters.get();
     dtype glorot_scale = std::sqrt(6.) / std::sqrt(fan_in + fan_out);
     int cols = _filters.get();
     int rows = _channels.get() * _kernel.first() * _kernel.second();
+    srand((unsigned int)0);
     Matrix weights = Matrix::Random(rows, cols);
     weights *= glorot_scale;
-    // Matrix weights = init->weights(rows, cols);
     parameters.push_back(std::make_shared<Storage>(weights));
 }
 

Diff for: test/CMakeLists.txt

@@ -1,12 +1,9 @@
 
 set(${LIBRARY_TARGET_NAME}_TEST
     #normalization.cpp
-    #mnist_lenet.cpp
-    #mnist.cpp
     lstm.cpp
     inits.cpp
-    #im2col_layer.cpp
-    #convolution.cpp
+    convolution.cpp
     softmax.cpp
     dense.cpp
     cross_entropy.cpp

Diff for: test/convolution.cpp

@@ -52,7 +52,6 @@ TEST_CASE("NeuralNetwork cpu", "[cpu]") {
     // l2);
     Matrix _input = Matrix::Random(
         image.first() * image.second() * channels.get(), batches);
-    print_Matrix_to_stdout(_input, "input_cpu");
     std::shared_ptr<Storage> input = std::make_shared<Storage>(_input);
     Matrix _out =
         Matrix::Zero(out_height * out_width, channels.get() * kernel.first() *
@@ -63,7 +62,6 @@ TEST_CASE("NeuralNetwork cpu", "[cpu]") {
     std::shared_ptr<Storage> conv_out = std::make_shared<Storage>(_out2);
     inp1->forward_cpu(input, output_cpu, "train");
     l2->forward_cpu(output_cpu, conv_out, "train");
-    print_Matrix_to_stdout(output_cpu->return_data_const(), "cpu");
 }
 
 TEST_CASE("NeuralNetwork gpu", "[gpu]") {
@@ -86,7 +84,6 @@ TEST_CASE("NeuralNetwork gpu", "[gpu]") {
         make_shared<Im2ColLayer>(std::dynamic_pointer_cast<Convolution>(l2));
     Matrix _input = Matrix::Random(
         image.first() * image.second() * channels.get(), batches);
-    print_Matrix_to_stdout(_input, "input_gpu");
     std::shared_ptr<Storage> input = std::make_shared<Storage>(_input);
     Matrix _out =
         Matrix::Zero(out_height * out_width, channels.get() * kernel.first() *
@@ -97,7 +94,6 @@ TEST_CASE("NeuralNetwork gpu", "[gpu]") {
     std::shared_ptr<Storage> conv_out = std::make_shared<Storage>(_out2);
     inp1->forward_gpu(input, output_cpu, "train");
     l2->forward_gpu(output_cpu, conv_out, "train");
-    print_Matrix_to_stdout(output_cpu->return_data_const(), "cpu");
 }
 
 TEST_CASE("NeuralNetwork forward equivalence", "[forward equivalance]") {
@@ -118,6 +114,7 @@ TEST_CASE("NeuralNetwork forward equivalence", "[forward equivalance]") {
                                                 image, channels, init);
     s_Layer im2col_cpu = make_shared<Im2ColLayer>(
         std::dynamic_pointer_cast<Convolution>(conv_cpu));
+    //Init* init_gpu = new Glorot();
     s_Layer conv_gpu = make_shared<Convolution>(kernel, pad, stride, filters,
                                                 image, channels, init);
     s_Layer im2col_gpu = make_shared<Im2ColLayer>(