Bug with lenet

Author: FabianSchuetze

CMakeLists.txt

@@ -40,14 +40,14 @@ if (UNIX)
 endif(UNIX)
 
 # moved the CXX flags to after the cuda_add_library call
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O3 -fPIC -lopenblas\
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fPIC -lopenblas\
     -lcudnn -lcurand -L /usr/lib/cuda/lib64 -lcudart  -lcublas\
     -L /usr/lib/x86_64-linux-gnu -lblas\
     -Wall -std=c++17 -Wno-error=unused-command-line-argument")
 set(${LIBRARY_TARGET_NAME}_TEST
     #test/normalization.cpp
     #test/cifar10.cpp
-    #test/mnist_lenet.cpp
+    test/mnist_lenet.cpp
     test/mnist.cpp
     #test/inits.cpp
     #test/im2col_layer.cpp

include/layer/im2col_layer.h

@@ -26,7 +26,6 @@ class Im2ColLayer : public Layer {
     Stride _stride;
     ImageShape _inp, _out;
     Channels _channels;
-    //std::shared_ptr<Convolution> next;
 
     void initialize_output_dimension() override;
     void check_size(const SharedStorage&, const SharedStorage&);

include/layer/layer.h

@@ -9,17 +9,17 @@ class Layer {
     friend class NeuralNetwork;
     typedef std::shared_ptr<Storage> SharedStorage;
     typedef std::vector<std::shared_ptr<Storage>> VecSharedStorage;
+    std::string _name;
     std::vector<int> _out_dim;
     std::vector<SharedStorage> parameters;
     std::vector<SharedStorage> gradients;
-    std::string _name;
     std::shared_ptr<Layer> _previous;
     virtual void initialize_output_dimension(const std::shared_ptr<Layer>&);
     virtual void initialize_output_dimension();
 
    public:
-    Layer() : _name("Template"){};
-    explicit Layer(const std::string& s): _name(s) {};
+    Layer();
+    explicit Layer(const std::string& s);
     virtual ~Layer() = default;
     virtual int input_dimension() {return 0;}
     virtual int input_dimension() const {return 0;}

include/layer/pooling.h

@@ -8,7 +8,7 @@ class Pooling : public Layer {
    public:
     //Pooling(int);
     Pooling(Window, Stride, ImageShape, Channels);
-    Pooling(Window, Stride, const std::shared_ptr<Convolution>&);
+    Pooling(Window, Stride, const std::shared_ptr<Layer>&);
     virtual ~Pooling() = default;
     void forward_gpu(const std::shared_ptr<Storage>&,
                      std::shared_ptr<Storage>&, const std::string&) override;
@@ -32,6 +32,6 @@ class Pooling : public Layer {
     void initialize_masking();
     void inline check_input_size(const SharedStorage&);
     void initialize_output_dimension() override;
-    //void initialize_previous(Layer*);
+    void initialize_from_previous(const std::shared_ptr<Layer>&);
 };
 #endif

include/network.h

@@ -95,5 +95,6 @@ class NeuralNetwork {
                        std::mt19937&);
     void prepare_subset(const std::vector<int>&, std::vector<int>&, int&,
                         const int&);
+    int check_input_dimension(const std::vector<int>&);
 };
 #endif

src/layer/convolution.cpp

@@ -4,6 +4,7 @@
 //#include "/usr/lib/x86_64-linux-gnu/cblas_atlas.h>
 // c
 #include <iostream>
+#include <iterator>
 #include <memory>
 #include <random>
 #include <stdexcept>
@@ -23,7 +24,7 @@ Convolution::Convolution(FilterShape filtershape, Pad pad, Stride stride,
       _channels(channels) {
     cublasStatus_t stat = cublasCreate(&_handle);
     CHECK_CUBLAS(stat);
-    //initialize_output_dimension();
+    // initialize_output_dimension();
     initialize_weight(init);
     initialize_bias();
     initialize_grad();
@@ -42,8 +43,8 @@ Convolution::Convolution(FilterShape filtershape, Pad pad, Stride stride,
       _channels(0) {
     cublasStatus_t stat = cublasCreate(&_handle);
     CHECK_CUBLAS(stat);
-    initialize_output_dimension();
     initialize_input_dimension(previous);
+    initialize_output_dimension();
     initialize_weight(init);
     initialize_bias();
     initialize_grad();
@@ -60,7 +61,14 @@ void Convolution::initialize_input_dimension(
         _channels = Channels(channels);
         _inp = ImageShape(height, width);
     } else {
-        throw std::invalid_argument("Cannot construct the thing");
+        std::stringstream ss;
+        ss << "Cannot construct the convolution layer as the previous layer's"
+              " output don't match. Received\n";
+        std::copy(shapes.begin(), shapes.end(),
+                  std::ostream_iterator<int>(ss, " "));
+        ss << "in:\n" << __PRETTY_FUNCTION__ << "\ncalled from " << __FILE__
+           << " at " << __LINE__;
+        throw std::invalid_argument(ss.str());
     }
 }
 
@@ -76,7 +84,7 @@ void Convolution::initialize_output_dimension() {
     int out_width =
         (_inp.second() + 2 * _pad.get() - _kernel.second()) / _stride.get() + 1;
     _out = ImageShape(out_height, out_width);
-    _out_dim.push_back(_channels.get());
+    _out_dim[0] = _channels.get();
     _out_dim.push_back(out_height);
     _out_dim.push_back(out_width);
 }

src/layer/dense.cpp

@@ -71,7 +71,7 @@ void Dense::initialize_input_dimension(const std::shared_ptr<Layer>& previous) {
     _in = Features(i);
 }
 
-void Dense::initialize_output_dimension() { _out_dim.push_back(_out.get()); }
+void Dense::initialize_output_dimension() { _out_dim[0] =_out.get(); }
 
 void Dense::forward_cpu(const SharedStorage& in, SharedStorage& out,
                         const std::string&) {

src/layer/im2col_layer.cpp

@@ -13,7 +13,7 @@ Im2ColLayer::Im2ColLayer(const std::shared_ptr<Convolution>& convolution_layer)
       _out(convolution_layer->_out),
       _channels(convolution_layer->_channels) {
     _previous = convolution_layer->_previous;
-    //convolution_layer->reset_previous(std::make_shared<Layer>(this));
+    // convolution_layer->reset_previous(std::make_shared<Layer>(this));
     initialize_output_dimension();
     ;
 }

src/layer/input.cpp

@@ -5,12 +5,12 @@ Input::Input(Features features)
       _features(features),
       _channels(0),
       _img(0, 0) {
-    _out_dim.push_back(_features.get());
+    _out_dim[0] = _features.get();
 }
 
 Input::Input(Channels channels, ImageShape img)
     : Layer("Input"), _features(0), _channels(channels), _img(img) {
-    _out_dim.push_back(_channels.get());
+    _out_dim[0] = _channels.get();
     _out_dim.push_back(_img.first());
     _out_dim.push_back(_img.second());
 };

src/layer/layer.cpp

@@ -1,12 +1,33 @@
 #include "../../include/layer/layer.h"
+
+Layer::Layer(const std::string& s)
+    : _name(s), _out_dim(1), parameters(), gradients(), _previous(NULL) {
+    _out_dim[0] = 0;
+}
+
+Layer::Layer()
+    : _name("Layer"), _out_dim(1), parameters(), gradients(), _previous(NULL) {
+    _out_dim[0] = 0;
+}
+
 typedef std::vector<std::shared_ptr<Storage>> VecSharedStorage;
 
-void Layer::forward_gpu(const SharedStorage&, SharedStorage&, const std::string&) { ; };
-void Layer::forward_cpu(const SharedStorage&, SharedStorage&, const std::string&) { ; };
+void Layer::forward_gpu(const SharedStorage&, SharedStorage&,
+                        const std::string&) {
+    ;
+};
+void Layer::forward_cpu(const SharedStorage&, SharedStorage&,
+                        const std::string&) {
+    ;
+};
 void Layer::backward_gpu(const SharedStorage&, const SharedStorage&,
-                         SharedStorage&) { ; };
+                         SharedStorage&) {
+    ;
+};
 void Layer::backward_cpu(const SharedStorage&, const SharedStorage&,
-                         SharedStorage&) { ; };
+                         SharedStorage&) {
+    ;
+};
 VecSharedStorage Layer::return_parameters() { return parameters; };
 VecSharedStorage Layer::return_gradients() { return gradients; };
 VecSharedStorage Layer::return_parameters() const { return parameters; };
@@ -21,6 +42,4 @@ void Layer::initialize_output_dimension(
     }
 }
 
-void Layer::initialize_output_dimension() {
-    _out_dim.push_back(0);
-}
+void Layer::initialize_output_dimension() { _out_dim.push_back(0); }

src/layer/pooling.cpp

@@ -1,4 +1,5 @@
 #include "../../include/layer/pooling.h"
+#include <stdexcept>
 #include <sys/time.h>
 #include <iostream>
 #include <memory>
@@ -19,20 +20,33 @@ Pooling::Pooling(Window window, Stride stride, ImageShape imageshape,
 }
 
 Pooling::Pooling(Window window, Stride stride,
-                 const std::shared_ptr<Convolution>& previous)
+                 const std::shared_ptr<Layer>& previous)
     : Layer("Pooling"),
       _window(window),
       _stride(stride),
-      _inp(previous->_inp),
-      _channels(previous->_channels),
-      _out(previous->_out),
+      _inp(0,0),
+      _channels(0),
+      _out(0, 0),
       batch_size(0) {
-    // initialize_previous();
+    initialize_from_previous(previous);
     initialize_masking();
     initialize_output_dimension();
     _previous = previous;
 }
 
+void Pooling::initialize_from_previous(const std::shared_ptr<Layer>& previous) {
+    if (previous->name() == "Convolution") {
+        std::shared_ptr<Convolution> conv =
+            std::dynamic_pointer_cast<Convolution>(previous);
+        _inp = conv->_inp;
+        _channels = conv->_channels;
+        _out = conv->_out;
+    }
+    else {
+        throw std::runtime_error("Can only convert conv\n");
+    }
+}
+
 void Pooling::initialize_output_dimension() {
     int out_height =
         static_cast<int>(ceil(static_cast<float>(_inp.first() - _window.get()) /
@@ -43,7 +57,7 @@ void Pooling::initialize_output_dimension() {
                              _stride.get())) +
                     1;
     _out = ImageShape(out_height, out_width);
-    _out_dim.push_back(_channels.get());
+    _out_dim[0] = _channels.get();
     _out_dim.push_back(out_height);
     _out_dim.push_back(out_width);
 }

src/math.cpp

@@ -59,7 +59,12 @@ void my_add_vec_to_mat_colwise(SharedStorage& A, const SharedStorage& B,
     dtype* d_A = A->gpu_pointer();
     const dtype* d_B = B->gpu_pointer_const();
     if (rows != B->get_rows()) {
-        throw std::runtime_error("Invalid size in addtion");
+        std::stringstream ss;
+        ss << "\nCannot add the two matrices as row numbers differ."
+            " A rows: " << rows << " vs B rows: " << B->get_rows() << " in:\n"
+           << __PRETTY_FUNCTION__ << "\ncalled from " << __FILE__ << " at "
+           << __LINE__;
+        throw std::invalid_argument(ss.str());
     }
     add_vec_to_mat_colwise(rows, cols, d_A, d_B, alpha);
     // cudaDeviceSyncronize();

src/network.cpp

@@ -49,10 +49,8 @@ NeuralNetwork::NeuralNetwork(const std::shared_ptr<Layer>& last_layer,
 
 void NeuralNetwork::insert_cnn_layer(const std::shared_ptr<Layer>& layer) {
     std::shared_ptr<Convolution> derived =
-               std::dynamic_pointer_cast<Convolution> (layer);
-    //std::shared_ptr<Convolution> d = dynamic_cast<Derived<int> *>(b);
-    std::shared_ptr<Layer> im2col =
-        std::make_shared<Im2ColLayer>(Im2ColLayer(derived));
+        std::dynamic_pointer_cast<Convolution>(layer);
+    std::shared_ptr<Layer> im2col = std::make_shared<Im2ColLayer>(derived);
     layer->_previous = im2col;
     layers.push_front(im2col);
     layers.push_front(layer);
@@ -68,9 +66,13 @@ void NeuralNetwork::construct_layers(std::shared_ptr<Layer> curr) {
         curr.swap(tmp);
     }
     if (curr->name() == "Input")
-            layers.push_front(curr);
+        layers.push_front(curr);
     else {
-        throw std::runtime_error("Must finish with the input");
+        std::stringstream ss;
+        ss << "Cannot recognize the layer name " << curr->name() << " in:\n"
+           << __PRETTY_FUNCTION__ << "\ncalled from " << __FILE__ << " at "
+           << __LINE__;
+        throw std::invalid_argument(ss.str());
     }
 }
 
@@ -91,7 +93,6 @@ void NeuralNetwork::allocate_storage(int obs, std::vector<SharedStorage>& inp,
 
 vector<SharedStorage> NeuralNetwork::allocate_forward(int obs) {
     vector<SharedStorage> vals;
-    // int out_dim(0);
     for (shared_ptr<Layer> layer : layers) {
         allocate_storage(obs, vals, layer);
     }
@@ -100,12 +101,10 @@ vector<SharedStorage> NeuralNetwork::allocate_forward(int obs) {
 
 vector<SharedStorage> NeuralNetwork::allocate_backward(int obs) {
     vector<SharedStorage> vals;
-    // int out_dim(0);
     std::deque<shared_ptr<Layer>>::iterator layer = layers.begin();
     std::deque<shared_ptr<Layer>>::iterator end = layers.end();
     --end;
     while (layer != end) {
-        // for (size_t i = 0; i < layers.size() - 1; i++) {
         allocate_storage(obs, vals, *layer);
         ++layer;
     }
@@ -128,8 +127,6 @@ void NeuralNetwork::forward_gpu(vector<SharedStorage>& values,
     std::deque<shared_ptr<Layer>>::iterator layer = layers.begin();
     ++layer;
     while (layer != layers.end()) {
-        // for (; it != layers.end(); ++it) {
-        // for (size_t layer_idx = 1; layer_idx < layers.size(); ++layer_idx) {
         (*layer)->forward_gpu(values[i], values[i + 1], type);
         i++;
         ++layer;
@@ -142,8 +139,6 @@ void NeuralNetwork::forward_cpu(vector<SharedStorage>& values,
     std::deque<shared_ptr<Layer>>::iterator layer = layers.begin();
     ++layer;
     while (layer != layers.end()) {
-        // for (; it != layers.end(); ++it) {
-        // for (size_t layer_idx = 1; layer_idx < layers.size(); ++layer_idx) {
         (*layer)->forward_gpu(values[i], values[i + 1], type);
         i++;
         ++layer;

src/train.cpp

@@ -155,12 +155,28 @@ void NeuralNetwork::get_new_sample(const vector<int>& samples, Matrix& x_train,
 
 // I need to instantiate a vector of shared pointers to SGD one for each layer
 // use this as part of train args!!!
+//
+int NeuralNetwork::check_input_dimension(const std::vector<int>& dim) {
+    int i = 1;
+    for (int shape : dim) i *= shape;
+    return i;
+}
+
 void NeuralNetwork::train(const Matrix& features, const Matrix& targets,
                           std::shared_ptr<GradientDescent>& sgd, Epochs _epoch,
                           Patience _patience, BatchSize _batch_size) {
-    if ((*layers.begin())->output_dimension()[0] != features.cols()) {
-        std::string m("N of input features != col in features, in:\n");
-        throw std::invalid_argument(m + __PRETTY_FUNCTION__);
+    std::vector<int> input_dim = layers[0]->output_dimension();
+    int expected_cols = check_input_dimension(input_dim);
+    if (expected_cols != features.cols()) {
+        std::stringstream ss;
+        ss << "The number of input features is: " << features.cols()
+           << " but the input layer expects: ";
+        std::copy(input_dim.begin(), input_dim.end(),
+                  std::ostream_iterator<int>(ss, " "));
+        ss << "in:\n"
+           << __PRETTY_FUNCTION__ << "\ncalled from " << __FILE__ << " at "
+           << __LINE__;
+        throw std::invalid_argument(ss.str());
     }
     train_args = std::make_unique<trainArgs>(
         features, targets, _epoch, _patience, _batch_size, sgd, layers);