implement the sparse submanifold without kernel center consideration

Author: vera121

FEATURES.md

@@ -99,6 +99,7 @@ where4_layer
 DIV and MIN in EltwiseOp  
 axis in EltwiseParameter (broadcasting support for 2nd bottom blob in eltwise_layer)    
 min_first in ArgMaxParameter  
+submanifold_sparse in ConvolutionParameter  
 pad_type (deprecated, "SAME" style padding) in ConvolutionParameter and PoolingParameter  
 pad_l, pad_r, pad_t and pad_b (arbitrary 2D padding) in ConvolutionParameter and PoolingParameter  
 AVE_EXC_PAD (average pooling excluding the paddings), AVE_TF (deprecated, alias for AVE_EXC_PAD) in PoolingParameter  

include/caffe/layers/base_conv_layer.hpp

@@ -111,6 +111,7 @@ class BaseConvolutionLayer : public Layer<Dtype> {
   bool per_channel_scale_weight_; //CUSTOMIZATION
   bool per_channel_scale_output_; //CUSTOMIZATION
   int quantize_method_; //CUSTOMIZATION
+  bool submanifold_sparse_;
 
  private:
   // wrap im2col/col2im so we don't have to remember the (long) argument lists

src/caffe/layers/base_conv_layer.cpp

@@ -87,6 +87,8 @@ void BaseConvolutionLayer<Dtype>::LayerSetUpInternal(LayerParam conv_param,
   quantize_method_ = conv_param.quantize_method();
   per_channel_scale_weight_ = conv_param.per_channel_scale_weight();
   per_channel_scale_output_ = conv_param.per_channel_scale_output();
+
+  submanifold_sparse_ = conv_param.submanifold_sparse();
   //CUSTOMIZATION-->
 
   // Setup pad dimensions (pad_).

src/caffe/layers/conv_layer.cpp

@@ -233,6 +233,55 @@ void ConvolutionLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
       weight_mutable += slice;
     }
   }
+
+  if(this->submanifold_sparse_)
+  {
+    if(bottom[0]->num_axes()==4)
+    {
+      CHECK_EQ(bottom[0]->height(), top[0]->height())<<
+          "Input and output blob height not equal! Submanifold sparse computation is invalid!";
+      CHECK_EQ(bottom[0]->width(), top[0]->width())<<
+          "Input and output blob width not equal! Submanifold sparse computation is invalid!";
+    }
+    else if(bottom[0]->num_axes()==5)
+    {
+      CHECK_EQ(bottom[0]->shape(2), top[0]->shape(2))<<
+          "Input and output blob depth not equal! Submanifold sparse computation is invalid!";
+      CHECK_EQ(bottom[0]->shape(3), top[0]->shape(3))<<
+          "Input and output blob height not equal! Submanifold sparse computation is invalid!";
+      CHECK_EQ(bottom[0]->shape(4), top[0]->shape(4))<<
+          "Input and output blob width not equal! Submanifold sparse computation is invalid!";
+    }
+    else
+    {
+      CHECK_EQ(bottom[0]->num_axes(), 3)<<"Not support Submanifold sparse computation for such blob dimension yet!";
+      CHECK_EQ(bottom[0]->shape(2), top[0]->shape(2))<<
+           "Input and output blob length not equal! Submanifold sparse computation is invalid!";
+    }
+    LOG(INFO)<<"Starts submanifold sparse computation.";
+
+    for(int index=0; index<bottom[0]->count(2); index++)
+    {
+      bool active=false;
+      // TODO: add handling for pre-quantized model with non 0 zero-points
+      for(int in_c=0; in_c<bottom[0]->shape(1); in_c++)
+      {
+        Dtype data=bottom[0]->cpu_data()[in_c*bottom[0]->count(2)+index];
+        if(data!=Dtype(0))
+        {
+          active = true;
+          break;
+        }
+      }
+      if(!active)
+      {
+        for(int out_c=0; out_c<top[0]->shape(1); out_c++)
+        {
+          top[0]->mutable_cpu_data()[out_c*bottom[0]->count(2)+index]=0;
+        }
+      }
+    }
+  }
 }
 
 template <typename Dtype>

src/caffe/proto/caffe.proto

@@ -1428,7 +1428,7 @@ message ConvolutionParameter {
   optional bool per_channel_scale_weight = 39 [default = false];
   // CUSTOMIZATION, whether to have per-channel scale & zero_points for output (scale data will be stored in caffemodel)
   optional bool per_channel_scale_output = 41 [default = false];
-
+  optional bool submanifold_sparse = 42 [default = false];
 
   //<--CUSTOMIZATION
   enum SaturateMethod {
@@ -2746,7 +2746,7 @@ message SqueezeConvolutionParameter {
   // CUSTOMIZATION, whether to have per-channel scale & zero_points for weights/bias (data will be stored in caffemodel)
   optional bool per_channel_scale_weight = 59 [default = false];
   optional bool per_channel_scale_output = 61 [default = false];
-
+  optional bool submanifold_sparse = 62 [default = false];
 
   //<--CUSTOMIZATION
   enum SaturateMethod {