Travis scripts for python3 and pytest for cmake. Also fixes CUDA CMake build issue #2722.

Author: philkr

.travis.yml

@@ -6,24 +6,31 @@ env:
     - WITH_CUDA=false WITH_CMAKE=true
     - WITH_CUDA=true WITH_CMAKE=false
     - WITH_CUDA=true WITH_CMAKE=true
+    - WITH_CUDA=false WITH_CMAKE=true PYTHON_VERSION=3
 
 language: cpp
 
 # Cache Ubuntu apt packages.
-cache: apt
+cache:
+  apt: true
+  directories:
+  - /home/travis/miniconda
+  - /home/travis/miniconda2
+  - /home/travis/miniconda3
 
 compiler: gcc
 
 before_install:
   - export NUM_THREADS=4
   - export SCRIPTS=./scripts/travis
+  - export CONDA_DIR="/home/travis/miniconda$PYTHON_VERSION"
 
 install:
   - sudo -E $SCRIPTS/travis_install.sh
 
 before_script:
-  - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/cuda/lib64
-  - export PATH=/home/travis/miniconda/bin:$PATH
+  - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/cuda/lib64:$CONDA_DIR/lib
+  - export PATH=$CONDA_DIR/bin:$PATH
   - if ! $WITH_CMAKE; then $SCRIPTS/travis_setup_makefile_config.sh; fi
 
 script: $SCRIPTS/travis_build_and_test.sh

CMakeLists.txt

@@ -62,6 +62,10 @@ add_subdirectory(docs)
 # ---[ Linter target
 add_custom_target(lint COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/lint.cmake)
 
+# ---[ pytest target
+add_custom_target(pytest COMMAND python${python_version} -m unittest discover -s caffe/test WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/python )
+add_dependencies(pytest pycaffe)
+
 # ---[ Configuration summary
 caffe_print_configuration_summary()
 

Makefile

@@ -228,7 +228,7 @@ ifeq ($(LINUX), 1)
 	CXX ?= /usr/bin/g++
 	GCCVERSION := $(shell $(CXX) -dumpversion | cut -f1,2 -d.)
 	# older versions of gcc are too dumb to build boost with -Wuninitalized
-	ifeq ($(shell echo $(GCCVERSION) \< 4.6 | bc), 1)
+	ifeq ($(shell echo | awk '{exit $(GCCVERSION) < 4.6;}'), 1)
 		WARNINGS += -Wno-uninitialized
 	endif
 	# boost::thread is reasonably called boost_thread (compare OS X)
@@ -243,7 +243,7 @@ ifeq ($(OSX), 1)
 	CXX := /usr/bin/clang++
 	ifneq ($(CPU_ONLY), 1)
 		CUDA_VERSION := $(shell $(CUDA_DIR)/bin/nvcc -V | grep -o 'release \d' | grep -o '\d')
-		ifeq ($(shell echo $(CUDA_VERSION) \< 7.0 | bc), 1)
+		ifeq ($(shell echo | awk '{exit $(CUDA_VERSION) < 7.0;}'), 1)
 			CXXFLAGS += -stdlib=libstdc++
 			LINKFLAGS += -stdlib=libstdc++
 		endif

cmake/Dependencies.cmake

@@ -106,14 +106,15 @@ if(BUILD_python)
 
     while(NOT "${version}" STREQUAL "" AND NOT Boost_PYTHON_FOUND)
       STRING( REGEX REPLACE "([0-9.]+).[0-9]+" "\\1" version ${version} )
-      STRING( REGEX MATCHALL "([0-9.]+).[0-9]+" has_more_version ${version} )
-      if("${has_more_version}" STREQUAL "")
-        break()
-      endif()
 
       STRING( REPLACE "." "" boost_py_version ${version} )
       find_package(Boost 1.46 COMPONENTS "python-py${boost_py_version}")
       set(Boost_PYTHON_FOUND ${Boost_PYTHON-PY${boost_py_version}_FOUND})
+      
+      STRING( REGEX MATCHALL "([0-9.]+).[0-9]+" has_more_version ${version} )
+      if("${has_more_version}" STREQUAL "")
+        break()
+      endif()
     endwhile()
     if(NOT Boost_PYTHON_FOUND)
       find_package(Boost 1.46 COMPONENTS python)

python/caffe/test/test_net.py

@@ -2,6 +2,7 @@
 import tempfile
 import os
 import numpy as np
+import six
 
 import caffe
 
@@ -10,7 +11,7 @@ def simple_net_file(num_output):
     """Make a simple net prototxt, based on test_net.cpp, returning the name
     of the (temporary) file."""
 
-    f = tempfile.NamedTemporaryFile(delete=False)
+    f = tempfile.NamedTemporaryFile(mode='w+', delete=False)
     f.write("""name: 'testnet' force_backward: true
     layer { type: 'DummyData' name: 'data' top: 'data' top: 'label'
       dummy_data_param { num: 5 channels: 2 height: 3 width: 4
@@ -47,7 +48,7 @@ def setUp(self):
     def test_memory(self):
         """Check that holding onto blob data beyond the life of a Net is OK"""
 
-        params = sum(map(list, self.net.params.itervalues()), [])
+        params = sum(map(list, six.itervalues(self.net.params)), [])
         blobs = self.net.blobs.values()
         del self.net
 
@@ -67,7 +68,7 @@ def test_inputs_outputs(self):
         self.assertEqual(self.net.outputs, ['loss'])
 
     def test_save_and_read(self):
-        f = tempfile.NamedTemporaryFile(delete=False)
+        f = tempfile.NamedTemporaryFile(mode='w+', delete=False)
         f.close()
         self.net.save(f.name)
         net_file = simple_net_file(self.num_output)

python/caffe/test/test_net_spec.py

@@ -43,7 +43,7 @@ def anon_lenet(batch_size):
 
 class TestNetSpec(unittest.TestCase):
     def load_net(self, net_proto):
-        f = tempfile.NamedTemporaryFile(delete=False)
+        f = tempfile.NamedTemporaryFile(mode='w+', delete=False)
         f.write(str(net_proto))
         f.close()
         return caffe.Net(f.name, caffe.TEST)

python/caffe/test/test_python_layer.py

@@ -1,6 +1,7 @@
 import unittest
 import tempfile
 import os
+import six
 
 import caffe
 
@@ -22,7 +23,7 @@ def backward(self, top, propagate_down, bottom):
 
 
 def python_net_file():
-    with tempfile.NamedTemporaryFile(delete=False) as f:
+    with tempfile.NamedTemporaryFile(mode='w+', delete=False) as f:
         f.write("""name: 'pythonnet' force_backward: true
         input: 'data' input_shape { dim: 10 dim: 9 dim: 8 }
         layer { type: 'Python' name: 'one' bottom: 'data' top: 'one'
@@ -58,6 +59,6 @@ def test_reshape(self):
         s = 4
         self.net.blobs['data'].reshape(s, s, s, s)
         self.net.forward()
-        for blob in self.net.blobs.itervalues():
+        for blob in six.itervalues(self.net.blobs):
             for d in blob.data.shape:
                 self.assertEqual(s, d)

python/caffe/test/test_solver.py

@@ -2,6 +2,7 @@
 import tempfile
 import os
 import numpy as np
+import six
 
 import caffe
 from test_net import simple_net_file
@@ -11,7 +12,7 @@ class TestSolver(unittest.TestCase):
     def setUp(self):
         self.num_output = 13
         net_f = simple_net_file(self.num_output)
-        f = tempfile.NamedTemporaryFile(delete=False)
+        f = tempfile.NamedTemporaryFile(mode='w+', delete=False)
         f.write("""net: '""" + net_f + """'
         test_iter: 10 test_interval: 10 base_lr: 0.01 momentum: 0.9
         weight_decay: 0.0005 lr_policy: 'inv' gamma: 0.0001 power: 0.75
@@ -45,8 +46,8 @@ def test_net_memory(self):
 
         total = 0
         for net in nets:
-            for ps in net.params.itervalues():
+            for ps in six.itervalues(net.params):
                 for p in ps:
                     total += p.data.sum() + p.diff.sum()
-            for bl in net.blobs.itervalues():
+            for bl in six.itervalues(net.blobs):
                 total += bl.data.sum() + bl.diff.sum()

scripts/travis/travis_build_and_test.sh

@@ -7,8 +7,17 @@ MAKE="make --jobs=$NUM_THREADS --keep-going"
 if $WITH_CMAKE; then
   mkdir build
   cd build
-  cmake -DBUILD_python=ON -DCMAKE_BUILD_TYPE=Release -DCPU_ONLY=ON ..
+  CPU_ONLY=" -DCPU_ONLY=ON"
+  if ! $WITH_CUDA; then
+    CPU_ONLY=" -DCPU_ONLY=OFF"
+  fi
+  PYTHON_ARGS=""
+  if [ "$PYTHON_VERSION" = "3" ]; then
+    PYTHON_ARGS="$PYTHON_ARGS -Dpython_version=3 -DBOOST_LIBRARYDIR=$CONDA_DIR/lib/"
+  fi
+  cmake -DBUILD_python=ON -DCMAKE_BUILD_TYPE=Release $CPU_ONLY $PYTHON_ARGS -DCMAKE_INCLUDE_PATH="$CONDA_DIR/include/" -DCMAKE_LIBRARY_PATH="$CONDA_DIR/lib/" ..
   $MAKE
+  $MAKE pytest
   if ! $WITH_CUDA; then
     $MAKE runtest
     $MAKE lint

scripts/travis/travis_install.sh

@@ -4,17 +4,15 @@
 set -e
 
 MAKE="make --jobs=$NUM_THREADS"
-
 # Install apt packages where the Ubuntu 12.04 default and ppa works for Caffe
 
 # This ppa is for gflags and glog
 add-apt-repository -y ppa:tuleu/precise-backports
 apt-get -y update
 apt-get install \
     wget git curl \
-    python-dev python-numpy \
+    python-dev python-numpy python3-dev\
     libleveldb-dev libsnappy-dev libopencv-dev \
-    libboost-dev libboost-system-dev libboost-python-dev libboost-thread-dev \
     libprotobuf-dev protobuf-compiler \
     libatlas-dev libatlas-base-dev \
     libhdf5-serial-dev libgflags-dev libgoogle-glog-dev \
@@ -24,9 +22,10 @@ apt-get install \
 # if needed.  By default, Aptitude in Ubuntu 12.04 installs CMake 2.8.7, but
 # Caffe requires a minimum CMake version of 2.8.8.
 if $WITH_CMAKE; then
-  add-apt-repository -y ppa:ubuntu-sdk-team/ppa
-  apt-get -y update
-  apt-get -y install cmake
+  # cmake 3 will make sure that the python interpreter and libraries match
+  wget http://www.cmake.org/files/v3.2/cmake-3.2.3-Linux-x86_64.sh -O cmake3.sh
+  chmod +x cmake3.sh
+  ./cmake3.sh --prefix=/usr/ --skip-license --exclude-subdir
 fi
 
 # Install CUDA, if needed
@@ -60,10 +59,37 @@ rm -f $LMDB_FILE
 
 # Install the Python runtime dependencies via miniconda (this is much faster
 # than using pip for everything).
-wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh
-chmod +x miniconda.sh
-./miniconda.sh -b
-export PATH=/home/travis/miniconda/bin:$PATH
-conda update --yes conda
-conda install --yes numpy scipy matplotlib scikit-image pip
-pip install protobuf
+export PATH=$CONDA_DIR/bin:$PATH
+if [ ! -d $CONDA_DIR ]; then
+	if [ "$PYTHON_VERSION" -eq "3" ]; then
+		wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+	else
+		wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh
+	fi
+	chmod +x miniconda.sh
+	./miniconda.sh -b -p $CONDA_DIR
+	
+	conda update --yes conda
+	conda install --yes numpy scipy matplotlib scikit-image pip
+	# Let conda install boost (so that boost_python matches)
+	conda install --yes -c https://conda.binstar.org/menpo boost=1.56.0
+fi
+
+# install protobuf 3 (just use the miniconda3 directory to avoid having to setup the path again)
+if [ "$PYTHON_VERSION" -eq "3" ] && [ ! -e "$CONDA_DIR/bin/protoc" ]; then
+	pushd .
+	wget https://github.com/google/protobuf/archive/v3.0.0-alpha-3.1.tar.gz -O protobuf-3.tar.gz
+	tar -C /tmp -xzvf protobuf-3.tar.gz
+	cd /tmp/protobuf-3*/
+	./autogen.sh
+	./configure --prefix=$CONDA_DIR
+	$MAKE
+	$MAKE install
+	popd
+fi
+
+if [ "$PYTHON_VERSION" -eq "3" ]; then
+	pip install --pre protobuf
+else
+	pip install protobuf
+fi

scripts/travis/travis_setup_makefile_config.sh

@@ -12,7 +12,9 @@ if $WITH_CUDA; then
 fi
 
 cat << 'EOF' >> Makefile.config
-ANACONDA_HOME := $(HOME)/miniconda
+# Travis' nvcc doesn't like newer boost versions
+NVCCFLAGS := -Xcudafe --diag_suppress=cc_clobber_ignored -Xcudafe --diag_suppress=useless_using_declaration -Xcudafe --diag_suppress=set_but_not_used
+ANACONDA_HOME := $(CONDA_DIR)
 PYTHON_INCLUDE := $(ANACONDA_HOME)/include \
 		$(ANACONDA_HOME)/include/python2.7 \
 		$(ANACONDA_HOME)/lib/python2.7/site-packages/numpy/core/include

src/caffe/layer_factory.cpp

@@ -1,3 +1,8 @@
+// Make sure we include Python.h before any system header
+// to avoid _POSIX_C_SOURCE redefinition
+#ifdef WITH_PYTHON_LAYER
+#include <boost/python.hpp>
+#endif
 #include <string>
 
 #include "caffe/layer.hpp"