Restructure project layout to allow mixing of pure Python and compiled module

- c++ library code in /lib
- pybind11 c++/python binding code in /src
- pure python code in /src/pybind11_numpy_example
  - add example pure python function `pure_python_list()`
- replace `-` with `_` in all filenames and paths
- update plots

Author: lkeegan

.github/workflows/ci.yml

@@ -14,23 +14,13 @@ jobs:
         with:
           submodules: "recursive"
 
-      - name: Make build directory
-        run: cmake -E make_directory ${{runner.workspace}}/build
-
-      - name: Configure cmake
-        shell: bash
-        working-directory: ${{runner.workspace}}/build
-        run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=Debug -DBUILD_DOCS=OFF -DBUILD_TESTING=ON -DBUILD_PYTHON=OFF
-
-      - name: Build
-        shell: bash
-        working-directory: ${{runner.workspace}}/build
-        run: cmake --build .
-
-      - name: Run tests
-        shell: bash
-        working-directory: ${{runner.workspace}}/build
-        run: ctest
+      - name: Build and run c++ tests
+        run: |
+          mkdir build
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_DOCS=OFF -DBUILD_PYTHON=OFF -DBUILD_TESTING=ON ..
+          cmake --build .
+          ctest
 
   python:
     name: "${{ matrix.os }} :: Python ${{ matrix.python-version }}"

.readthedocs.yml

@@ -1,9 +1,9 @@
 version: 2
 
 build:
-  os: "ubuntu-20.04"
+  os: "ubuntu-22.04"
   tools:
-    python: "3.9"
+    python: "3.12"
 
 sphinx:
   builder: html

CMakeLists.txt

@@ -3,14 +3,14 @@ cmake_minimum_required(VERSION 3.15..3.26)
 # Set a name and a version number for your project:
 project(
   pybind11-numpy-example
-  VERSION 0.0.7
+  VERSION 0.0.9
   LANGUAGES CXX)
 
 # Initialize some default paths
 include(GNUInstallDirs)
 
 # Define the minimum C++ standard that is required
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -19,15 +19,15 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 option(BUILD_PYTHON "Enable building of Python bindings" ON)
 option(BUILD_DOCS "Enable building of documentation" ON)
 
-# Build the library
-add_subdirectory(src)
+# Build the c++ library
+add_subdirectory(lib)
 
-# Build the tests
+# Build the c++ tests
 include(CTest)
 if(BUILD_TESTING)
   add_subdirectory(ext/Catch2)
   include(./ext/Catch2/extras/Catch.cmake)
-  add_subdirectory(tests)
+  add_subdirectory(tests/cpp)
 endif()
 
 # Build the documentation
@@ -37,26 +37,26 @@ endif()
 
 # Build the python bindings
 if(BUILD_PYTHON)
-  add_subdirectory(python)
+  add_subdirectory(src)
 endif()
 
 # Add an alias target for use if this project is included as a subproject in
 # another project
-add_library(pybind11-numpy-example::pybind11-numpy-example ALIAS
-            pybind11-numpy-example)
+add_library(pybind11_numpy_example::pybind11_numpy_example ALIAS
+            pybind11_numpy_example)
 
 # Install targets and configuration
 install(
-  TARGETS pybind11-numpy-example
-  EXPORT pybind11-numpy-example-config
+  TARGETS pybind11_numpy_example
+  EXPORT pybind11_numpy_example_config
   RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
   LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
   ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
 
 install(
-  EXPORT pybind11-numpy-example-config
-  NAMESPACE pybind11-numpy-example::
-  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pybind11-numpy-example)
+  EXPORT pybind11_numpy_example_config
+  NAMESPACE pybind11_numpy_example::
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pybind11_numpy_example)
 
 install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/include/
         DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})

README.md

@@ -29,9 +29,9 @@ a Numpy array is much faster and uses a lot less memory:
 
 # How
 
-The pybind11 code is in [python/pybind11-numpy-example_python.cpp](https://github.com/ssciwr/pybind11-numpy-example/blob/main/python/pybind11-numpy-example_python.cpp).
+The pybind11 code is in [src/pybind11_numpy_example_python.cpp](https://github.com/ssciwr/pybind11-numpy-example/blob/main/src/pybind11_numpy_example_python.cpp).
 
-The python project is defined in [pyproject.toml](https://github.com/ssciwr/pybind11-numpy-example/blob/main/pyproject.toml)
+The python package is defined in [pyproject.toml](https://github.com/ssciwr/pybind11-numpy-example/blob/main/pyproject.toml)
 and uses [scikit-build-core](https://github.com/scikit-build/scikit-build-core).
 
 Each tagged commit triggers a [GitHub action job](https://github.com/ssciwr/pybind11-numpy-example/actions/workflows/pypi.yml)

doc/index.rst

@@ -42,7 +42,7 @@ How
 ===
 
 The pybind11 code is in
-`python/pybind11-numpy-example\_python.cpp <https://github.com/ssciwr/pybind11-numpy-example/blob/main/python/pybind11-numpy-example_python.cpp>`__.
+`src/pybind11\_numpy\_example\_python.cpp <https://github.com/ssciwr/pybind11-numpy-example/blob/main/src/pybind11_numpy_example_python.cpp>`__.
 
 The python project is defined in `pyproject.toml <https://github.com/ssciwr/pybind11-numpy-example/blob/main/pyproject.toml>`__
 and uses `scikit-build-core <https://github.com/scikit-build/scikit-build-core>`__.

ext/Catch2

@@ -0,0 +1,5 @@
+add_library(pybind11_numpy_example pybind11_numpy_example.cpp)
+target_include_directories(
+  pybind11_numpy_example
+  PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/include/>
+         $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)

include/pybind11-numpy-example/pybind11-numpy-example.hpp include/pybind11_numpy_example/pybind11_numpy_example.hpp

@@ -0,0 +1 @@
+#include "pybind11_numpy_example/pybind11_numpy_example.hpp"

lib/CMakeLists.txt

@@ -9,7 +9,9 @@ description = "An example of using numpy with pybind11"
 readme = "README.md"
 license = {text = "MIT"}
 authors=[{name="Liam Keegan", email="[email protected]"}]
+maintainers=[{name="Liam Keegan", email="[email protected]"}]
 requires-python = ">=3.7"
+keywords = ["pybind11", "cibuildwheel", "c++", "pypi", "numpy", "simple", "example"]
 classifiers=[
     "Programming Language :: C++",
     "Programming Language :: Python :: 3 :: Only",
@@ -29,6 +31,7 @@ classifiers=[
 
 [project.urls]
 Github = "https://github.com/ssciwr/pybind11-numpy-example"
+Documentation = "https://pybind11-numpy-example.readthedocs.io"
 
 [project.optional-dependencies]
 test = ["pytest", "numpy"]
@@ -41,5 +44,5 @@ BUILD_DOCS = "OFF"
 
 [tool.cibuildwheel]
 test-extras = "test"
-test-command = "python -m pytest {project}/python/tests -v"
+test-command = "python -m pytest {project}/tests/python -v"
 test-skip = "pp* *-musllinux* *-manylinux_i686"

lib/pybind11_numpy_example.cpp

@@ -8,8 +8,9 @@ echo "#n time (seconds)" > time_list.txt
 cp time_list.txt time_array.txt
 cp time_list.txt time_array_nocopy.txt
 
-for n in 1000 10000 100000 1000000 10000000 50000000 100000000 200000000 300000000 400000000
+for n in 1000 10000 100000 1000000 10000000 50000000 100000000 200000000 300000000 400000000 600000000 800000000 1000000000 1200000000
 do
+    echo $n
     m_list=$(./memory.py $n 0)
     m_array=$(./memory.py $n 1)
     m_array_nocopy=$(./memory.py $n 2)
@@ -24,9 +25,9 @@ do
     echo "${n} ${t_array}" >> time_array.txt
     echo "${n} ${t_array_nocopy}" >> time_array_nocopy.txt
 done
-
-for n in 1000000000 2000000000 3000000000 4000000000
+for n in 2000000000 3000000000 4000000000 6000000000 8000000000 10000000000 12000000000
 do
+    echo $n
     m_array=$(./memory.py $n 1)
     echo "${n} ${m_array}" >> mem_array.txt
     m_array_nocopy=$(./memory.py $n 2)
@@ -38,8 +39,9 @@ do
     echo "${n} ${t_array_nocopy}" >> time_array_nocopy.txt
 done
 
-for n in 5000000000 6000000000 7000000000 8000000000
+for n in 14000000000 16000000000 18000000000 20000000000 24000000000
 do
+    echo $n
     m_array_nocopy=$(./memory.py $n 2)
     echo "${n} ${m_array_nocopy}" >> mem_array_nocopy.txt
 

pyproject.toml

@@ -1,15 +1,22 @@
 #n memory (kb)
-1000 18896
-10000 18864
-100000 19560
-1000000 23260
-10000000 58380
-50000000 214536
-100000000 410024
-200000000 800452
-300000000 1191012
-400000000 1581412
-1000000000 3925392
-2000000000 7831684
-3000000000 11737892
-4000000000 15644044
+1000 20648
+10000 20664
+100000 20840
+1000000 24520
+10000000 59672
+50000000 215920
+100000000 411412
+200000000 801848
+300000000 1192548
+400000000 1583112
+600000000 2363992
+800000000 3145808
+1000000000 3926680
+1200000000 4708296
+2000000000 7832800
+3000000000 11739192
+4000000000 15645488
+6000000000 23458288
+8000000000 31270592
+10000000000 39083108
+12000000000 46895616

python/CMakeLists.txt

@@ -1,19 +1,27 @@
 #n memory (kb)
-1000 18808
-10000 18864
-100000 18936
-1000000 20912
-10000000 38284
-50000000 116396
-100000000 214164
-200000000 409508
-300000000 604768
-400000000 800084
-1000000000 1971984
-2000000000 3925016
-3000000000 5878236
-4000000000 7831396
-5000000000 9784604
-6000000000 11737484
-7000000000 13690668
-8000000000 15643960
+1000 20656
+10000 20668
+100000 20844
+1000000 22596
+10000000 39936
+50000000 118080
+100000000 215964
+200000000 411264
+300000000 606336
+400000000 801716
+600000000 1192512
+800000000 1583148
+1000000000 1973368
+1200000000 2364404
+2000000000 3926908
+3000000000 5880000
+4000000000 7832964
+6000000000 11739264
+8000000000 15645624
+10000000000 19551908
+12000000000 23457968
+14000000000 27364404
+16000000000 31270648
+18000000000 35176512
+20000000000 39083136
+24000000000 46895548

scripts/benchmarks.sh

@@ -1,11 +1,15 @@
 #n memory (kb)
-1000 0
-10000 444
-100000 4164
-1000000 41356
-10000000 414140
-50000000 2069920
-100000000 4139380
-200000000 8278588
-300000000 12418828
-400000000 16557252
+1000 192
+10000 384
+100000 4224
+1000000 41088
+10000000 410304
+50000000 2050752
+100000000 4101504
+200000000 8203200
+300000000 12304704
+400000000 16405824
+600000000 24609216
+800000000 32811840
+1000000000 41014464
+1200000000 49218432

scripts/mem_array.txt

@@ -2,7 +2,7 @@
 
 # simple script to print approx memory usage
 
-import pybind11numpyexample
+import pybind11_numpy_example
 import resource
 import sys
 
@@ -11,11 +11,11 @@
 max_mem_before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
 
 if data_type == 0:
-    a = pybind11numpyexample.vector_as_list(n)
+    a = pybind11_numpy_example.vector_as_list(n)
 elif data_type == 1:
-    a = pybind11numpyexample.vector_as_array(n)
+    a = pybind11_numpy_example.vector_as_array(n)
 elif data_type == 2:
-    a = pybind11numpyexample.vector_as_array_nocopy(n)
+    a = pybind11_numpy_example.vector_as_array_nocopy(n)
 
 max_mem_after = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
 

scripts/mem_array_nocopy.txt

@@ -2,7 +2,7 @@
 
 # simple script to print approx time taken
 
-import pybind11numpyexample
+import pybind11_numpy_example
 import timeit
 import sys
 
@@ -13,11 +13,11 @@
 
 def doit():
     if data_type == 0:
-        return pybind11numpyexample.vector_as_list(n)
+        return pybind11_numpy_example.vector_as_list(n)
     elif data_type == 1:
-        return pybind11numpyexample.vector_as_array(n)
+        return pybind11_numpy_example.vector_as_array(n)
     elif data_type == 2:
-        return pybind11numpyexample.vector_as_array_nocopy(n)
+        return pybind11_numpy_example.vector_as_array_nocopy(n)
 
 
 print(timeit.timeit(doit, number=iters) / iters)