Skip to content

Commit 4029342

Browse files
ilumsdenilumsden
authored
Adds Python bindings to Caliper with pybind11 (#573)
* Implements Python bindings for Caliper * Improves the exported type enums * Adds examples of the Python API * Fixes default parameter for cali_attr_properties * Removes default arugments for Attribute and Annotation in place of multiple constructors * Adds unit tests for Python bindings * Adds docstrings to Python bindings * Moves Python bindings to src/interface and adds logic to help with unit testing * Updates GH Actions runner to install Pybind11 correctly * Removes Variant from Python bindings * Removes redundant install of Pybind11 with apt * Enables bitwise arithmetic in bindings for cali_attr_properties
1 parent d1f280e commit 4029342

28 files changed

+1233
-5
lines changed

‎.github/workflows/cmake.yml

+9-2
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,21 @@ jobs:
2020

2121
steps:
2222
- uses: actions/checkout@v2
23+
24+
- uses: actions/setup-python@v5
25+
with:
26+
python-version: '3.10'
27+
cache: 'pip'
2328

2429
- name: Install dependencies
25-
run: sudo apt-get install libdw-dev libunwind-dev gfortran
30+
run: |
31+
sudo apt-get install libdw-dev libunwind-dev gfortran
32+
python3 -m pip install pybind11
2633
2734
- name: Configure CMake
2835
# Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
2936
# See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
30-
run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -C ${{github.workspace}}/cmake/hostconfig/github-actions.cmake
37+
run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -Dpybind11_DIR=$(pybind11-config --cmakedir) -C ${{github.workspace}}/cmake/hostconfig/github-actions.cmake
3138

3239
- name: Build
3340
# Build your program with the given configuration

‎CMakeLists.txt

+13-3
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,9 @@ option(BUILD_SHARED_LIBS "Build shared libraries" TRUE)
5050
option(CMAKE_INSTALL_RPATH_USE_LINK_PATH "Add rpath for all dependencies" TRUE)
5151

5252
# Optional Fortran
53-
add_caliper_option(WITH_FORTRAN "Install Fortran interface" FALSE)
54-
add_caliper_option(WITH_TOOLS "Build Caliper tools" TRUE)
53+
add_caliper_option(WITH_FORTRAN "Install Fortran interface" FALSE)
54+
add_caliper_option(WITH_PYTHON_BINDINGS "Install Python bindings" FALSE)
55+
add_caliper_option(WITH_TOOLS "Build Caliper tools" TRUE)
5556

5657
add_caliper_option(WITH_NVTX "Enable NVidia nvtx bindings for NVprof and NSight (requires CUDA)" FALSE)
5758
add_caliper_option(WITH_CUPTI "Enable CUPTI service (CUDA performance analysis)" FALSE)
@@ -421,7 +422,12 @@ if(WITH_TAU)
421422
endif()
422423

423424
# Find Python
424-
find_package(Python COMPONENTS Interpreter REQUIRED)
425+
set(FIND_PYTHON_COMPONENTS "Interpreter")
426+
if (WITH_PYTHON_BINDINGS)
427+
set(FIND_PYTHON_COMPONENTS "Development" ${FIND_PYTHON_COMPONENTS})
428+
endif ()
429+
430+
find_package(Python COMPONENTS ${FIND_PYTHON_COMPONENTS} REQUIRED)
425431
set(CALI_PYTHON_EXECUTABLE Python::Interpreter)
426432

427433
if (WITH_SAMPLER)
@@ -495,6 +501,10 @@ configure_file(
495501
include_directories(${PROJECT_BINARY_DIR}/include)
496502
include_directories(include)
497503

504+
if (WITH_PYTHON_BINDINGS AND BUILD_TESTING)
505+
set(PYPATH_TESTING "" CACHE INTERNAL "")
506+
endif()
507+
498508
add_subdirectory(ext)
499509
add_subdirectory(src)
500510

‎cmake/get_python_install_paths.py

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
import sys
2+
import sysconfig
3+
4+
if len(sys.argv) != 3 or sys.argv[1] not in ("purelib", "platlib"):
5+
raise RuntimeError(
6+
"Usage: python get_python_install_paths.py <purelib | platlib> <sysconfig_scheme>"
7+
)
8+
9+
install_dir = sysconfig.get_path(sys.argv[1], sys.argv[2], {"userbase": "", "base": ""})
10+
11+
if install_dir.startswith("/"):
12+
install_dir = install_dir[1:]
13+
14+
print(install_dir, end="")

‎cmake/hostconfig/github-actions.cmake

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ set(WITH_NVPROF Off CACHE BOOL "")
1212
set(WITH_PAPI Off CACHE BOOL "")
1313
set(WITH_SAMPLER On CACHE BOOL "")
1414
set(WITH_VTUNE Off CACHE BOOL "")
15+
set(WITH_PYTHON_BINDINGS On CACHE BOOL "")
1516

1617
set(WITH_DOCS Off CACHE BOOL "")
1718
set(BUILD_TESTING On CACHE BOOL "")

‎examples/apps/cali-perfproblem-branch-mispred.py

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# Copyright (c) 2024, Lawrence Livermore National Security, LLC.
2+
# See top-level LICENSE file for details.
3+
4+
from pycaliper.high_level import annotate_function
5+
from pycaliper.annotation import Annotation
6+
7+
import numpy as np
8+
9+
@annotate_function()
10+
def init(arraySize: int, sort: bool) -> np.array:
11+
data = np.random.randint(256, size=arraySize)
12+
if sort:
13+
data = np.sort(data)
14+
return data
15+
16+
17+
@annotate_function()
18+
def work(data: np.array):
19+
data_sum = 0
20+
for _ in range(100):
21+
for val in np.nditer(data):
22+
if val >= 128:
23+
data_sum += val
24+
print("sum =", data_sum)
25+
26+
27+
@annotate_function()
28+
def benchmark(arraySize: int, sort: bool):
29+
sorted_ann = Annotation("sorted")
30+
sorted_ann.set(sort)
31+
print("Intializing benchmark data with sort =", sort)
32+
data = init(arraySize, sort)
33+
print("Calculating sum of values >= 128")
34+
work(data)
35+
print("Done!")
36+
sorted_ann.end()
37+
38+
39+
@annotate_function()
40+
def main():
41+
arraySize = 32768
42+
benchmark(arraySize, True)
43+
benchmark(arraySize, False)
44+
45+
46+
if __name__ == "__main__":
47+
main()
48+

‎examples/apps/py-example.py

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# Copyright (c) 2024, Lawrence Livermore National Security, LLC.
2+
# See top-level LICENSE file for details.
3+
4+
from pycaliper.high_level import annotate_function
5+
from pycaliper.config_manager import ConfigManager
6+
from pycaliper.instrumentation import (
7+
set_global_byname,
8+
begin_region,
9+
end_region,
10+
)
11+
from pycaliper.loop import Loop
12+
13+
import argparse
14+
import sys
15+
import time
16+
17+
18+
def get_available_specs_doc(mgr: ConfigManager):
19+
doc = ""
20+
for cfg in mgr.available_config_specs():
21+
doc += mgr.get_documentation_for_spec(cfg)
22+
doc += "\n"
23+
return doc
24+
25+
26+
@annotate_function()
27+
def foo(i: int) -> float:
28+
nsecs = max(i * 500, 100000)
29+
secs = nsecs / 10**9
30+
time.sleep(secs)
31+
return 0.5 * i
32+
33+
34+
def main():
35+
mgr = ConfigManager()
36+
37+
parser = argparse.ArgumentParser()
38+
parser.add_argument("--caliper_config", "-P", type=str, default="",
39+
help="Configuration for Caliper\n{}".format(get_available_specs_doc(mgr)))
40+
parser.add_argument("iterations", type=int, nargs="?", default=4,
41+
help="Number of iterations")
42+
args = parser.parse_args()
43+
44+
mgr.add(args.caliper_config)
45+
46+
if mgr.error():
47+
print("Caliper config error:", mgr, file=sys.stderr)
48+
49+
mgr.start()
50+
51+
set_global_byname("iterations", args.iterations)
52+
set_global_byname("caliper.config", args.caliper_config)
53+
54+
begin_region("main")
55+
56+
begin_region("init")
57+
t = 0
58+
end_region("init")
59+
60+
loop_ann = Loop("mainloop")
61+
62+
for i in range(args.iterations):
63+
loop_ann.start_iteration(i)
64+
t *= foo(i)
65+
loop_ann.end_iteration()
66+
67+
loop_ann.end()
68+
69+
end_region("main")
70+
71+
mgr.flush()
72+
73+
74+
if __name__ == "__main__":
75+
main()
76+

‎src/CMakeLists.txt

+4
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,10 @@ if (WITH_TOOLS)
5858
add_subdirectory(tools)
5959
endif()
6060

61+
if (WITH_PYTHON_BINDINGS)
62+
add_subdirectory(interface/python)
63+
endif()
64+
6165
install(
6266
TARGETS
6367
caliper

‎src/interface/python/CMakeLists.txt

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
set(PYCALIPER_BINDING_SOURCES
2+
annotation.cpp
3+
config_manager.cpp
4+
instrumentation.cpp
5+
loop.cpp
6+
mod.cpp
7+
)
8+
9+
set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
10+
11+
find_package(pybind11 CONFIG REQUIRED)
12+
13+
set(PYCALIPER_SYSCONFIG_SCHEME "posix_user" CACHE STRING "Scheme used for searching for pycaliper's install path. Valid options can be determined with 'sysconfig.get_scheme_names()'")
14+
15+
execute_process(COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/get_python_install_paths.py purelib ${PYCALIPER_SYSCONFIG_SCHEME} OUTPUT_VARIABLE PYCALIPER_SITELIB)
16+
execute_process(COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/get_python_install_paths.py platlib ${PYCALIPER_SYSCONFIG_SCHEME} OUTPUT_VARIABLE PYCALIPER_SITEARCH)
17+
18+
message(STATUS "Pycaliper sitelib: ${PYCALIPER_SITELIB}")
19+
message(STATUS "Pycaliper sitearch: ${PYCALIPER_SITEARCH}")
20+
21+
set(PYCALIPER_SITELIB "${PYCALIPER_SITELIB}/pycaliper")
22+
set(PYCALIPER_SITEARCH "${PYCALIPER_SITEARCH}/pycaliper")
23+
24+
pybind11_add_module(__pycaliper_impl ${PYCALIPER_BINDING_SOURCES})
25+
target_link_libraries(__pycaliper_impl PUBLIC caliper)
26+
target_compile_features(__pycaliper_impl PUBLIC cxx_std_11)
27+
target_include_directories(__pycaliper_impl PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
28+
29+
add_custom_target(
30+
pycaliper_test ALL # Always build pycaliper_test
31+
COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/pycaliper
32+
COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/pycaliper ${CMAKE_CURRENT_BINARY_DIR}/pycaliper
33+
COMMENT "Copying pycaliper Python source to ${CMAKE_CURRENT_BINARY_DIR}/pycaliper"
34+
)
35+
add_dependencies(__pycaliper_impl pycaliper_test)
36+
37+
if (BUILD_TESTING)
38+
set(PYPATH_TESTING ${CMAKE_CURRENT_BINARY_DIR} CACHE INTERNAL "")
39+
add_custom_target(
40+
pycaliper_symlink_lib_in_build ALL
41+
COMMAND ${CMAKE_COMMAND} -E create_symlink
42+
$<TARGET_FILE:__pycaliper_impl>
43+
${CMAKE_CURRENT_BINARY_DIR}/pycaliper/$<TARGET_FILE_NAME:__pycaliper_impl>
44+
COMMENT "Creating symlink between Python C module and build directory for testing"
45+
DEPENDS __pycaliper_impl
46+
)
47+
message(STATUS "Will add ${PYPATH_TESTING} to PYTHONPATH during test")
48+
endif()
49+
50+
install(
51+
DIRECTORY
52+
pycaliper/
53+
DESTINATION
54+
${PYCALIPER_SITELIB}
55+
)
56+
57+
install(
58+
TARGETS
59+
__pycaliper_impl
60+
ARCHIVE DESTINATION
61+
${PYCALIPER_SITEARCH}
62+
LIBRARY DESTINATION
63+
${PYCALIPER_SITEARCH}
64+
)

0 commit comments

Comments
 (0)