Some CI app test refactoring, add C API test cases

Author: daboehme

CMakeLists.txt

@@ -28,10 +28,17 @@ option(BUILD_SHARED_LIBS "Build shared libraries" TRUE)
 # RPATH setup. By default, rpath everything.
 option(CMAKE_INSTALL_RPATH_USE_LINK_PATH "Add rpath for all dependencies" TRUE)
 
-option(WITH_TOOLS "Build Caliper tools" TRUE)
-option(WITH_TESTS "Build Caliper test apps" TRUE)
-option(WITH_DOCS  "Build Caliper documentation" FALSE)
-option(WITH_CUDA  "Enable Caliper CUDA services" FALSE)
+option(WITH_TOOLS     "Build Caliper tools" TRUE)
+option(WITH_TEST_APPS "Build Caliper test apps" TRUE)
+option(WITH_DOCS      "Build Caliper documentation" FALSE)
+option(WITH_CUDA      "Enable Caliper CUDA services" FALSE)
+
+# configure testing explicitly rather than with include(CTest) - avoids some clutter
+option(BUILD_TESTING  "Build continuous integration app and unit tests" ON)
+
+if (BUILD_TESTING)
+  enable_testing()
+endif()
 
 if(WITH_CUDA)
    find_package(CUDA)
@@ -104,7 +111,7 @@ endif()
 
 # Find Python
 
-find_package(PythonLibs)
+find_package(PythonInterp REQUIRED)
 
 # Sampler is currently Linux-specific: check for Linux
 if (${CMAKE_SYSTEM_NAME} MATCHES Linux)
@@ -148,7 +155,7 @@ install(FILES ${PROJECT_BINARY_DIR}/caliper.pc
 
 add_subdirectory(src)
 
-if (WITH_TESTS)
+if (WITH_TEST_APPS)
   add_subdirectory(test)
 endif()
 if (WITH_DOCS)

src/services/CMakeLists.txt

@@ -85,8 +85,6 @@ configure_file(
    ${CMAKE_CURRENT_BINARY_DIR}/gen_services_inc.py
 )
 
-find_package(PythonInterp REQUIRED)
-
 add_custom_target(services.inc.cpp
   COMMAND ${PYTHON_EXECUTABLE} gen_services_inc.py
   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}

test/CMakeLists.txt

@@ -26,4 +26,7 @@ target_link_libraries(cali-test-c caliper)
 
 target_link_libraries(cali-simplereader-test caliper-reader)
 
-add_subdirectory(apptests)
+if (BUILD_TESTING)
+  add_subdirectory(ci_app_tests)
+endif()
+

test/ci_tests/CMakeLists.txt test/ci_app_tests/CMakeLists.txt

@@ -5,7 +5,8 @@ include_directories("../../src/caliper")
 set(CALIPER_CI_CXX_TEST_APPS
   ci_test_basic)
 set(CALIPER_CI_C_TEST_APPS
-  ci_test_ann_c)
+  ci_test_c_ann
+  ci_test_c_snapshot)
 
 foreach(app ${CALIPER_CI_CXX_TEST_APPS})
   add_executable(${app} ${app}.cpp)
@@ -21,6 +22,7 @@ endforeach()
 # add symlinks to the python test driver scripts
 set(PYTHON_SCRIPTS
   test_basictrace.py
+  test_c_api.py
   calipertest.py)
 
 foreach(file ${PYTHON_SCRIPTS})
@@ -29,3 +31,5 @@ foreach(file ${PYTHON_SCRIPTS})
       ${CMAKE_CURRENT_SOURCE_DIR}/${file} 
       ${CMAKE_CURRENT_BINARY_DIR}/${file})
 endforeach()
+
+add_test(NAME CI_app_tests COMMAND ${PYTHON_EXECUTABLE} -m unittest discover -p "test_*.py")

test/ci_tests/README.md test/ci_app_tests/README.md

@@ -2,4 +2,4 @@ Automated continuous integration application tests for Caliper
 ========================
 
 This directory contains driver scripts and test cases for automated
-full application/workflow tests for continuous integration.
+continuous integration full application/workflow tests.

test/ci_tests/calipertest.py test/ci_app_tests/calipertest.py

@@ -0,0 +1,34 @@
+// Caliper continuous integration test app for C snapshot interface
+
+#include <cali.h>
+
+#include <string.h>
+
+int main()
+{
+  cali_begin_string_byname("ci_test_c", "snapshot");
+
+  /* Test w/o event trigger info */ 
+  cali_push_snapshot(CALI_SCOPE_PROCESS | CALI_SCOPE_THREAD,
+                     0, NULL, NULL, NULL);
+  
+  cali_id_t event_str_attr =
+    cali_create_attribute("string_arg", CALI_TYPE_STRING, CALI_ATTR_DEFAULT);
+  cali_id_t event_val_attr =
+    cali_create_attribute("int_arg",    CALI_TYPE_INT,    CALI_ATTR_ASVALUE);
+
+  int64_t     event_val = 42;
+  const char* event_str = "teststring";
+  
+  cali_id_t   event_attr[2] = { event_str_attr,
+                                event_val_attr  };
+  const void* event_data[2] = { event_str,
+                                &event_val      };
+  size_t      event_size[2] = { strlen(event_str),
+                                sizeof(int64_t) };
+
+  cali_push_snapshot(CALI_SCOPE_PROCESS | CALI_SCOPE_THREAD,
+                     2, event_attr, event_data, event_size);
+
+  cali_end_byname("ci_test_c");
+}

test/ci_app_tests/calipertest.pyc

@@ -1,4 +1,4 @@
-# Basic smoke tests using the cali-basic example app
+# Basic smoke tests: create and read a simple trace 
 
 import unittest
 

test/ci_tests/ci_test_basic.cpp test/ci_app_tests/ci_test_basic.cpp

@@ -0,0 +1,50 @@
+# Tests of the C API
+
+import unittest
+
+import calipertest as calitest
+
+class CaliperCAPITest(unittest.TestCase):
+    """ Caliper C API test cases """
+
+    def test_c_ann_trace(self):
+        target_cmd = [ './ci_test_c_ann' ]
+        query_cmd  = [ '../../src/tools/cali-query/cali-query', '-e' ]
+
+        caliper_config = {
+            'CALI_CONFIG_PROFILE'    : 'serial-trace',
+            'CALI_RECORDER_FILENAME' : 'stdout',
+            'CALI_LOG_VERBOSITY'     : '0'
+        }
+
+        query_output = calitest.run_test_with_query(target_cmd, query_cmd, caliper_config)
+        snapshots = calitest.get_snapshots_from_text(query_output)
+
+        self.assertTrue(len(snapshots) >= 10)
+
+        self.assertTrue(calitest.has_snapshot_with_keys(
+            snapshots, {'iteration', 'phase', 'time.inclusive.duration'}))
+        self.assertTrue(calitest.has_snapshot_with_attributes(
+            snapshots, {'event.end#phase': 'initialization', 'phase': 'initialization'}))
+        self.assertTrue(calitest.has_snapshot_with_attributes(
+            snapshots, {'event.end#iteration': 3, 'iteration': 3, 'phase': 'loop'}))
+
+    def test_c_ann_snapshot(self):
+        target_cmd = [ './ci_test_c_snapshot' ]
+        query_cmd  = [ '../../src/tools/cali-query/cali-query', '-e' ]
+
+        caliper_config = {
+            'CALI_CONFIG_PROFILE'    : 'serial-trace',
+            'CALI_RECORDER_FILENAME' : 'stdout',
+            'CALI_LOG_VERBOSITY'     : '0'
+        }
+
+        query_output = calitest.run_test_with_query(target_cmd, query_cmd, caliper_config)
+        snapshots = calitest.get_snapshots_from_text(query_output)
+
+        self.assertTrue(len(snapshots) >= 4)
+        self.assertTrue(calitest.has_snapshot_with_attributes(
+            snapshots, {'ci_test_c': 'snapshot', 'string_arg': 'teststring', 'int_arg': 42 }))
+
+if __name__ == "__main__":
+    unittest.main()