Fix tau service compilation, move it to mpi dir (LLNL#178)

Author: daboehme

CMakeLists.txt

@@ -109,18 +109,6 @@ if(WITH_CUPTI)
   endif()
 endif()
 
-if(WITH_TAU)
-  find_package(TAU QUIET)
-  if (TAU_FOUND)
-    set(CALIPER_HAVE_TAU TRUE)
-    set(CALIPER_TAU_CMAKE_MSG "Yes, using ${TAU_LIBRARIES}")
-    list(APPEND CALIPER_EXTERNAL_LIBS ${TAU_LIBRARIES})
-  else()
-    message(WARNING "TAU bindings requested but TAU API was not found!\n"
-  "Set TAU_PREFIX to installation path and re-run cmake.")
-  endif()
-endif()
-
 # Find PAPI
 if (WITH_PAPI)
   include(FindPAPI)
@@ -262,6 +250,23 @@ if (WITH_MPI)
   endif()
 endif()
 
+
+if(WITH_TAU)
+  if (CALIPER_HAVE_MPI)
+    find_package(TAU QUIET)
+    if (TAU_FOUND)
+      set(CALIPER_HAVE_TAU TRUE)
+      set(CALIPER_TAU_CMAKE_MSG "Yes, using ${TAU_LIBRARIES}")
+      list(APPEND CALIPER_MPI_EXTERNAL_LIBS ${TAU_LIBRARIES})
+    else()
+      message(WARNING "TAU bindings requested but TAU API was not found!\n"
+        "Set TAU_PREFIX to installation path and re-run cmake.")
+    endif()
+  else()
+    message(WARNING "TAU support requires MPI-enabled build!\n")
+  endif()
+endif()
+
 # Find Python
 
 find_package(PythonInterp REQUIRED)

mpi/mpi-rt/CMakeLists.txt

@@ -30,6 +30,10 @@ endmacro()
 add_subdirectory(services/mpiwrap)
 add_subdirectory(services/mpireport)
 
+if (CALIPER_HAVE_TAU)
+  add_subdirectory(services/tau)
+endif()
+
 add_subdirectory(controllers)
 
 if (CALIPER_HAVE_MPIT)
@@ -45,6 +49,10 @@ add_library(caliper-mpi
 set_target_properties(caliper-mpi PROPERTIES SOVERSION ${CALIPER_MAJOR_VERSION})
 set_target_properties(caliper-mpi PROPERTIES VERSION ${CALIPER_VERSION})
 
+foreach (_extlib ${CALIPER_MPI_EXTERNAL_LIBS})
+  target_link_libraries(caliper-mpi PRIVATE ${_extlib})
+endforeach()
+
 target_link_libraries(caliper-mpi PUBLIC caliper-mpi-common caliper)
 target_link_libraries(caliper-mpi PRIVATE ${MPI_CXX_LIBRARIES})
 

src/services/tau/CMakeLists.txt mpi/mpi-rt/services/tau/CMakeLists.txt

@@ -5,5 +5,5 @@ set(CALIPER_TAU_SOURCES
 
 add_library(caliper-tau OBJECT ${CALIPER_TAU_SOURCES})
 
-add_service_objlib("caliper-tau")
-add_caliper_service("tau CALIPER_HAVE_TAU")
+add_mpi_service_objlib("caliper-tau")
+

src/services/tau/README.md mpi/mpi-rt/services/tau/README.md

@@ -44,6 +44,7 @@
 #define TAU_DOT_H_LESS_HEADERS
 #include "TAU.h"
 
+#include <mpi.h>
 
 using namespace cali;
 
@@ -62,19 +63,22 @@ class TAUBinding : public cali::AnnotationBinding
         char* argv[1];
         argv[0] = const_cast<char*>(dummy);
         Tau_init(argc,argv);
-        // want to get the *real* MPI rank.  How do I get it? like this.
-        Attribute mpi_rank_attr = c->get_attribute("mpi.rank");
-        if (mpi_rank_attr == Attribute::invalid) {
-            // no MPI
+
+        int flag = 0;        
+        PMPI_Initialized(&flag);
+
+        if (flag == 0) {
             Tau_set_node(0);
         } else {
-            // have MPI, get the rank
-            Tau_set_node(c->get(chn, mpi_rank_attr).value().to_int());
+            int rank = 0;
+            PMPI_Comm_rank(MPI_COMM_WORLD, &rank);
+            
+            Tau_set_node(rank);
         }
         // register for events of interest?
     }
 
-    void finalize(Caliper* c) {
+    void finalize(Caliper*, Channel*) {
         // do something?
     }
 
@@ -92,7 +96,7 @@ class TAUBinding : public cali::AnnotationBinding
     // handle an end event by stopping a TAU timer
     void on_end(Caliper* c, Channel*, const Attribute& attr, const Variant& value) {
         if (attr.type() == CALI_TYPE_STRING) {
-            Tau_stop((const char*)(value.data());
+            Tau_stop((const char*) value.data());
         } else {
             Tau_stop((const char*)(value.to_string().data()));
         }
@@ -101,7 +105,10 @@ class TAUBinding : public cali::AnnotationBinding
 
 } // namespace
 
-namespace cali {
-    CaliperService tau_service { "tau", &AnnotationBinding::make_binding<::TAUBinding> };
+namespace cali
+{
+
+CaliperService tau_service { "tau", &AnnotationBinding::make_binding<::TAUBinding> };
+
 }
 

src/services/tau/tau.cpp mpi/mpi-rt/services/tau/tau.cpp

@@ -54,12 +54,18 @@ extern CaliperService mpireport_service;
 #ifdef CALIPER_HAVE_MPIT
 extern CaliperService mpit_service;
 #endif
+#ifdef CALIPER_HAVE_TAU
+extern CaliperService tau_service;
+#endif
 
 CaliperService cali_mpi_services[] = {
     mpiwrap_service,
     mpireport_service,
 #ifdef CALIPER_HAVE_MPIT
     mpit_service,
+#endif
+#ifdef CALIPER_HAVE_TAU
+    tau_service,
 #endif
     { nullptr, nullptr }
 };

mpi/mpi-rt/setup_mpi.cpp

@@ -78,9 +78,6 @@ if (CALIPER_HAVE_DYNINST)
   add_subdirectory(symbollookup)
   add_subdirectory(instlookup)
 endif()
-if (CALIPER_HAVE_TAU)
-  add_subdirectory(tau)
-endif()
 
 add_subdirectory(trace)
 # if (CALIPER_HAVE_OMPT)

src/services/CMakeLists.txt

@@ -74,12 +74,6 @@ if (CALIPER_HAVE_SAMPLER AND CALIPER_HAVE_DYNINST)
   list(APPEND PYTHON_SCRIPTS test_sampler.py)
 endif()
 
-if (CALIPER_HAVE_TAU)
-  add_executable(ci_test_tau_service ci_test_tau_service.cpp)
-  target_link_libraries(ci_test_tau_service caliper ${TAU_LIBRARIES})
-  list(APPEND PYTHON_SCRIPTS test_tau_service.py)
-endif()
-
 if (CALIPER_HAVE_MPI)
   include_directories(${MPI_CXX_INCLUDE_PATH})
   include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../mpi/include)
@@ -91,7 +85,11 @@ if (CALIPER_HAVE_MPI)
 
   if (RUN_MPI_TESTS)
     list(APPEND PYTHON_SCRIPTS test_mpi.py)
-  endif()
+
+    if (CALIPER_HAVE_TAU)
+      list(APPEND PYTHON_SCRIPTS test_tau_service.py)
+    endif()
+  endif()  
 endif()
 
 foreach(file ${PYTHON_SCRIPTS})

test/ci_app_tests/CMakeLists.txt

@@ -8,14 +8,13 @@ class CaliperReportTest(unittest.TestCase):
     """ Caliper TAU service test case """
 
     def test_tau(self):
-        target_cmd = [ './ci_test_tau_service' ]
-        query_cmd  = [ '../../src/tools/cali-query/cali-query',
-                       '-q', 'SELECT count(),iteration#fooloop WHERE loop=fooloop GROUP BY iteration#fooloop FORMAT expand' ]
+        target_cmd = [ './ci_test_mpi_before_cali' ]
 
         caliper_config = {
-            'CALI_SERVICES_ENABLE'   : 'event,trace,report,tau',
-            'CALI_REPORT_CONFIG'     : 'format cali',
-            'CALI_LOG_VERBOSITY'     : '0'
+            'CALI_SERVICES_ENABLE'   : 'tau',
+            'CALI_LOG_VERBOSITY'     : '0',
+            'TAU_PROFILE'            : '0',
+            'TAU_TRACE'              : '0'
         }
 
         #query_output = cat.run_test_with_query(target_cmd, query_cmd, caliper_config)