PLUME-67: multiple hook points

README.md

@@ -27,6 +27,30 @@ plume features 3 major components:
 
 plume offers API to this mechanism, available in multiple languages (currently C, C++ and Fortran)
 
+### Hook points
+
+A model can call plume from several points within a single time step. It registers those points by
+name during negotiation (`offerHook()` on the protocol it already builds), and invokes one of them
+with `Manager::run("<name>")`. A plugin says where it wants to run either in its `negotiate()`
+(`requireHook()`) or through an optional `hooks:` list in its entry of the plume configuration,
+which *replaces* what the plugin declared and so lets a deployment re-target a plugin without
+recompiling it. The match is resolved during negotiation: a plugin asking for a hook point the
+model did not register is rejected. During its run a plugin can ask which hook point it is being
+called from, and after negotiation the model can ask which parameters a given hook point consumes
+(`getActiveParamsAtHook()`, `isParamRequestedAtHook()`) so that it can refresh exactly that data.
+
+plume defines an implicit `"default"` hook point that is always registered. A plugin that declares
+no hook point is bound to it, and the argument-less `Manager::run()` targets it — so models and
+plugins that know nothing about hook points behave exactly as before.
+
+> **When adopting hook points, keep calling the argument-less `Manager::run()`** where your single
+> run call used to be. A model that only invokes its own named hook points leaves every plugin
+> that declares no hook point — that is, every plugin written before this feature — silently
+> dormant. The hook point summary that plume logs at the end of negotiation shows which plugins
+> ended up bound to which hook point.
+
+See `examples/example3.{cc,F90}` with `examples/plume_config_hooks.yml` for a worked example.
+
 ### Requirements
 Build dependencies:
 

cmake/fortran_plugin.F90.in

@@ -14,6 +14,7 @@ use iso_c_binding, only : c_ptr, c_null_char, c_loc
 use fckit_configuration_module, only : fckit_configuration
 use plume_module, only : plume_check
 use plume_data_module, only : plume_data
+use plume_utils_module, only : fortranise_cstr
 
 ! this module contains the user-methods
 ! that will be executed by the plugin
@@ -47,13 +48,21 @@ bind(C, name="plugincore_setup__@PLUGIN_NAME@__@PLUGINCORE_NAME@")
     call plugincore_setup__@PLUGIN_NAME@(@PLUGIN_NAME@_configuration, @PLUGIN_NAME@_data)
 end subroutine
 
-subroutine internal_plugincore_run__@PLUGINCORE_NAME@() &
+subroutine internal_plugincore_run__@PLUGINCORE_NAME@(conf_cptr, data_cptr, hook_cptr) &
 bind(C, name="plugincore_run__@PLUGIN_NAME@__@PLUGINCORE_NAME@")
-    call plugincore_run__@PLUGIN_NAME@(@PLUGIN_NAME@_configuration, @PLUGIN_NAME@_data)
+    type(c_ptr), intent(in), value :: conf_cptr
+    type(c_ptr), intent(in), value :: data_cptr
+    type(c_ptr), intent(in), value :: hook_cptr
+
+    ! the hook point name is only forwarded to plugins that declare PLUGIN_REQUIRED_HOOKS
+    @PLUGINCORE_RUN_FORWARD@
 end subroutine
 
-subroutine internal_plugincore_teardown__@PLUGINCORE_NAME@() &
+subroutine internal_plugincore_teardown__@PLUGINCORE_NAME@(conf_cptr, data_cptr) &
 bind(C, name="plugincore_teardown__@PLUGIN_NAME@__@PLUGINCORE_NAME@")
+    type(c_ptr), intent(in), value :: conf_cptr
+    type(c_ptr), intent(in), value :: data_cptr
+
     call plugincore_teardown__@PLUGIN_NAME@(@PLUGIN_NAME@_configuration, @PLUGIN_NAME@_data)
 
     ! finalise the data structure

cmake/fortran_plugin.h.in

@@ -20,7 +20,7 @@
 
 // plugincore
 extern "C" void plugincore_setup__@PLUGIN_NAME@__@PLUGINCORE_NAME@(void* config, void* modelData);
-extern "C" void plugincore_run__@PLUGIN_NAME@__@PLUGINCORE_NAME@(void* config, void* modelData);
+extern "C" void plugincore_run__@PLUGIN_NAME@__@PLUGINCORE_NAME@(void* config, void* modelData, const char* hook);
 extern "C" void plugincore_teardown__@PLUGIN_NAME@__@PLUGINCORE_NAME@(void* config, void* modelData);
 
 
@@ -35,8 +35,8 @@ public:
         plugincore_setup__@PLUGIN_NAME@__@PLUGINCORE_NAME@(&config_, &modelData());
     }
 
-    void run() override { 
-        plugincore_run__@PLUGIN_NAME@__@PLUGINCORE_NAME@(&config_, &modelData());
+    void run() override {
+        plugincore_run__@PLUGIN_NAME@__@PLUGINCORE_NAME@(&config_, &modelData(), currentHook().c_str());
     }
 
     virtual void teardown() override {
@@ -67,6 +67,7 @@ public:
     plume::Protocol negotiate() override {
         plume::Protocol protocol;
         @REQUIRED_PARAM_LIST@
+        @REQUIRED_HOOK_LIST@
         return protocol;
     }
 

cmake/plume-plugin-interface.cmake

@@ -30,6 +30,7 @@ function( plume_plugin_interface )
 
     set(multi_value_args
         PLUGIN_REQUIRED_PARAMS
+        PLUGIN_REQUIRED_HOOKS
     )
 
     cmake_parse_arguments( _PAR "${options}" "${single_value_args}" "${multi_value_args}"  ${_FIRST_ARG} ${ARGN} )
@@ -43,6 +44,7 @@ function( plume_plugin_interface )
     set(PLUGIN_SHA ${_PAR_PLUGIN_SHA})
     set(PLUGINCORE_NAME ${_PAR_PLUGINCORE_NAME})
     set(PLUGIN_REQUIRED_PARAMS ${_PAR_PLUGIN_REQUIRED_PARAMS})
+    set(PLUGIN_REQUIRED_HOOKS ${_PAR_PLUGIN_REQUIRED_HOOKS})
     set(PLUGIN_PRECISION ${_PAR_PLUGIN_PRECISION})
 
     # list of required parameters
@@ -65,6 +67,22 @@ function( plume_plugin_interface )
         endif()
     endforeach()
 
+    # list of required hook points (the plugin runs at the default hook point if none are given)
+    set(REQUIRED_HOOK_LIST "")
+    foreach(hook ${PLUGIN_REQUIRED_HOOKS})
+        message("required hook point ${hook}")
+        set(REQUIRED_HOOK_LIST "${REQUIRED_HOOK_LIST} protocol.requireHook(\"${hook}\");\n")
+    endforeach()
+
+    # A plugin that declares hook points gets the current hook point name forwarded to its "run"
+    # subroutine. A plugin that does not keeps the original 2-argument signature, so existing
+    # Fortran plugins do not need any change.
+    if(PLUGIN_REQUIRED_HOOKS)
+        set(PLUGINCORE_RUN_FORWARD "call plugincore_run__${PLUGIN_NAME}(${PLUGIN_NAME}_configuration, ${PLUGIN_NAME}_data, fortranise_cstr(hook_cptr))")
+    else()
+        set(PLUGINCORE_RUN_FORWARD "call plugincore_run__${PLUGIN_NAME}(${PLUGIN_NAME}_configuration, ${PLUGIN_NAME}_data)")
+    endif()
+
     get_filename_component( PLUGIN_TEMPLATE_FILENAME ${_PAR_PLUGIN_TEMPLATE} NAME)
     get_filename_component( GENERATED_USER_SOURCE_FILE ${_PAR_PLUGIN_TEMPLATE} NAME_WLE)
     get_filename_component( GENERATED_USER_SOURCE_FILE_WEXT ${GENERATED_USER_SOURCE_FILE} NAME_WLE)

examples/example3.F90

@@ -50,6 +50,11 @@ program my_program
 call plume_check(offers%offer_double("config-param-2", "always", "this is param config-param-2"))
 call plume_check(offers%offer_atlas_field("config-param-3", "always", "this is param config-param-3"))
 
+! Register the hook points that this model is able to call Plume from. The "default" hook point is
+! always registered, and is the one that the argument-less manager%run() targets.
+call plume_check(offers%offer_hook("pre-compute", "before the step is computed"))
+call plume_check(offers%offer_hook("post-compute", "after the step has been computed"))
+
 
 ! negotiate
 call plume_check(manager%initialise())
@@ -80,13 +85,20 @@ program my_program
 ! Run the model for 10 iterations
 do iter=1,10
 
+  ! first hook point of the step
+  call plume_check(manager%run("pre-compute"))
+
   ! update the internal parameters
   call plume_check(data%update_int("I", 0+iter) )
   call plume_check(data%update_int("J", 10+iter) )
   call plume_check(data%update_int("K", 100+iter) )
 
-  ! run the model..
+  ! The default hook point: this is where the single manager%run() call has always been, and it
+  ! is where plugins that declare no hook point run.
   call plume_check(manager%run())
+
+  ! second hook point of the step
+  call plume_check(manager%run("post-compute"))
 enddo
 
 ! finalise

examples/example3.cc

@@ -52,9 +52,26 @@ int main(int argc, char** argv) {
     offers.offer<double>("config-param-2", "on-request", "this is param config-param-2");
     offers.offer<atlas::Field>("config-param-3", "on-request", "this is param config-param-3");
 
+    // Register the hook points that this model is able to call Plume from. The "default" hook
+    // point is always registered, and is the one that the argument-less Manager::run() targets.
+    offers.offerHook("pre-compute", "before the step is computed");
+    offers.offerHook("post-compute", "after the step has been computed");
+
     // Negotiate with plugins
     plume::Manager::negotiate(offers);
 
+    // After negotiation Plume knows which params each hook point actually consumes, so the model
+    // can refresh exactly what is needed before invoking it.
+    for (const auto& hook : plume::Manager::registeredHooks()) {
+        std::cout << "Hook point '" << hook << "' is "
+                  << (plume::Manager::isHookActive(hook) ? "active" : "not used by any plugin")
+                  << ", params: [";
+        for (const auto& param : plume::Manager::getActiveParamsAtHook(hook, false)) {
+            std::cout << param << " ";
+        }
+        std::cout << "]" << std::endl;
+    }
+
     // data
     atlas::Field field = createAtlasField();
 
@@ -79,15 +96,22 @@ int main(int argc, char** argv) {
     plume::Manager::feedPlugins(data);
 
     // Run the model for 10 iterations
-    for (int i=0; i<10; i++){        
+    for (int i=0; i<10; i++){
+
+        // first hook point of the step
+        plume::Manager::run("pre-compute");
 
         // Update the values
         data.updateParam("I", i);
         data.updateParam("J", 10+i);
         data.updateParam("K", 100+i);
 
-        // run
+        // The default hook point: this is where the single Manager::run() call has always been,
+        // and it is where plugins that declare no hook point run.
         plume::Manager::run();
+
+        // second hook point of the step
+        plume::Manager::run("post-compute");
     }
 
     // Teardown as necessary

examples/plugin_bar.cc

@@ -31,6 +31,7 @@ static plume::PluginCoreBuilder<PluginCoreBar> runnable_plugincore_BarBuilder_;
 PluginCoreBar::PluginCoreBar(const eckit::Configuration& conf) : PluginCore(conf) {}
 
 void PluginCoreBar::run() {
+    eckit::Log::info() << "Plugin Bar running at hook point: " << currentHook() << std::endl;
 
     eckit::Log::info() << "Plugin Bar running..." << std::endl;
 

examples/plugin_bar.h

@@ -43,6 +43,10 @@ class PluginBar final : public plume::Plugin {
         plume::Protocol protocol;
         protocol.require<int>("K");
         protocol.require<atlas::Field>("field_dummy_1");
+
+        // No requireHook() call here, so this plugin is bound to the implicit "default" hook
+        // point and runs wherever the model calls Manager::run(). See plume_config_hooks.yml for
+        // how a deployment can re-target it onto another hook point without recompiling it.
         return protocol;
     }
 

examples/plume_config.yml

@@ -1,3 +1,6 @@
+# Neither plugin declares a hook point, so both are bound to the implicit "default" hook point
+# and run wherever the model calls plume::Manager::run().
+# See plume_config_hooks.yml for a configuration that re-targets a plugin onto a named hook point.
 verbose: true
 plugins:
   - name: PluginFoo

examples/plume_config_hooks.yml

@@ -0,0 +1,37 @@
+# Configuration used by example 3, which registers two hook points ("pre-compute" and
+# "post-compute") on top of the implicit "default" one.
+#
+# Neither PluginFoo nor PluginBar declares a hook point in its negotiate(). PluginFoo therefore
+# stays on the default hook point, while the "hooks" key below re-targets PluginBar onto
+# "post-compute" - without recompiling the plugin.
+#
+# Note that a "hooks" key *replaces* whatever the plugin declared, and that a hook point the model
+# does not register causes the plugin to be rejected during negotiation.
+verbose: true
+plugins:
+  - name: PluginFoo
+    lib: plugin_foo
+  - name: PluginBar
+    lib: plugin_bar
+    hooks: [post-compute]
+    parameters:
+      -
+        - name: config-param-1
+          type: int
+        - name: config-param-2
+          type: double
+        - name: config-param-3
+          type: atlas_field
+        - name: config-param-4
+          type: atlas_field
+        - name: config-param-5
+          type: atlas_field
+      -
+        - name: config-param-1
+          type: int
+        - name: config-param-3
+          type: int
+    core-config:
+      exceptions-dump-trace: true
+      key-2: val-2
+      key-3: val-3

examples/run_examples.sh.in

@@ -2,6 +2,7 @@
 
 exe_dir=@CMAKE_BINARY_DIR@/bin
 plume_config=@CMAKE_CURRENT_SOURCE_DIR@/plume_config.yml
+plume_config_hooks=@CMAKE_CURRENT_SOURCE_DIR@/plume_config_hooks.yml
 
 # ===================== Example 1 =====================
 # example 1: demonstrate how to configure plume data
@@ -33,10 +34,12 @@ $exe_dir/plume_example2_fort.x $plume_config
 
 # ===================== Example 3 =====================
 # example 3: Similar to example 1, but this time some parameters
-# are "created" and then updated by the model during the run
+# are "created" and then updated by the model during the run.
+# It also registers two hook points on top of the implicit "default" one, and uses a
+# configuration that binds one of the plugins to one of them.
 
 # Example C++
-$exe_dir/plume_example3_cpp.x $plume_config
+$exe_dir/plume_example3_cpp.x $plume_config_hooks
 
 # Example Fortran
-$exe_dir/plume_example3_fort.x $plume_config
+$exe_dir/plume_example3_fort.x $plume_config_hooks

src/nwp_emulator/nwp_emulator_core.cc

@@ -229,6 +229,12 @@ bool NWPEmulatorCore::setupPlume(NWPDataProvider& dataProvider) {
     for (const auto& field: fields) {
         offers.offer<atlas::Field>(field.name(), "on-request", field.name());
     }
+
+    // Hook points the emulator is able to call Plume from, in addition to the implicit default
+    // one that it keeps calling where it always has.
+    offers.offerHook("step-begin", "Beginning of a model step, before the step data is marked as updated");
+    offers.offerHook("step-end", "End of a model step, after all plugins bound to the default hook point");
+
     plume::Manager::negotiate(offers);
 
     // Scalar parameters are initialised once before the first plugin step.
@@ -250,13 +256,20 @@ bool NWPEmulatorCore::setupPlume(NWPDataProvider& dataProvider) {
 }
 
 void NWPEmulatorCore::runPlume(int step) {
+    // Plugins bound to "step-begin" see the previous step's values: the step metadata has not
+    // been updated yet.
+    plume::Manager::run("step-begin");
+
     // Update step metadata and mark backing fields as updated only after the
     // provider has populated this step's data.
     plumeData_.updateParam("NSTEP", step);
     plumeData_.updateParam("WSTEP", std::ceil(step * plumeData_.getParam<double>("TSTEP")));
     plumeData_.setUpdated(plumeUpdatingParams_);
 
+    // The default hook point, kept exactly where the single run call has always been.
     plume::Manager::run();
+
+    plume::Manager::run("step-end");
 }
 
 }  // namespace nwp_emulator

src/plume/CMakeLists.txt

@@ -21,13 +21,14 @@ install(FILES
 
 
 # #################### Plume plugin ######################
-set(PLUGIN_FILES_H    
+set(PLUGIN_FILES_H
     Plugin.h
     PluginDecision.h
     PluginHandler.h
     Protocol.h
     PluginCore.h
     Configurable.h
+    Hook.h
     data/ModelData.h
     data/ParameterCatalogue.h
     data/ParameterType.h
@@ -82,6 +83,7 @@ set(PLUME_PLUGIN_MANAGER_SOURCES
     Protocol.cc
     Configurable.h
     Configurable.cc
+    Hook.h
     PluginConfig.h
     data/ModelData.h
     data/ModelData.cc

src/plume/Hook.h

@@ -0,0 +1,30 @@
+/*
+ * (C) Copyright 2023- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+#pragma once
+
+
+namespace plume {
+
+/**
+ * @brief The implicit hook point.
+ *
+ * It is always offered by every model, whether or not the model registers hook points of its own.
+ * A plugin that requires no hook point is bound to it, and the hook-less Manager::run() targets
+ * it. This is what makes hook-unaware models and plugins behave exactly as they did before hook
+ * points existed.
+ *
+ * NOTE for models adopting hook points: keep invoking Manager::run() (or Manager::run(DEFAULT_HOOK))
+ * at the point where the single run call used to be. A model that only invokes its own named hook
+ * points leaves every plugin that declares no hook point silently dormant.
+ */
+constexpr const char* DEFAULT_HOOK = "default";
+
+}  // namespace plume