Couple to Kynema for ALM and templatize existing OpenFAST interface

.codespellrc

@@ -1,3 +1,3 @@
 [codespell]
-skip = .git,*.ipynb,*.bib,*.ps,*.patch,spack-build-*,*/build,*/submods,plt*,chk*,__pycache__,.ccls,.ccls-cache,*.pdf,DU21_A17.txt,inspect_abl_io.html,*.dat
+skip = .git,*.ipynb,*.bib,*.ps,*.patch,spack-build-*,*/build,*/submods,plt*,chk*,__pycache__,.ccls,.ccls-cache,*.pdf,DU21_A17.txt,inspect_abl_io.html,*.dat,*.yaml
 ignore-words = .codespell-ignore-words

CMakeLists.txt

@@ -45,6 +45,7 @@ option(AMR_WIND_ENABLE_MASA "Enable MASA library" OFF)
 option(AMR_WIND_ENABLE_HELICS "Enable HELICS library" OFF)
 option(AMR_WIND_ENABLE_HYPRE "Enable HYPRE integration" OFF)
 option(AMR_WIND_ENABLE_OPENFAST "Enable OpenFAST integration" OFF)
+option(AMR_WIND_ENABLE_KYNEMA "Enable Kynema integration" OFF)
 option(AMR_WIND_ENABLE_ASCENT "Enable Ascent visualization library" OFF)
 option(AMR_WIND_ENABLE_UMPIRE "Enable Umpire GPU memory pools" OFF)
 option(AMR_WIND_ENABLE_W2A "Enable Waves2AMR library" OFF)
@@ -197,6 +198,13 @@ if(AMR_WIND_ENABLE_OPENFAST)
   target_compile_definitions(${amr_wind_lib_name} PUBLIC OPENFAST_VERSION_MINOR=${OPENFAST_VERSION_MINOR})
 endif()
 
+if(AMR_WIND_ENABLE_KYNEMA)
+  set(CMAKE_PREFIX_PATH ${KYNEMA_DIR} ${CMAKE_PREFIX_PATH})
+  find_package(Kynema REQUIRED)
+  target_link_libraries(${amr_wind_lib_name} PUBLIC Kynema::kynema_library)
+  target_compile_definitions(${amr_wind_lib_name} PUBLIC AMR_WIND_USE_KYNEMA)
+endif()
+
 if(AMR_WIND_ENABLE_ASCENT)
   set(CMAKE_PREFIX_PATH ${ASCENT_DIR} ${CMAKE_PREFIX_PATH})
   find_package(Ascent REQUIRED)

amr-wind/main.cpp

@@ -2,13 +2,20 @@
 #include "amr-wind/utilities/console_io.H"
 
 #include "AMReX_FileSystem.H"
+#ifdef AMR_WIND_USE_KYNEMA
+#include <Kokkos_Core.hpp>
+#endif
 
 int main(int argc, char* argv[])
 {
 #ifdef AMREX_USE_MPI
     MPI_Init(&argc, &argv);
 #endif
 
+#ifdef AMR_WIND_USE_KYNEMA
+    Kokkos::initialize(argc, argv);
+#endif
+
     using namespace amrex::mpidatatypes;
 
     if (argc < 2) {

amr-wind/utilities/sampling/DTUSpinnerSampler.cpp

@@ -237,7 +237,7 @@ void DTUSpinnerSampler::get_turbine_data(const std::string& turbine_label)
 
     // Read and broadcast data
     if (testline) {
-        const auto& actdata = actline->meta().fast_data;
+        const auto& actdata = actline->meta().ext_data;
         const auto& info = actline->info();
 
         // Create buffer object
@@ -255,7 +255,7 @@ void DTUSpinnerSampler::get_turbine_data(const std::string& turbine_label)
 
         bcast_turbine(turbine_pack, info.root_proc);
     } else if (testdisk) {
-        const auto& actdata = actdisk->meta().fast_data;
+        const auto& actdata = actdisk->meta().ext_data;
         const auto& info = actdisk->info();
 
         // Create buffer object

amr-wind/wind_energy/actuator/turbine/CMakeLists.txt

@@ -3,3 +3,4 @@ target_sources(${amr_wind_lib_name} PRIVATE
   )
 
 add_subdirectory(fast)
+add_subdirectory(kynema)

amr-wind/wind_energy/actuator/turbine/external/ExtTurbIface.H

@@ -0,0 +1,177 @@
+#ifndef EXTTURBIFACE_H
+#define EXTTURBIFACE_H
+
+#include "amr-wind/core/ExtSolver.H"
+#include "amr-wind/wind_energy/actuator/turbine/external/external_base_types.H"
+#include <map>
+#include <vector>
+
+namespace ncutils {
+class NCFile;
+}
+
+namespace amr_wind {
+class CFDSim;
+}
+
+namespace ext_turb {
+
+template <typename SolverTurbine, typename SolverData>
+class ExtTurbIface
+    : public ::amr_wind::ExtSolver::Register<
+          ExtTurbIface<SolverTurbine, SolverData>>
+{
+public:
+    static std::string identifier() { return ext_id<SolverData>(); }
+
+    explicit ExtTurbIface(const ::amr_wind::CFDSim& sim);
+
+    /* -- General for all external turbines -- */
+    int register_turbine(SolverTurbine& data);
+
+    void init_turbine(const int local_id);
+
+    void advance_turbine(const int local_id);
+
+    int num_local_turbines() const
+    {
+        return static_cast<int>(m_turbine_data.size());
+    }
+
+    /* -- Specific to each type of external turbine -- */
+
+    ~ExtTurbIface() override;
+
+    void parse_inputs(
+        const amr_wind::CFDSim& /*sim*/, const std::string& /*inp_name*/);
+
+    void init_solution(const int local_id);
+
+    void get_hub_stats(const int local_id);
+
+    void do_turbine_step(SolverTurbine& fi);
+
+    void write_turbine_checkpoint(int& tid);
+
+protected:
+    void allocate_ext_turbines();
+
+    void ext_init_turbine(SolverTurbine& /*fi*/);
+
+    void ext_restart_turbine(SolverTurbine& /*fi*/);
+
+    void ext_replay_turbine(SolverTurbine& /*fi*/);
+
+    void prepare_netcdf_file(SolverTurbine& /*unused*/);
+
+    void write_velocity_data(const SolverTurbine& /*unused*/);
+
+    static void read_velocity_data(
+        SolverTurbine& /*unused*/,
+        const ncutils::NCFile& /*unused*/,
+        const size_t tid);
+
+    //! Global to local index lookup map
+    std::map<int, int> m_turbine_map;
+
+    std::vector<SolverTurbine*> m_turbine_data;
+
+    double m_dt_cfd{0.0};
+    double m_start_time{0.0};
+    double m_stop_time{-1.0};
+    ::ext_turb::SimMode m_sim_mode{::ext_turb::SimMode::init};
+
+    SolverData m_solver_data;
+
+    bool m_is_initialized{false};
+};
+
+// General implementations of some functions
+template <typename SolverTurbine, typename SolverData>
+ExtTurbIface<SolverTurbine, SolverData>::ExtTurbIface(
+    const amr_wind::CFDSim& /*unused*/)
+{}
+
+template <typename SolverTurbine, typename SolverData>
+int ExtTurbIface<SolverTurbine, SolverData>::register_turbine(
+    SolverTurbine& data)
+{
+    BL_PROFILE("amr-wind::ExtTurbIface::register_turbine");
+    AMREX_ALWAYS_ASSERT(!m_is_initialized);
+    const int local_id = static_cast<int>(m_turbine_data.size());
+    const int gid = data.tid_global;
+    m_turbine_map[gid] = local_id;
+    data.tid_local = local_id;
+    m_turbine_data.emplace_back(&data);
+
+    return local_id;
+}
+
+template <typename SolverTurbine, typename SolverData>
+void ExtTurbIface<SolverTurbine, SolverData>::init_turbine(const int local_id)
+{
+    AMREX_ALWAYS_ASSERT(local_id < static_cast<int>(m_turbine_data.size()));
+    if (!m_is_initialized) {
+        allocate_ext_turbines();
+    }
+    auto& fi = *m_turbine_data[local_id];
+
+    switch (fi.sim_mode) {
+    case ::ext_turb::SimMode::init: {
+        ext_init_turbine(fi);
+        prepare_netcdf_file(fi);
+        break;
+    }
+
+    case ::ext_turb::SimMode::replay: {
+        ext_init_turbine(fi);
+        ext_replay_turbine(fi);
+        break;
+    }
+
+    case ::ext_turb::SimMode::restart: {
+        ext_restart_turbine(fi);
+        prepare_netcdf_file(fi);
+        break;
+    }
+    }
+}
+
+template <typename SolverTurbine, typename SolverData>
+void ExtTurbIface<SolverTurbine, SolverData>::advance_turbine(
+    const int local_id)
+{
+    BL_PROFILE("amr-wind::ExtTurbIface::advance_turbine");
+    AMREX_ASSERT(local_id < static_cast<int>(m_turbine_data.size()));
+
+    auto& fi = *m_turbine_data[local_id];
+    AMREX_ASSERT(!fi.is_solution0);
+    {
+        // Default is off, unless a turbine model reads, populates stop_time
+        const auto& tmax = fi.stop_time;
+        const auto& telapsed = (fi.time_index + fi.num_substeps) * fi.dt_ext;
+        if (telapsed > (tmax + 1.0e-8) && fi.stop_time > 0.) {
+            // clang-format off
+            amrex::OutStream()
+                << "\nWARNING: ExtTurbIface:\n"
+                << "  Elapsed simulation time will exceed max "
+                << "time set for External Turbine Solver"
+                << std::endl << std::endl;
+            // clang-format on
+        }
+    }
+
+    write_velocity_data(fi);
+    for (int i = 0; i < fi.num_substeps; ++i, ++fi.time_index) {
+        do_turbine_step(fi);
+    }
+
+    if (fi.chkpt_interval > 0 &&
+        (fi.time_index / fi.num_substeps) % fi.chkpt_interval == 0) {
+        write_turbine_checkpoint(fi.tid_local);
+    }
+}
+
+} // namespace ext_turb
+
+#endif /* EXTTURBIFACE_H */

amr-wind/wind_energy/actuator/turbine/external/TurbineExternal.H

@@ -0,0 +1,38 @@
+#ifndef TURBINEEXTERNAL_H
+#define TURBINEEXTERNAL_H
+
+#include "amr-wind/wind_energy/actuator/turbine/turbine_types.H"
+#include "amr-wind/wind_energy/actuator/turbine/external/ExtTurbIface.H"
+#include "amr-wind/core/ExtSolver.H"
+
+namespace amr_wind::actuator {
+
+template <typename SolverTurbine, typename SolverData>
+struct TurbineExternalData : public TurbineBaseData
+{
+    amrex::Real density{1.0};
+
+    SolverTurbine ext_data;
+    ::ext_turb::ExtTurbIface<SolverTurbine, SolverData>* ext_ptr{nullptr};
+
+    MPI_Comm tcomm{MPI_COMM_NULL};
+};
+
+template <typename SolverTurbine, typename SolverData>
+struct TurbineFromExtSolver : public TurbineType
+{
+    using InfoType = TurbineInfo;
+    using GridType = ActGrid;
+    using MetaType = TurbineExternalData<SolverTurbine, SolverData>;
+    using DataType =
+        ActDataHolder<TurbineFromExtSolver<SolverTurbine, SolverData>>;
+
+    static std::string identifier()
+    {
+        return ext_turb::ext_id<SolverTurbine>();
+    }
+};
+
+} // namespace amr_wind::actuator
+
+#endif /* TURBINEEXTERNAL_H */

amr-wind/wind_energy/actuator/turbine/external/external_base_types.H

@@ -0,0 +1,105 @@
+#ifndef EXTERNAL_BASE_TYPES_H
+#define EXTERNAL_BASE_TYPES_H
+
+#include <string>
+
+namespace ext_turb {
+
+enum class SimMode : int {
+    init = 0, ///< Clean start
+    replay,   ///< Replay using velocities stored in file
+    restart   ///< Restart using external checkpoint files
+};
+
+template <typename ExtData>
+std::string ext_id();
+
+/** Representation of a turbine for exchanging data with external solver
+ */
+struct ExternalTurbine
+{
+    static constexpr int ndim = 3;
+
+    //! Unique string identifier for this turbine
+    std::string tlabel;
+
+    //! Local ID for this turbine (provided by external interface)
+    int tid_local;
+
+    //! Global ID for this turbine (set by Turbine instance)
+    int tid_global;
+
+    //! Number of actuator points per blade
+    int num_pts_blade;
+
+    //! Number of actuator points for tower
+    int num_pts_tower;
+
+    //! Position of tower base in global coordinate system
+    amrex::Array<float, ndim> base_pos;
+
+    //! Hub stats for Spinner Lidar
+    amrex::Array<float, 3> hub_abs_pos;
+
+    amrex::Array<float, 3> hub_rot_vel;
+
+    amrex::Array<double, 9> hub_orient;
+
+    SimMode sim_mode{SimMode::init};
+
+    //! External input file
+    std::string input_file;
+
+    //! Checkpoint file name
+    std::string checkpoint_file;
+
+    //! Number of blades
+    int num_blades;
+
+    //! Total number of elements along the blade
+    int num_blade_elem;
+    //! Total number of elements along the tower
+    int num_tower_elem;
+
+    bool is_solution0{true};
+
+    //! Start time for this turbine
+    double start_time{0.0};
+
+    //! End time for this turbine
+    double stop_time{-1.0};
+
+    //! Timestep for CFD
+    double dt_cfd;
+
+    //! Timestep for external turbine solver
+    double dt_ext;
+
+    //! Number of sub-steps of external solver per CFD timestep
+    int num_substeps;
+
+    //! Time step index for external
+    int time_index{0};
+
+    //! Checkpoint interval for external
+    int chkpt_interval;
+
+    //! Data access functions that have to be defined for each type
+    virtual float* position_at_vel(int dir) const = 0;
+    virtual float* solid_velocity(int dir) const = 0;
+    virtual float* fluid_velocity(int dir) const = 0;
+    virtual float* force(int dir) const = 0;
+    virtual float* position_at_force(int dir) const = 0;
+    virtual float* chord_at_force() const = 0;
+    virtual float* orientation() const = 0;
+
+    virtual int length_fluid_velocity(int dir) const = 0;
+    virtual int length_force(int dir) const = 0;
+    virtual int length_position_at_force(int dir) const = 0;
+    virtual int length_orientation() const = 0;
+    virtual int num_vel_pts_blade() const = 0;
+};
+
+} // namespace ext_turb
+
+#endif /* EXTERNAL_BASE_TYPES_H */