API for state saving and restoring (NGWPC-8996)

CMakeLists.txt

@@ -165,7 +165,12 @@ add_compile_definitions(NGEN_SHARED_LIB_EXTENSION)
 set(Boost_USE_STATIC_LIBS OFF)
 set(Boost_USE_MULTITHREADED ON)
 set(Boost_USE_STATIC_RUNTIME OFF)
-find_package(Boost 1.86.0 REQUIRED)
+if(CMAKE_CXX_STANDARD LESS 17)
+    # requires non-header filesystem for state saving if C++ 11 or lower
+    find_package(Boost 1.86.0 REQUIRED COMPONENTS system filesystem)
+else()
+    find_package(Boost 1.86.0 REQUIRED)
+endif()
 
 # -----------------------------------------------------------------------------
 if(NGEN_WITH_SQLITE)
@@ -318,6 +323,7 @@ add_subdirectory("src/geojson")
 add_subdirectory("src/bmi")
 add_subdirectory("src/realizations/catchment")
 add_subdirectory("src/forcing")
+add_subdirectory("src/state_save_restore")
 add_subdirectory("src/utilities")
 add_subdirectory("src/utilities/mdarray")
 add_subdirectory("src/utilities/mdframe")
@@ -337,7 +343,9 @@ target_link_libraries(ngen
         NGen::core_mediator
         NGen::logging
         NGen::parallel
+        NGen::state_save_restore
         NGen::bmi_protocols
+        NGen::state_save_restore
 )
 
 if(NGEN_WITH_SQLITE)

Dockerfile

@@ -317,7 +317,8 @@ RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-soilfreezethaw \
 
 RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-ueb-bmi \
     set -eux && \
-    cmake -B extern/ueb-bmi/cmake_build -S extern/ueb-bmi/ -DBMICXX_INCLUDE_DIRS=/ngen-app/ngen/extern/bmi-cxx/ -DBOOST_ROOT=/opt/boost && \
+    cmake -B extern/ueb-bmi/cmake_build -S extern/ueb-bmi/ \
+        -DUEB_SUPPRESS_OUTPUTS=ON -DBMICXX_INCLUDE_DIRS=/ngen-app/ngen/extern/bmi-cxx/ -DBOOST_ROOT=/opt/boost && \
     cmake --build extern/ueb-bmi/cmake_build/ && \
     find /ngen-app/ngen/extern/ueb-bmi/ -name '*.o' -exec rm -f {} +
 

data/example_state_saving_config.json

@@ -0,0 +1,109 @@
+{
+    "global": {
+      "formulations": [
+        {
+          "name": "bmi_c++",
+          "params": {
+            "model_type_name": "test_bmi_cpp",
+            "library_file": "./extern/test_bmi_cpp/cmake_build/libtestbmicppmodel.so",
+            "init_config": "./data/bmi/c/test/test_bmi_c_config.ini",
+            "main_output_variable": "OUTPUT_VAR_2",
+            "variables_names_map" : {
+              "INPUT_VAR_2": "TMP_2maboveground",
+              "INPUT_VAR_1": "precip_rate"
+            },
+            "create_function": "bmi_model_create",
+            "destroy_function": "bmi_model_destroy",
+            "uses_forcing_file": false
+          }
+        }
+      ],
+      "forcing": {
+          "file_pattern": ".*{{id}}.*.csv",
+          "path": "./data/forcing/"
+      }
+    },
+    "state_saving": {
+        "label": "end",
+        "path": "state_end",
+        "type": "FilePerUnit",
+        "when": "EndOfRun"
+    },
+    "time": {
+        "start_time": "2015-12-01 00:00:00",
+        "end_time": "2015-12-30 23:00:00",
+        "output_interval": 3600
+    },
+    "output_root": "./output_dir/",
+    "catchments": {
+        "cat-27": {
+          "formulations": [
+            {
+              "name": "bmi_c++",
+              "params": {
+                "model_type_name": "test_bmi_cpp",
+                "library_file": "./extern/test_bmi_cpp/cmake_build/libtestbmicppmodel.so",
+                "init_config": "./data/bmi/c/test/test_bmi_c_config.ini",
+                "main_output_variable": "OUTPUT_VAR_2",
+                "variables_names_map" : {
+                  "INPUT_VAR_2": "TMP_2maboveground",
+                  "INPUT_VAR_1": "precip_rate"
+                },
+                "create_function": "bmi_model_create",
+                "destroy_function": "bmi_model_destroy",
+                "uses_forcing_file": false
+              }
+            }
+          ],
+          "forcing": {
+              "path": "./data/forcing/cat-27_2015-12-01 00_00_00_2015-12-30 23_00_00.csv"
+          }
+        },
+        "cat-52": {
+          "formulations": [
+            {
+              "name": "bmi_c++",
+              "params": {
+                "model_type_name": "test_bmi_cpp",
+                "library_file": "./extern/test_bmi_cpp/cmake_build/libtestbmicppmodel.so",
+                "init_config": "./data/bmi/c/test/test_bmi_c_config.ini",
+                "main_output_variable": "OUTPUT_VAR_2",
+                "variables_names_map" : {
+                  "INPUT_VAR_2": "TMP_2maboveground",
+                  "INPUT_VAR_1": "precip_rate"
+                },
+                "create_function": "bmi_model_create",
+                "destroy_function": "bmi_model_destroy",
+                "uses_forcing_file": false
+              }
+            }
+          ],
+          "forcing": {
+              "path": "./data/forcing/cat-52_2015-12-01 00_00_00_2015-12-30 23_00_00.csv"
+          }
+        },
+        "cat-67": {
+          "formulations": [
+            {
+              "name": "bmi_c++",
+              "params": {
+                "model_type_name": "test_bmi_cpp",
+                "library_file": "./extern/test_bmi_cpp/cmake_build/libtestbmicppmodel.so",
+                "init_config": "./data/bmi/c/test/test_bmi_c_config.ini",
+                "main_output_variable": "OUTPUT_VAR_2",
+                "variables_names_map" : {
+                  "INPUT_VAR_2": "TMP_2maboveground",
+                  "INPUT_VAR_1": "precip_rate"
+                },
+                "create_function": "bmi_model_create",
+                "destroy_function": "bmi_model_destroy",
+                "uses_forcing_file": false
+              }
+            }
+          ],
+          "forcing": {
+              "path": "./data/forcing/cat-67_2015-12-01 00_00_00_2015-12-30 23_00_00.csv"
+          }
+        }
+    }
+}

data/example_state_saving_config_multi.json

@@ -0,0 +1,117 @@
+{
+    "global": {
+        "formulations": [
+            {
+                "name": "bmi_multi",
+                "params": {
+                    "model_type_name": "bmi_multi_noahowp_cfe",
+                    "forcing_file": "",
+                    "init_config": "",
+                    "allow_exceed_end_time": true,
+                    "main_output_variable": "Q_OUT",
+                    "modules": [
+			{
+			    "name": "bmi_c++",
+			    "params": {
+                                "model_type_name": "bmi_c++_sloth",
+                                "library_file": "./extern/sloth/cmake_build/libslothmodel.so",
+                                "init_config": "/dev/null",
+                                "allow_exceed_end_time": true,
+                                "main_output_variable": "z",
+                                "uses_forcing_file": false,
+                                "model_params": {
+				    "sloth_ice_fraction_schaake(1,double,m,node)": 0.0,
+				    "sloth_ice_fraction_xinanjiang(1,double,1,node)": 0.0,
+				    "sloth_smp(1,double,1,node)": 0.0
+                                }
+			    }
+                        },
+                        {
+                            "name": "bmi_fortran",
+                            "params": {
+                                "model_type_name": "bmi_fortran_noahowp",
+                                "library_file": "./extern/noah-owp-modular/cmake_build/libsurfacebmi",
+                                "forcing_file": "",
+                                "init_config": "./data/bmi/fortran/noah-owp-modular-init-{{id}}.namelist.input",
+                                "allow_exceed_end_time": true,
+                                "main_output_variable": "QINSUR",
+                                "variables_names_map": {
+                                    "PRCPNONC": "atmosphere_water__liquid_equivalent_precipitation_rate",
+                                    "Q2": "atmosphere_air_water~vapor__relative_saturation",
+                                    "SFCTMP": "land_surface_air__temperature",
+                                    "UU": "land_surface_wind__x_component_of_velocity",
+                                    "VV": "land_surface_wind__y_component_of_velocity",
+                                    "LWDN": "land_surface_radiation~incoming~longwave__energy_flux",
+                                    "SOLDN": "land_surface_radiation~incoming~shortwave__energy_flux",
+                                    "SFCPRS": "land_surface_air__pressure"
+                                },
+                                "uses_forcing_file": false
+                            }
+                        },
+                        {
+                            "name": "bmi_c",
+                            "params": {
+                                "model_type_name": "bmi_c_pet",
+                                "library_file": "./extern/evapotranspiration/evapotranspiration/cmake_build/libpetbmi",
+                                "forcing_file": "",
+                                "init_config": "./data/bmi/c/pet/{{id}}_bmi_config.ini",
+                                "allow_exceed_end_time": true,
+                                "main_output_variable": "water_potential_evaporation_flux",
+                                "registration_function": "register_bmi_pet",
+                                "variables_names_map": {
+                                    "water_potential_evaporation_flux": "potential_evapotranspiration"
+                                },
+                                "uses_forcing_file": false
+                            }
+                        },
+                        {
+                            "name": "bmi_c",
+                            "params": {
+                                "model_type_name": "bmi_c_cfe",
+                                "library_file": "./extern/cfe/cmake_build/libcfebmi",
+                                "forcing_file": "",
+                                "init_config": "./data/bmi/c/cfe/{{id}}_bmi_config.ini",
+                                "allow_exceed_end_time": true,
+                                "main_output_variable": "Q_OUT",
+                                "registration_function": "register_bmi_cfe",
+                                "variables_names_map": {
+                                    "water_potential_evaporation_flux": "potential_evapotranspiration",
+                                    "atmosphere_air_water~vapor__relative_saturation": "SPFH_2maboveground",
+                                    "land_surface_air__temperature": "TMP_2maboveground",
+                                    "land_surface_wind__x_component_of_velocity": "UGRD_10maboveground",
+                                    "land_surface_wind__y_component_of_velocity": "VGRD_10maboveground",
+                                    "land_surface_radiation~incoming~longwave__energy_flux": "DLWRF_surface",
+                                    "land_surface_radiation~incoming~shortwave__energy_flux": "DSWRF_surface",
+                                    "land_surface_air__pressure": "PRES_surface",
+				    "ice_fraction_schaake" : "sloth_ice_fraction_schaake",
+				    "ice_fraction_xinanjiang" : "sloth_ice_fraction_xinanjiang",
+				    "soil_moisture_profile" : "sloth_smp"
+                                },
+                                "uses_forcing_file": false
+                            }
+                        }
+                    ],
+                    "uses_forcing_file": false
+                }
+            }
+        ],
+        "forcing": {
+            "file_pattern": ".*{{id}}.*..csv",
+            "path": "./data/forcing/",
+            "provider": "CsvPerFeature"
+        }
+    },
+    "state_saving": [{
+        "direction": "save",
+        "label": "end",
+        "path": "state_end",
+        "type": "FilePerUnit",
+        "when": "EndOfRun"
+    }],
+    "time": {
+        "start_time": "2015-12-01 00:00:00",
+        "end_time": "2015-12-30 23:00:00",
+        "output_interval": 3600
+    },
+    "output_root": "./output_dir/"
+}

include/bmi/Bmi_Py_Adapter.hpp

@@ -541,33 +541,30 @@ namespace models {
                 int itemSize = GetVarItemsize(name);
                 std::string py_type = GetVarType(name);
                 std::string cxx_type = get_analogous_cxx_type(py_type, (size_t) itemSize);
-
-                if (cxx_type == "short") {
-                    set_value<short>(name, (short *) src);
-                } else if (cxx_type == "int") {
-                    set_value<int>(name, (int *) src);
-                } else if (cxx_type == "long") {
-                    set_value<long>(name, (long *) src);
-                } else if (cxx_type == "long long") {
-                    //FIXME this gets dicey -- if a python numpy array is of type np.int64 (long long), 
-                    //but a c++ int* is passed to this function as src, it will fail in undefined ways...
-                    //the template type overload may be perferred for doing SetValue from framework components
-                    //such as forcing providers...
-                    set_value<long long>(name, (long long *) src);
-                } else if (cxx_type == "float") {
-                    set_value<float>(name, (float *) src);
-                } else if (cxx_type == "double") {
-                    set_value<double>(name, (double *) src);
-                } else if (cxx_type == "long double") {
-                    set_value<long double>(name, (long double *) src);
-                } else {
+                // macro for checking type and setting value
+                #define BMI_PY_SET_VALUE(type) if (cxx_type == #type) {\
+                                                this->set_value<type>(name, static_cast<type *>(src)); }
+                BMI_PY_SET_VALUE(signed char)
+                else BMI_PY_SET_VALUE(unsigned char)
+                else BMI_PY_SET_VALUE(short)
+                else BMI_PY_SET_VALUE(unsigned short)
+                else BMI_PY_SET_VALUE(int)
+                else BMI_PY_SET_VALUE(unsigned int)
+                else BMI_PY_SET_VALUE(long)
+                else BMI_PY_SET_VALUE(unsigned long)
+                else BMI_PY_SET_VALUE(long long)
+                else BMI_PY_SET_VALUE(unsigned long long)
+                else BMI_PY_SET_VALUE(float)
+                else BMI_PY_SET_VALUE(double)
+                else BMI_PY_SET_VALUE(long double)
+                else {
                     std::string throw_msg; throw_msg.assign("Bmi_Py_Adapter cannot set values for variable '" + name +
                                              "' that has unrecognized C++ type '" + cxx_type + "'");
                     LOG(throw_msg, LogLevel::WARNING);
                     throw std::runtime_error(throw_msg);
                 }
+                #undef BMI_PY_SET_VALUE
             }
-
             /**
              * Set the values of the given BMI variable for the model, sourcing new data from the provided vector.
              *
@@ -597,6 +594,24 @@ namespace models {
                 }
             }
 
+            /**
+             * Set the value of a variable. This version of setting a variable will send an array with the `size` specified instead of checking the BMI for its current size of the variable.
+             * Ownership of the pointer will remain in C++, so the consuming BMI should not maintain a reference to the values beyond the scope of its `set_value` method.
+             * 
+             * @param name The name of the BMI variable.
+             * @param src Pointer to the data that will be sent to the BMI.
+             * @param size The number of items represented by the pointer.
+             */
+            template <typename T>
+            void set_value_unchecked(const std::string &name, T *src, size_t size) {
+                // declare readonly array info with the pointer and size
+                py::buffer_info info(src, static_cast<py::ssize_t>(size), true);
+                // create the array with the info and NULL handler so python doesn't take ownership
+                py::array_t<T> src_array(info, nullptr);
+                // pass the array to python to read; the BMI should not attempt to maintain a reference beyond the scope of this function to prevent trying to use freed memory
+                bmi_model->attr("set_value")(name, src_array);
+            }
+
             /**
              * Set values for a model's BMI variable at specified indices.
              *