diff --git a/examples/sph/run_sph_rendering.py b/examples/sph/run_sph_rendering.py index 1c8d660d0a..26100aa85f 100644 --- a/examples/sph/run_sph_rendering.py +++ b/examples/sph/run_sph_rendering.py @@ -213,7 +213,16 @@ def H_profile(r): model.change_htolerances(coarse=1.3, fine=1.1) model.timestep() model.change_htolerances(coarse=1.1, fine=1.1) +model.dump("tmp.sham") + +ctx = shamrock.Context() +model = shamrock.get_Model_SPH(context=ctx, vector_type="f64_3", sph_kernel="M4") +model.load_from_dump("tmp.sham") + +model.dump("tmp2.sham") + +exit() for i in range(5): model.timestep() diff --git a/src/shammodels/gsph/include/shammodels/gsph/Model.hpp b/src/shammodels/gsph/include/shammodels/gsph/Model.hpp index bada70a5e6..ee774da144 100644 --- a/src/shammodels/gsph/include/shammodels/gsph/Model.hpp +++ b/src/shammodels/gsph/include/shammodels/gsph/Model.hpp @@ -344,10 +344,9 @@ namespace shammodels::gsph { logger::info_ln("GSPH", "Loading state from dump", fname); } - std::string metadata_user{}; - shamrock::load_shamrock_dump(fname, metadata_user, ctx); + nlohmann::json j; + shamrock::load_shamrock_dump(fname, j, ctx); - nlohmann::json j = nlohmann::json::parse(metadata_user); j.at("solver_config").get_to(solver.solver_config); solver.init_ghost_layout(); @@ -372,8 +371,7 @@ namespace shammodels::gsph { nlohmann::json metadata; metadata["solver_config"] = solver.solver_config; - shamrock::write_shamrock_dump( - fname, metadata.dump(4), shambase::get_check_ref(ctx.sched)); + shamrock::write_shamrock_dump(fname, metadata, shambase::get_check_ref(ctx.sched)); } //////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/shammodels/ramses/include/shammodels/ramses/Model.hpp b/src/shammodels/ramses/include/shammodels/ramses/Model.hpp index fdaf470d22..1b28a9dcda 100644 --- a/src/shammodels/ramses/include/shammodels/ramses/Model.hpp +++ b/src/shammodels/ramses/include/shammodels/ramses/Model.hpp @@ -131,8 +131,7 @@ namespace shammodels::basegodunov { nlohmann::json metadata; metadata["solver_config"] = solver.solver_config; - shamrock::write_shamrock_dump( - fname, metadata.dump(4), shambase::get_check_ref(ctx.sched)); + shamrock::write_shamrock_dump(fname, metadata, shambase::get_check_ref(ctx.sched)); } /** @@ -145,11 +144,10 @@ namespace shammodels::basegodunov { logger::info_ln("Godunov", "Loading state from dump", fname); } - // Load the context state and recover user metadata - std::string metadata_user{}; - shamrock::load_shamrock_dump(fname, metadata_user, ctx); + // load the dump and recover the user metadata as json + nlohmann::json j; + shamrock::load_shamrock_dump(fname, j, ctx); - nlohmann::json j = nlohmann::json::parse(metadata_user); j.at("solver_config").get_to(solver.solver_config); // modules::GhostZones gz(ctx, solver.solver_config, storage); diff --git a/src/shammodels/sph/include/shammodels/sph/Model.hpp b/src/shammodels/sph/include/shammodels/sph/Model.hpp index 97f514113a..ae186c2acb 100644 --- a/src/shammodels/sph/include/shammodels/sph/Model.hpp +++ b/src/shammodels/sph/include/shammodels/sph/Model.hpp @@ -836,11 +836,9 @@ namespace shammodels::sph { } // Load the context state and recover user metadata - std::string metadata_user{}; - shamrock::load_shamrock_dump(fname, metadata_user, ctx); + nlohmann::json j; + shamrock::load_shamrock_dump(fname, j, ctx); - /// TODO: load solver config from metadata - nlohmann::json j = nlohmann::json::parse(metadata_user); // std::cout << j << std::endl; j.at("solver_config").get_to(solver.solver_config); @@ -887,8 +885,7 @@ namespace shammodels::sph { // Dump the state of the SPH model to a file /// TODO: replace supplied metadata by solver config json - shamrock::write_shamrock_dump( - fname, metadata.dump(4), shambase::get_check_ref(ctx.sched)); + shamrock::write_shamrock_dump(fname, metadata, shambase::get_check_ref(ctx.sched)); } //////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/shamrock/include/shamrock/io/ShamrockDump.hpp b/src/shamrock/include/shamrock/io/ShamrockDump.hpp index 4186d2a26b..82fdfae4e4 100644 --- a/src/shamrock/include/shamrock/io/ShamrockDump.hpp +++ b/src/shamrock/include/shamrock/io/ShamrockDump.hpp @@ -16,6 +16,7 @@ * */ +#include "nlohmann/json.hpp" #include "shamalgs/collective/io.hpp" #include "shamcomm/io.hpp" #include "shamrock/scheduler/PatchScheduler.hpp" @@ -34,7 +35,8 @@ namespace shamrock { * @param metadata_user The user-provided metadata to add to the dump * @param sched The patch scheduler to dump */ - void write_shamrock_dump(std::string fname, std::string metadata_user, PatchScheduler &sched); + void write_shamrock_dump( + std::string fname, const nlohmann::json &metadata_user, PatchScheduler &sched); /** * @brief Load a Shamrock dump file and restore the state of the patches and retreive user @@ -43,6 +45,6 @@ namespace shamrock { * @param metadata_user The user-provided metadata to store * @param ctx The Shamrock context to restore */ - void load_shamrock_dump(std::string fname, std::string &metadata_user, ShamrockCtx &ctx); + void load_shamrock_dump(std::string fname, nlohmann::json &metadata_user, ShamrockCtx &ctx); } // namespace shamrock diff --git a/src/shamrock/src/io/ShamrockDump.cpp b/src/shamrock/src/io/ShamrockDump.cpp index e3e7c97e82..89c333804e 100644 --- a/src/shamrock/src/io/ShamrockDump.cpp +++ b/src/shamrock/src/io/ShamrockDump.cpp @@ -19,13 +19,20 @@ #include "shamcomm/logs.hpp" #include "shamrock/io/ShamrockDump.hpp" +/// Currently this is the only valid tag as "1.0" of Shamrock dumps did not include any versioning +/// :'( +#define WRITER_VERSION_TAG "2.0" + +bool fused_metadata = true; + namespace shamrock { - void write_shamrock_dump(std::string fname, std::string metadata_user, PatchScheduler &sched) { + void write_shamrock_dump( + std::string fname, const nlohmann::json &metadata_user, PatchScheduler &sched) { StackEntry stack_loc{}; - std::string metadata_patch = sched.serialize_patch_metadata().dump(4); + nlohmann::json metadata_patch = sched.serialize_patch_metadata(); using namespace shamrock::patch; @@ -63,9 +70,12 @@ namespace shamrock { using namespace nlohmann; json j; - j["pids"] = all_pids; - j["bytecounts"] = all_bytecounts; - j["offsets"] = all_offsets; + j["pids"] = all_pids; + j["bytecounts"] = all_bytecounts; + j["offsets"] = all_offsets; + j["metadata_patch"] = metadata_patch; + j["metadata_user"] = metadata_user; + j["versioning"] = WRITER_VERSION_TAG; std::string sout = j.dump(4); @@ -82,19 +92,14 @@ namespace shamrock { // do some perf investigation before enabling preallocation bool preallocate = false; if (preallocate) { - MPI_Offset tot_byte = all_offsets.back() + all_bytecounts.back() + metadata_user.size() - + metadata_patch.size() + sout.size() + sizeof(std::size_t) * 3; + MPI_Offset tot_byte + = all_offsets.back() + all_bytecounts.back() + sout.size() + sizeof(std::size_t); MPICHECK(MPI_File_preallocate(mfile, tot_byte)); } - shamalgs::collective::write_header(mfile, metadata_user, head_ptr); - shamalgs::collective::write_header(mfile, metadata_patch, head_ptr); shamalgs::collective::write_header(mfile, sout, head_ptr); - shamlog_debug_ln( - "ShamrockDump", - shambase::format( - "table sizes {} {} {}", metadata_patch.size(), metadata_user.size(), sout.size())); + shamlog_debug_ln("ShamrockDump", shambase::format("table sizes {}", sout.size())); if (/*do check*/ true) { auto check_same_mpi = [](std::string s) { @@ -112,8 +117,6 @@ namespace shamrock { } }; - check_same_mpi(metadata_user); - check_same_mpi(metadata_patch); check_same_mpi(sout); } @@ -159,7 +162,7 @@ namespace shamrock { } } - void load_shamrock_dump(std::string fname, std::string &metadata_user, ShamrockCtx &ctx) { + void load_shamrock_dump(std::string fname, nlohmann::json &metadata_user, ShamrockCtx &ctx) { StackEntry stack_loc{}; @@ -171,12 +174,9 @@ namespace shamrock { shambase::Timer timer; timer.start(); - std::string metadata_patch{}; - std::string patchdata_infos{}; + std::string metadata{}; - metadata_user = shamalgs::collective::read_header(mfile, head_ptr); - metadata_patch = shamalgs::collective::read_header(mfile, head_ptr); - patchdata_infos = shamalgs::collective::read_header(mfile, head_ptr); + metadata = shamalgs::collective::read_header(mfile, head_ptr); if (!shamcmdopt::getenv_str("SHAMDUMP_OFFSET_MODE_OLD").has_value()) { // reset MPI view @@ -186,8 +186,11 @@ namespace shamrock { using namespace nlohmann; - json jmeta_patch = json::parse(metadata_patch); - json jpdat_info = json::parse(patchdata_infos); + json jmeta = json::parse(metadata); + + metadata_user = jmeta.at("metadata_user"); + + auto jmeta_patch = jmeta.at("metadata_patch"); ctx.pdata_layout_new(); *ctx.pdl = jmeta_patch.at("patchdata_layout").get(); @@ -217,9 +220,9 @@ namespace shamrock { std::vector all_pids; std::vector all_bytecounts; - all_bytecounts = jpdat_info.at("bytecounts").get>(); - all_offsets = jpdat_info.at("offsets").get>(); - all_pids = jpdat_info.at("pids").get>(); + all_bytecounts = jmeta.at("bytecounts").get>(); + all_offsets = jmeta.at("offsets").get>(); + all_pids = jmeta.at("pids").get>(); struct PatchFileOffset { u64 offset, bytecount;