From 5e1730fd84d4dfe6bf98bfb912a5ef2c0779f2e6 Mon Sep 17 00:00:00 2001 From: David Schneider Date: Mon, 7 Mar 2022 12:16:30 +0100 Subject: [PATCH 1/8] Add a first version for storing a checkpoint --- include/base/checkpoint.h | 89 +++++++++++++++++++++++++ include/solid_mechanics/mf_elasticity.h | 28 +++++++- 2 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 include/base/checkpoint.h diff --git a/include/base/checkpoint.h b/include/base/checkpoint.h new file mode 100644 index 0000000..7626d1a --- /dev/null +++ b/include/base/checkpoint.h @@ -0,0 +1,89 @@ +#pragma once + +#include + +#include +#include + +#include + +// for implementing a check-pointing and restart mechanism. +#include +#include + +DEAL_II_NAMESPACE_OPEN + +namespace Utilities +{ + /** + * @brief Create a checkpoint consisting of the triangulation, vectors, and some metadata + * + * @tparam dim + * @tparam VectorType + * @param triangulation + * @param dof_handler + * @param vectors + * @param name + * @param t + */ + template + void + create_checkpoint( + const dealii::parallel::distributed::Triangulation &triangulation, + const dealii::DoFHandler & dof_handler, + const std::vector & vectors, + const std::string & name, + const double t) + { + dealii::parallel::distributed::SolutionTransfer + solution_transfer(dof_handler); + + solution_transfer.prepare_for_serialization(vectors); + + triangulation.save(name + "-checkpoint.mesh"); + + if (dealii::Utilities::MPI::this_mpi_process(MPI_COMM_WORLD) == 0) + { + std::ofstream file(name + "-checkpoint.metadata", std::ios::binary); + boost::archive::binary_oarchive oa(file); + oa << t; + } + } + + + /** + * @brief Load a checkpoint previously stored using create_checkpoint + * + * @tparam dim + * @tparam VectorType + * @param triangulation + * @param dof_handler + * @param vectors + * @param name + * @param t + */ + // template + // void + // load_checkpoint( + // const dealii::parallel::distributed::Triangulation &triangulation, + // const dealii::DoFHandler & dof_handler, + // const std::vector & vectors, + // const std::string & name, + // const double t) + // { + // parallel::distributed::SolutionTransfer solution_transfer( + // dof_handler); + + // solution_transfer.deserialize(vectors); + + // for (auto &it : vectors) + // it.update_ghost_values(); + + // std::ifstream file(name + "-checkpoint.metadata", std::ios::binary); + + // boost::archive::binary_iarchive ia(file); + // ia >> t; + // } +} // namespace Utilities + +DEAL_II_NAMESPACE_CLOSE diff --git a/include/solid_mechanics/mf_elasticity.h b/include/solid_mechanics/mf_elasticity.h index 6e0b010..826dff8 100644 --- a/include/solid_mechanics/mf_elasticity.h +++ b/include/solid_mechanics/mf_elasticity.h @@ -60,6 +60,7 @@ static const unsigned int debug_level = 0; #include #include +#include #include #include #include @@ -79,8 +80,6 @@ using namespace dealii; // into it: namespace FSI { - using namespace dealii; - // The Solid class is the central class. template class Solid @@ -146,6 +145,9 @@ namespace FSI void output_results(const unsigned int result_number) const; + void + write_checkpoint(); + // Set up an Additional data object template void @@ -573,6 +575,8 @@ namespace FSI mf_data_reference); precice_adapter->initialize(total_displacement); + write_checkpoint(); + // At the beginning, we reset the solution update for this time step... while (precice_adapter->is_coupling_ongoing()) { @@ -2041,6 +2045,26 @@ namespace FSI return std::make_tuple(lin_it, lin_res, cond_number); } + + + template + void + Solid::write_checkpoint() + { + // pcout << "checkpoint computation"; + + Utilities::create_checkpoint(triangulation, + dof_handler, + {&total_displacement, + &velocity}, + "test", + time.current() + ); + // pcout << "resume interrupted computation"; + } + + + template void Solid::output_results(const unsigned int result_number) const From 7ec4b0bd5c091b6f79c38f09aa299d28372639fa Mon Sep 17 00:00:00 2001 From: David Schneider Date: Tue, 8 Mar 2022 17:14:21 +0100 Subject: [PATCH 2/8] Add reload functionality --- include/base/checkpoint.h | 44 ++++++------- include/parameter/parameter_handling.h | 9 ++- include/solid_mechanics/mf_elasticity.h | 83 +++++++++++++++++++++---- 3 files changed, 100 insertions(+), 36 deletions(-) diff --git a/include/base/checkpoint.h b/include/base/checkpoint.h index 7626d1a..d9ca9ee 100644 --- a/include/base/checkpoint.h +++ b/include/base/checkpoint.h @@ -62,28 +62,28 @@ namespace Utilities * @param name * @param t */ - // template - // void - // load_checkpoint( - // const dealii::parallel::distributed::Triangulation &triangulation, - // const dealii::DoFHandler & dof_handler, - // const std::vector & vectors, - // const std::string & name, - // const double t) - // { - // parallel::distributed::SolutionTransfer solution_transfer( - // dof_handler); - - // solution_transfer.deserialize(vectors); - - // for (auto &it : vectors) - // it.update_ghost_values(); - - // std::ifstream file(name + "-checkpoint.metadata", std::ios::binary); - - // boost::archive::binary_iarchive ia(file); - // ia >> t; - // } + template + void + load_checkpoint( + const dealii::DoFHandler & dof_handler, + std::vector & vectors, + const std::string & name, + double & t) + { + // triangulation.load(name + "-checkpoint.mesh"); + dealii::parallel::distributed::SolutionTransfer + solution_transfer(dof_handler); + + solution_transfer.deserialize(vectors); + + for (auto &it : vectors) + it->update_ghost_values(); + + std::ifstream file(name + "-checkpoint.metadata", std::ios::binary); + + boost::archive::binary_iarchive ia(file); + ia >> t; + } } // namespace Utilities DEAL_II_NAMESPACE_CLOSE diff --git a/include/parameter/parameter_handling.h b/include/parameter/parameter_handling.h index 8d0f638..5f83845 100644 --- a/include/parameter/parameter_handling.h +++ b/include/parameter/parameter_handling.h @@ -311,8 +311,9 @@ namespace Parameters // Set the timestep size $ \varDelta t $ and the simulation end-time. struct Time { - double delta_t = 0.1; - double end_time = 1.; + double delta_t = 0.1; + double end_time = 1.; + bool read_checkpoint = false; void add_parameters(ParameterHandler &prm); @@ -329,6 +330,10 @@ namespace Parameters delta_t, "Time step size", Patterns::Double(0.)); + prm.add_parameter("Resume checkpoint", + read_checkpoint, + "Resume from a checkpoint", + Patterns::Bool()); } prm.leave_subsection(); } diff --git a/include/solid_mechanics/mf_elasticity.h b/include/solid_mechanics/mf_elasticity.h index 826dff8..fbf6641 100644 --- a/include/solid_mechanics/mf_elasticity.h +++ b/include/solid_mechanics/mf_elasticity.h @@ -148,6 +148,10 @@ namespace FSI void write_checkpoint(); + + void + load_checkpoint(); + // Set up an Additional data object template void @@ -575,7 +579,10 @@ namespace FSI mf_data_reference); precice_adapter->initialize(total_displacement); - write_checkpoint(); + if (parameters.read_checkpoint) + load_checkpoint(); + else + write_checkpoint(); // At the beginning, we reset the solution update for this time step... while (precice_adapter->is_coupling_ongoing()) @@ -618,6 +625,8 @@ namespace FSI time.increment(); } } + write_checkpoint(); + // for post-processing, print average CG iterations over the whole run: timer_out << std::endl @@ -639,7 +648,11 @@ namespace FSI { Assert(testcase.get() != nullptr, ExcInternalError()); testcase->make_coarse_grid_and_bcs(triangulation); - triangulation.refine_global(parameters.n_global_refinement); + + if (parameters.read_checkpoint) + triangulation.load("test-checkpoint.mesh"); + else + triangulation.refine_global(parameters.n_global_refinement); if (testcase->body_force.get() == nullptr) testcase->body_force = @@ -2051,16 +2064,62 @@ namespace FSI void Solid::write_checkpoint() { - // pcout << "checkpoint computation"; - - Utilities::create_checkpoint(triangulation, - dof_handler, - {&total_displacement, - &velocity}, - "test", - time.current() - ); - // pcout << "resume interrupted computation"; + pcout << "checkpoint computation" + << "\n"; + std::vector in_vectors({&total_displacement, + &velocity, + &acceleration, + &old_displacement, + &velocity_old, + &acceleration_old}); + + Utilities::create_checkpoint( + triangulation, dof_handler, in_vectors, "test", time.current()); + } + + + + template + void + Solid::load_checkpoint() + { + std::vector in_vectors({&total_displacement, + &velocity, + &acceleration, + &old_displacement, + &velocity_old, + &acceleration_old}); + + auto partitioner = std::make_shared( + dof_handler.locally_owned_dofs(), + locally_relevant_dofs, + mpi_communicator); + + // We need something temporarily because the reloading doesn't allow for + // ghosted vectors + std::vector tmp_vectors(in_vectors.size()); + std::vector tmp_vectors_ptr; + + for (auto &vec : tmp_vectors) + { + vec.reinit(partitioner); + tmp_vectors_ptr.emplace_back(&vec); + } + + double new_time = 0; + Utilities::load_checkpoint(dof_handler, + tmp_vectors_ptr, + "test", + new_time); + + + for (unsigned int i = 0; i < tmp_vectors_ptr.size(); ++i) + { + in_vectors[i]->copy_locally_owned_data_from(*(tmp_vectors_ptr[i])); + in_vectors[i]->update_ghost_values(); + } + + pcout << "resume interrupted computation at t = " << new_time << "\n"; } From 3c801959349cf4eecfc74c8a3c4af9a7b22c952a Mon Sep 17 00:00:00 2001 From: David Schneider Date: Mon, 2 Oct 2023 23:52:54 +0200 Subject: [PATCH 3/8] wip --- include/base/checkpoint.h | 42 +++++++++++++++++++------ include/solid_mechanics/mf_elasticity.h | 22 +------------ 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/include/base/checkpoint.h b/include/base/checkpoint.h index d9ca9ee..516169a 100644 --- a/include/base/checkpoint.h +++ b/include/base/checkpoint.h @@ -30,9 +30,9 @@ namespace Utilities void create_checkpoint( const dealii::parallel::distributed::Triangulation &triangulation, - const dealii::DoFHandler & dof_handler, - const std::vector & vectors, - const std::string & name, + const dealii::DoFHandler &dof_handler, + const std::vector &vectors, + const std::string &name, const double t) { dealii::parallel::distributed::SolutionTransfer @@ -64,25 +64,47 @@ namespace Utilities */ template void - load_checkpoint( - const dealii::DoFHandler & dof_handler, - std::vector & vectors, - const std::string & name, - double & t) + load_checkpoint(const dealii::DoFHandler &dof_handler, + std::vector &vectors, + const std::string &name, + double &t) { + auto partitioner = std::make_shared( + dof_handler.locally_owned_dofs(), + DoFTools::extract_locally_relevant_dofs(dof_handler), + MPI_COMM_WORLD); + + // We need something temporarily because the reloading doesn't allow for + // ghosted vectors + std::vector tmp_vectors(vectors.size()); + std::vector tmp_vectors_ptr; + + for (auto &vec : tmp_vectors) + { + vec.reinit(partitioner); + tmp_vectors_ptr.emplace_back(&vec); + } + // triangulation.load(name + "-checkpoint.mesh"); dealii::parallel::distributed::SolutionTransfer solution_transfer(dof_handler); - solution_transfer.deserialize(vectors); + solution_transfer.deserialize(tmp_vectors_ptr); - for (auto &it : vectors) + for (auto &it : tmp_vectors_ptr) it->update_ghost_values(); std::ifstream file(name + "-checkpoint.metadata", std::ios::binary); boost::archive::binary_iarchive ia(file); ia >> t; + + + for (unsigned int i = 0; i < tmp_vectors_ptr.size(); ++i) + { + vectors[i]->copy_locally_owned_data_from(*(tmp_vectors_ptr[i])); + vectors[i]->update_ghost_values(); + } } } // namespace Utilities diff --git a/include/solid_mechanics/mf_elasticity.h b/include/solid_mechanics/mf_elasticity.h index fbf6641..66e0830 100644 --- a/include/solid_mechanics/mf_elasticity.h +++ b/include/solid_mechanics/mf_elasticity.h @@ -2090,34 +2090,14 @@ namespace FSI &velocity_old, &acceleration_old}); - auto partitioner = std::make_shared( - dof_handler.locally_owned_dofs(), - locally_relevant_dofs, - mpi_communicator); - - // We need something temporarily because the reloading doesn't allow for - // ghosted vectors - std::vector tmp_vectors(in_vectors.size()); - std::vector tmp_vectors_ptr; - - for (auto &vec : tmp_vectors) - { - vec.reinit(partitioner); - tmp_vectors_ptr.emplace_back(&vec); - } double new_time = 0; Utilities::load_checkpoint(dof_handler, - tmp_vectors_ptr, + in_vectors, "test", new_time); - for (unsigned int i = 0; i < tmp_vectors_ptr.size(); ++i) - { - in_vectors[i]->copy_locally_owned_data_from(*(tmp_vectors_ptr[i])); - in_vectors[i]->update_ghost_values(); - } pcout << "resume interrupted computation at t = " << new_time << "\n"; } From 9f18c78b1b3e03225d36143fa62da52ddb32ffd3 Mon Sep 17 00:00:00 2001 From: David Schneider Date: Tue, 3 Oct 2023 01:03:06 +0200 Subject: [PATCH 4/8] Finaize reloading --- include/base/checkpoint.h | 44 +++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/include/base/checkpoint.h b/include/base/checkpoint.h index 516169a..641fdef 100644 --- a/include/base/checkpoint.h +++ b/include/base/checkpoint.h @@ -74,37 +74,49 @@ namespace Utilities DoFTools::extract_locally_relevant_dofs(dof_handler), MPI_COMM_WORLD); - // We need something temporarily because the reloading doesn't allow for - // ghosted vectors + // To load a checkpoint, we need write access to ghost entries + // Thus, we might need temporary vectors to comply with the required + // partitioner layout (fully distributed vector with write access to ghost + // entries). We use lazy allocation, if required. std::vector tmp_vectors(vectors.size()); std::vector tmp_vectors_ptr; - for (auto &vec : tmp_vectors) + for (std::size_t i = 0; i < vectors.size(); ++i) { - vec.reinit(partitioner); - tmp_vectors_ptr.emplace_back(&vec); + // With LA:d:V, we can grant write access using zero_out_ghost_values + // (this wouldn't work with distributed PETSc vectors or similar) + if (vectors[i]->has_ghost_elements()) + { + vectors[i]->zero_out_ghost_values(); + tmp_vectors_ptr.emplace_back(vectors[i]); + } + else + { + // in case our vector carries anyway only local data, i.e., no ghost + // values at all, we need to create a temporary vector + tmp_vectors[i].reinit(partitioner); + tmp_vectors_ptr.emplace_back(&tmp_vectors[i]); + } } // triangulation.load(name + "-checkpoint.mesh"); dealii::parallel::distributed::SolutionTransfer solution_transfer(dof_handler); - solution_transfer.deserialize(tmp_vectors_ptr); - for (auto &it : tmp_vectors_ptr) - it->update_ghost_values(); - - std::ifstream file(name + "-checkpoint.metadata", std::ios::binary); - - boost::archive::binary_iarchive ia(file); - ia >> t; - - for (unsigned int i = 0; i < tmp_vectors_ptr.size(); ++i) { - vectors[i]->copy_locally_owned_data_from(*(tmp_vectors_ptr[i])); + // Copy over data from the temporary vectors, if necessary + if (!vectors[i]->has_ghost_elements()) + vectors[i]->copy_locally_owned_data_from(*(tmp_vectors_ptr[i])); + // ... and update all ghost values vectors[i]->update_ghost_values(); } + + // Last but not least, retrieve the time-stamp data + std::ifstream file(name + "-checkpoint.metadata", std::ios::binary); + boost::archive::binary_iarchive ia(file); + ia >> t; } } // namespace Utilities From 5d0c2b0cfcbd939e323fb29ef479fdc753c19de7 Mon Sep 17 00:00:00 2001 From: David Schneider Date: Tue, 3 Oct 2023 10:46:44 +0200 Subject: [PATCH 5/8] Adjust creation to required partitioner, renamings and cosmetics --- include/base/{checkpoint.h => restart.h} | 80 +++++++++++++++--------- include/solid_mechanics/mf_elasticity.h | 40 +++++++----- 2 files changed, 76 insertions(+), 44 deletions(-) rename include/base/{checkpoint.h => restart.h} (52%) diff --git a/include/base/checkpoint.h b/include/base/restart.h similarity index 52% rename from include/base/checkpoint.h rename to include/base/restart.h index 641fdef..caef6ee 100644 --- a/include/base/checkpoint.h +++ b/include/base/restart.h @@ -16,35 +16,62 @@ DEAL_II_NAMESPACE_OPEN namespace Utilities { /** - * @brief Create a checkpoint consisting of the triangulation, vectors, and some metadata + * @brief Create files required for a restart consisting of the triangulation, vectors, and some metadata * - * @tparam dim - * @tparam VectorType - * @param triangulation - * @param dof_handler - * @param vectors - * @param name - * @param t + * @tparam PartitionerPtr shred_ptr holding a fully distributed Partitioner with ghost elements + * + * @param triangulation The triangulation used to create the checkpoint + * @param dof_handler DoFHandeler associated to the global vectors + * @param partitioner A valid shared_ptr with partitioner used to create tmp vectors if necessary + * @param vectors A list of vectors, which are stored in the checkpoint + * @param name File name(s) for the restart files + * @param t Absolute time associated to the data vectors */ - template + template void - create_checkpoint( + create_restart_snapshot( const dealii::parallel::distributed::Triangulation &triangulation, const dealii::DoFHandler &dof_handler, + PartitionerPtr partitioner, const std::vector &vectors, const std::string &name, const double t) { + // To load a restart, we need read access to ghost entries. + // Thus, we might need temporary vectors to comply with the required + // partitioner layout (fully distributed vector with read access to ghost + // entries). We use lazy allocation, if required. + std::vector tmp_vectors(vectors.size()); + std::vector tmp_vectors_ptr; + + for (std::size_t i = 0; i < vectors.size(); ++i) + { + if (!vectors[i]->partitioners_are_globally_compatible( + *partitioner.get())) + { + // in case our vector carries anyway only local data, i.e., no ghost + // values at all, we need to create a temporary vector + tmp_vectors[i].reinit(partitioner); + tmp_vectors[i] = *(vectors[i]); + tmp_vectors_ptr.emplace_back(&tmp_vectors[i]); + } + else + { + // if the partitioner is compatible, we can just use the existing + // vector + tmp_vectors_ptr.emplace_back(vectors[i]); + } + } + dealii::parallel::distributed::SolutionTransfer solution_transfer(dof_handler); - solution_transfer.prepare_for_serialization(vectors); - - triangulation.save(name + "-checkpoint.mesh"); + solution_transfer.prepare_for_serialization(tmp_vectors_ptr); + triangulation.save(name + ".mesh"); if (dealii::Utilities::MPI::this_mpi_process(MPI_COMM_WORLD) == 0) { - std::ofstream file(name + "-checkpoint.metadata", std::ios::binary); + std::ofstream file(name + ".metadata", std::ios::binary); boost::archive::binary_oarchive oa(file); oa << t; } @@ -52,29 +79,26 @@ namespace Utilities /** - * @brief Load a checkpoint previously stored using create_checkpoint + * @brief Load a restart snapshot previously stored using create_restart_snapshot * - * @tparam dim - * @tparam VectorType - * @param triangulation - * @param dof_handler - * @param vectors - * @param name - * @param t + * @param dof_handler DoFHandler associated to the vectors + * @param vectors Vectors container to load data into + * @param name Name of the file bundle + * @param t Absolute time stored in the metadata */ template void - load_checkpoint(const dealii::DoFHandler &dof_handler, - std::vector &vectors, - const std::string &name, - double &t) + load_restart_snapshot(const dealii::DoFHandler &dof_handler, + std::vector &vectors, + const std::string &name, + double &t) { auto partitioner = std::make_shared( dof_handler.locally_owned_dofs(), DoFTools::extract_locally_relevant_dofs(dof_handler), MPI_COMM_WORLD); - // To load a checkpoint, we need write access to ghost entries + // To load a restart, we need write access to ghost entries // Thus, we might need temporary vectors to comply with the required // partitioner layout (fully distributed vector with write access to ghost // entries). We use lazy allocation, if required. @@ -114,7 +138,7 @@ namespace Utilities } // Last but not least, retrieve the time-stamp data - std::ifstream file(name + "-checkpoint.metadata", std::ios::binary); + std::ifstream file(name + ".metadata", std::ios::binary); boost::archive::binary_iarchive ia(file); ia >> t; } diff --git a/include/solid_mechanics/mf_elasticity.h b/include/solid_mechanics/mf_elasticity.h index 66e0830..dbfa8c8 100644 --- a/include/solid_mechanics/mf_elasticity.h +++ b/include/solid_mechanics/mf_elasticity.h @@ -146,11 +146,11 @@ namespace FSI output_results(const unsigned int result_number) const; void - write_checkpoint(); + write_restart(); void - load_checkpoint(); + load_restart(); // Set up an Additional data object template @@ -580,9 +580,9 @@ namespace FSI precice_adapter->initialize(total_displacement); if (parameters.read_checkpoint) - load_checkpoint(); + load_restart(); else - write_checkpoint(); + write_restart(); // At the beginning, we reset the solution update for this time step... while (precice_adapter->is_coupling_ongoing()) @@ -625,8 +625,7 @@ namespace FSI time.increment(); } } - write_checkpoint(); - + write_restart(); // for post-processing, print average CG iterations over the whole run: timer_out << std::endl @@ -2062,9 +2061,9 @@ namespace FSI template void - Solid::write_checkpoint() + Solid::write_restart() { - pcout << "checkpoint computation" + pcout << "Creating restart files for t = " + std::to_string(time.current()) << "\n"; std::vector in_vectors({&total_displacement, &velocity, @@ -2073,15 +2072,24 @@ namespace FSI &velocity_old, &acceleration_old}); - Utilities::create_checkpoint( - triangulation, dof_handler, in_vectors, "test", time.current()); + // Make sure to pass updated vectors into the function + for (auto &in : in_vectors) + in->update_ghost_values(); + + Utilities::create_restart_snapshot( + triangulation, + dof_handler, + total_displacement.get_partitioner(), + in_vectors, + "test", + time.current()); } template void - Solid::load_checkpoint() + Solid::load_restart() { std::vector in_vectors({&total_displacement, &velocity, @@ -2092,14 +2100,14 @@ namespace FSI double new_time = 0; - Utilities::load_checkpoint(dof_handler, - in_vectors, - "test", - new_time); + Utilities::load_restart_snapshot(dof_handler, + in_vectors, + "test", + new_time); - pcout << "resume interrupted computation at t = " << new_time << "\n"; + pcout << "Loaded interrupted computation at t = " << new_time << "\n"; } From 683624dc29ceac9f6cbb93a606fc138d2e4c446b Mon Sep 17 00:00:00 2001 From: David Schneider Date: Tue, 3 Oct 2023 11:49:48 +0200 Subject: [PATCH 6/8] More wip --- include/base/restart.h | 10 ++++++-- include/base/time_handler.h | 6 +++++ include/solid_mechanics/mf_elasticity.h | 32 +++++++++++++++---------- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/include/base/restart.h b/include/base/restart.h index caef6ee..f22a131 100644 --- a/include/base/restart.h +++ b/include/base/restart.h @@ -26,6 +26,7 @@ namespace Utilities * @param vectors A list of vectors, which are stored in the checkpoint * @param name File name(s) for the restart files * @param t Absolute time associated to the data vectors + * @param t Timestep associated to the data vectors */ template void @@ -35,7 +36,8 @@ namespace Utilities PartitionerPtr partitioner, const std::vector &vectors, const std::string &name, - const double t) + const double t, + unsigned int timestep) { // To load a restart, we need read access to ghost entries. // Thus, we might need temporary vectors to comply with the required @@ -74,6 +76,7 @@ namespace Utilities std::ofstream file(name + ".metadata", std::ios::binary); boost::archive::binary_oarchive oa(file); oa << t; + oa << timestep; } } @@ -85,13 +88,15 @@ namespace Utilities * @param vectors Vectors container to load data into * @param name Name of the file bundle * @param t Absolute time stored in the metadata + * @param timestep Timestep stored in the metadata */ template void load_restart_snapshot(const dealii::DoFHandler &dof_handler, std::vector &vectors, const std::string &name, - double &t) + double &t, + unsigned int ×tep) { auto partitioner = std::make_shared( dof_handler.locally_owned_dofs(), @@ -141,6 +146,7 @@ namespace Utilities std::ifstream file(name + ".metadata", std::ios::binary); boost::archive::binary_iarchive ia(file); ia >> t; + ia >> timestep; } } // namespace Utilities diff --git a/include/base/time_handler.h b/include/base/time_handler.h index c7e29ef..d4a243b 100644 --- a/include/base/time_handler.h +++ b/include/base/time_handler.h @@ -45,6 +45,12 @@ class Time time_current += delta_t; ++timestep; } + void + set_time(double absolute_time, unsigned int nth_timestep) + { + time_current = absolute_time; + timestep = nth_timestep; + } private: unsigned int timestep; diff --git a/include/solid_mechanics/mf_elasticity.h b/include/solid_mechanics/mf_elasticity.h index dbfa8c8..bd0f6c5 100644 --- a/include/solid_mechanics/mf_elasticity.h +++ b/include/solid_mechanics/mf_elasticity.h @@ -60,9 +60,9 @@ static const unsigned int debug_level = 0; #include #include -#include #include #include +#include #include #include #include @@ -581,8 +581,6 @@ namespace FSI if (parameters.read_checkpoint) load_restart(); - else - write_restart(); // At the beginning, we reset the solution update for this time step... while (precice_adapter->is_coupling_ongoing()) @@ -625,6 +623,7 @@ namespace FSI time.increment(); } } + // has to come after we incremented the time write_restart(); // for post-processing, print average CG iterations over the whole run: @@ -649,7 +648,7 @@ namespace FSI testcase->make_coarse_grid_and_bcs(triangulation); if (parameters.read_checkpoint) - triangulation.load("test-checkpoint.mesh"); + triangulation.load(parameters.output_folder + "restart.mesh"); else triangulation.refine_global(parameters.n_global_refinement); @@ -2063,7 +2062,9 @@ namespace FSI void Solid::write_restart() { - pcout << "Creating restart files for t = " + std::to_string(time.current()) + pcout << "-- Creating restart files for t = " + + std::to_string(time.current()) + << " ( timestep " << time.get_timestep() << " ) " << "\n"; std::vector in_vectors({&total_displacement, &velocity, @@ -2081,8 +2082,9 @@ namespace FSI dof_handler, total_displacement.get_partitioner(), in_vectors, - "test", - time.current()); + parameters.output_folder + "restart", + time.current(), + time.get_timestep()); } @@ -2099,15 +2101,19 @@ namespace FSI &acceleration_old}); - double new_time = 0; + double loaded_time = 0; + unsigned int loaded_timestep = 0; Utilities::load_restart_snapshot(dof_handler, in_vectors, - "test", - new_time); - + parameters.output_folder + + "restart", + loaded_time, + loaded_timestep); - - pcout << "Loaded interrupted computation at t = " << new_time << "\n"; + pcout << "-- Loaded interrupted computation at t = " << loaded_time + << " ( timestep " << loaded_timestep << " ) " + << "\n"; + time.set_time(loaded_time, loaded_timestep); } From 66f7db73eb488edd6c8a751b4cfafeaf8f72f385 Mon Sep 17 00:00:00 2001 From: David Schneider Date: Tue, 3 Oct 2023 12:40:45 +0200 Subject: [PATCH 7/8] Cosmetics for the file naming --- include/parameter/parameter_handling.h | 14 +++---- include/solid_mechanics/mf_elasticity.h | 50 +++++++++++++++++-------- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/include/parameter/parameter_handling.h b/include/parameter/parameter_handling.h index 5f83845..8b9083b 100644 --- a/include/parameter/parameter_handling.h +++ b/include/parameter/parameter_handling.h @@ -311,9 +311,9 @@ namespace Parameters // Set the timestep size $ \varDelta t $ and the simulation end-time. struct Time { - double delta_t = 0.1; - double end_time = 1.; - bool read_checkpoint = false; + double delta_t = 0.1; + double end_time = 1.; + double start_time = 0; void add_parameters(ParameterHandler &prm); @@ -324,16 +324,16 @@ namespace Parameters { prm.enter_subsection("Time"); { + prm.add_parameter("Start time", + start_time, + "Restart simulation or start from the beginning", + Patterns::Double(0.)); prm.add_parameter("End time", end_time, "End time", Patterns::Double()); prm.add_parameter("Time step size", delta_t, "Time step size", Patterns::Double(0.)); - prm.add_parameter("Resume checkpoint", - read_checkpoint, - "Resume from a checkpoint", - Patterns::Bool()); } prm.leave_subsection(); } diff --git a/include/solid_mechanics/mf_elasticity.h b/include/solid_mechanics/mf_elasticity.h index bd0f6c5..cce049f 100644 --- a/include/solid_mechanics/mf_elasticity.h +++ b/include/solid_mechanics/mf_elasticity.h @@ -579,7 +579,7 @@ namespace FSI mf_data_reference); precice_adapter->initialize(total_displacement); - if (parameters.read_checkpoint) + if (parameters.start_time != 0) load_restart(); // At the beginning, we reset the solution update for this time step... @@ -647,8 +647,9 @@ namespace FSI Assert(testcase.get() != nullptr, ExcInternalError()); testcase->make_coarse_grid_and_bcs(triangulation); - if (parameters.read_checkpoint) - triangulation.load(parameters.output_folder + "restart.mesh"); + if (parameters.start_time != 0) + triangulation.load(parameters.output_folder + "restart-t-" + + std::to_string(parameters.start_time) + ".mesh"); else triangulation.refine_global(parameters.n_global_refinement); @@ -2062,8 +2063,28 @@ namespace FSI void Solid::write_restart() { - pcout << "-- Creating restart files for t = " + - std::to_string(time.current()) + // We need to apply some cosmetics to t to make it usable + std::string t_string = std::to_string(time.current()); + for (int i = t_string.size() - 1; i >= 1; i--) + { + if (t_string.at(i) == '0') + { + t_string.pop_back(); // Remove if last digit is '0'. + } + else if (t_string.at(i) == '.') + { + t_string.pop_back(); // Remove dot. + break; // Break after '.' is removed. + } + else + { + break; // Or break before a digit is removed. + } + } + std::string restart_file(parameters.output_folder + "restart-t-" + + t_string); + pcout << "-- Creating restart files \"" + restart_file + + "\" for t = " + t_string << " ( timestep " << time.get_timestep() << " ) " << "\n"; std::vector in_vectors({&total_displacement, @@ -2082,7 +2103,7 @@ namespace FSI dof_handler, total_displacement.get_partitioner(), in_vectors, - parameters.output_folder + "restart", + restart_file, time.current(), time.get_timestep()); } @@ -2100,18 +2121,17 @@ namespace FSI &velocity_old, &acceleration_old}); + std::string restart_file(parameters.output_folder + "restart-t-" + + std::to_string(parameters.start_time)); double loaded_time = 0; unsigned int loaded_timestep = 0; - Utilities::load_restart_snapshot(dof_handler, - in_vectors, - parameters.output_folder + - "restart", - loaded_time, - loaded_timestep); - - pcout << "-- Loaded interrupted computation at t = " << loaded_time - << " ( timestep " << loaded_timestep << " ) " + Utilities::load_restart_snapshot( + dof_handler, in_vectors, restart_file, loaded_time, loaded_timestep); + + pcout << "-- Restarting computation from files \"" + restart_file + + "\" at t = " + << loaded_time << " ( timestep " << loaded_timestep << " ) " << "\n"; time.set_time(loaded_time, loaded_timestep); } From c0e0109e1ac7bad9a0568530cf1add88dd442735 Mon Sep 17 00:00:00 2001 From: David Schneider Date: Tue, 3 Oct 2023 13:23:07 +0200 Subject: [PATCH 8/8] Make cosmetics consistent --- include/base/utilities.h | 37 +++++++++++++++- include/solid_mechanics/mf_elasticity.h | 56 +++++++++++++------------ 2 files changed, 65 insertions(+), 28 deletions(-) diff --git a/include/base/utilities.h b/include/base/utilities.h index b6c4ef3..8db5225 100644 --- a/include/base/utilities.h +++ b/include/base/utilities.h @@ -104,7 +104,42 @@ namespace Utilities << std::endl; } - + /** + * @brief Helper function to convert different representations of the same number + * (in double rpecision) to a common format by truncating trailing zeros. + * + * Examples: + * number = 0.0100 -> 0.01 + * number = 42.0100 -> 42.01 + * number = 42.00 -> 42 + * + * + * @param number the number to format + * @return std::string the string representation + */ + std::string + format_time_stamp_to_string(double number) + { + // We need to apply some cosmetics to t to make it usable + std::string str = std::to_string(number); + for (int i = str.size() - 1; i >= 1; i--) + { + if (str.at(i) == '0') + { + str.pop_back(); // Remove if last digit is '0'. + } + else if (str.at(i) == '.') + { + str.pop_back(); // Remove dot. + break; // Break after '.' is removed. + } + else + { + break; // Or break before a digit is removed. + } + } + return str; + } /** * @brief Rounds a given number up to a defined precision. diff --git a/include/solid_mechanics/mf_elasticity.h b/include/solid_mechanics/mf_elasticity.h index cce049f..2a93b47 100644 --- a/include/solid_mechanics/mf_elasticity.h +++ b/include/solid_mechanics/mf_elasticity.h @@ -564,7 +564,8 @@ namespace FSI testcase = testcase_; make_grid(); system_setup(); - output_results(0); + if (parameters.start_time != 0) + output_results(0); time.increment(); // We then declare the incremental solution update $\varDelta @@ -623,9 +624,12 @@ namespace FSI time.increment(); } } - // has to come after we incremented the time + // write the restart after we finished + // we decrement the time in the write_restart function, as we call the + // function after we incremented the time for the next iteration write_restart(); + // for post-processing, print average CG iterations over the whole run: timer_out << std::endl << "Average CG iter = " @@ -648,8 +652,14 @@ namespace FSI testcase->make_coarse_grid_and_bcs(triangulation); if (parameters.start_time != 0) - triangulation.load(parameters.output_folder + "restart-t-" + - std::to_string(parameters.start_time) + ".mesh"); + { + std::string restart_file( + parameters.output_folder + "restart-t-" + + Utilities::format_time_stamp_to_string(parameters.start_time) + + ".mesh"); + pcout << "-- Looking for restart file \"" << restart_file << "\"\n"; + triangulation.load(restart_file); + } else triangulation.refine_global(parameters.n_global_refinement); @@ -2063,29 +2073,18 @@ namespace FSI void Solid::write_restart() { - // We need to apply some cosmetics to t to make it usable - std::string t_string = std::to_string(time.current()); - for (int i = t_string.size() - 1; i >= 1; i--) - { - if (t_string.at(i) == '0') - { - t_string.pop_back(); // Remove if last digit is '0'. - } - else if (t_string.at(i) == '.') - { - t_string.pop_back(); // Remove dot. - break; // Break after '.' is removed. - } - else - { - break; // Or break before a digit is removed. - } - } + // first, decrement the time, as we call the function after an (unnecessary) + // increment + double t_decrement = time.current() - time.get_delta_t(); + unsigned int timestep_decrement = time.get_timestep() - 1; + + auto t_string = Utilities::format_time_stamp_to_string(t_decrement); + std::string restart_file(parameters.output_folder + "restart-t-" + t_string); pcout << "-- Creating restart files \"" + restart_file + "\" for t = " + t_string - << " ( timestep " << time.get_timestep() << " ) " + << " ( timestep " << timestep_decrement << " ) " << "\n"; std::vector in_vectors({&total_displacement, &velocity, @@ -2104,8 +2103,8 @@ namespace FSI total_displacement.get_partitioner(), in_vectors, restart_file, - time.current(), - time.get_timestep()); + t_decrement, + timestep_decrement); } @@ -2121,8 +2120,9 @@ namespace FSI &velocity_old, &acceleration_old}); - std::string restart_file(parameters.output_folder + "restart-t-" + - std::to_string(parameters.start_time)); + std::string restart_file( + parameters.output_folder + "restart-t-" + + Utilities::format_time_stamp_to_string(parameters.start_time)); double loaded_time = 0; unsigned int loaded_timestep = 0; @@ -2134,6 +2134,8 @@ namespace FSI << loaded_time << " ( timestep " << loaded_timestep << " ) " << "\n"; time.set_time(loaded_time, loaded_timestep); + // we loaded the time we already computed, so let's move on to the next step + time.increment(); }