From 49027c7aadf79ff0efa1093fb7de78dc3e8fda62 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 8 Jan 2024 20:58:04 +0100 Subject: [PATCH 01/70] Add OpenPMD as external lib --- CMakeLists.txt | 19 +++++++++++++++++++ src/CMakeLists.txt | 4 ++++ src/config.hpp.in | 3 +++ 3 files changed, 26 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 994399d4bdc0..dfe1ccf3c2ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,7 @@ option(PARTHENON_DISABLE_MPI "MPI is enabled by default if found, set this to Tr option(PARTHENON_ENABLE_HOST_COMM_BUFFERS "CUDA/HIP Only: Allocate communication buffers on host (may be slower)" OFF) option(PARTHENON_DISABLE_HDF5 "HDF5 is enabled by default if found, set this to True to disable HDF5" OFF) option(PARTHENON_DISABLE_HDF5_COMPRESSION "HDF5 compression is enabled by default, set this to True to disable compression in HDF5 output/restart files" OFF) +option(PARTHENON_DISABLE_OPENPMD "OpenPMD is enabled by default if found, set this to True to disable OpenPMD" OFF) option(PARTHENON_DISABLE_SPARSE "Sparse capability is enabled by default, set this to True to compile-time disable all sparse capability" OFF) option(PARTHENON_ENABLE_ASCENT "Enable Ascent for in situ visualization and analysis" OFF) option(PARTHENON_LINT_DEFAULT "Linting is turned off by default, use the \"lint\" target or set \ @@ -184,6 +185,24 @@ if (NOT PARTHENON_DISABLE_HDF5) install(TARGETS HDF5_C EXPORT parthenonTargets) endif() +if (NOT PARTHENON_DISABLE_OPENPMD) +#TODO(pgrete) add logic for serial/parallel +#TODO(pgrete) add logic for internal/external build + include(FetchContent) + set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) + set(openPMD_BUILD_CLI_TOOLS OFF) + set(openPMD_BUILD_EXAMPLES OFF) + set(openPMD_BUILD_TESTING OFF) + set(openPMD_BUILD_SHARED_LIBS OFF) # precedence over BUILD_SHARED_LIBS if needed + set(openPMD_INSTALL OFF) # or instead use: + # set(openPMD_INSTALL ${BUILD_SHARED_LIBS}) # only install if used as a shared library + set(openPMD_USE_PYTHON OFF) + FetchContent_Declare(openPMD + GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git" + GIT_TAG "0.15.2") + FetchContent_MakeAvailable(openPMD) +endif() + # Kokkos recommendatation resulting in not using default GNU extensions set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0843dc725dad..35a35279776c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -300,6 +300,10 @@ if (ENABLE_HDF5) target_link_libraries(parthenon PUBLIC HDF5_C) endif() +if (PARTHENON_ENABLE_OPENPMD) + target_link_libraries(parthenon PUBLIC openPMD::openPMD) +endif() + # For Cuda with NVCC (<11.2) and C++17 Kokkos currently does not work/compile with # relaxed-constexpr, see https://github.com/kokkos/kokkos/issues/3496 # However, Parthenon heavily relies on it and there is no harm in compiling Kokkos diff --git a/src/config.hpp.in b/src/config.hpp.in index 299fe0936978..a19033bddbcf 100644 --- a/src/config.hpp.in +++ b/src/config.hpp.in @@ -45,6 +45,9 @@ // defne ENABLE_HDF5 or not at all #cmakedefine ENABLE_HDF5 +// defne ENABLE_HDF5 or not at all +#cmakedefine PARTHENON_ENABLE_OPENPMD + // define PARTHENON_DISABLE_HDF5_COMPRESSION or not at all #cmakedefine PARTHENON_DISABLE_HDF5_COMPRESSION From 4525e86262a0fdcb145ae6eac636ee12d07efe23 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 11 Jan 2024 14:16:56 +0100 Subject: [PATCH 02/70] Add OpenPMD skeleto --- CMakeLists.txt | 7 +- src/CMakeLists.txt | 1 + src/config.hpp.in | 1 - src/outputs/openpmd.cpp | 213 ++++++++++++++++++++++++++++++++++++++++ src/outputs/outputs.hpp | 12 +++ 5 files changed, 230 insertions(+), 4 deletions(-) create mode 100644 src/outputs/openpmd.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index dfe1ccf3c2ff..b031bef6a060 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,12 +31,12 @@ include(CTest) # Compile time constants # Compile Options -option(PARTHENON_SINGLE_PRECISION "Run in single precision" OFF) +option(PARTHENON_SINGLE_PRENSION "Run in single precision" OFF) option(PARTHENON_DISABLE_MPI "MPI is enabled by default if found, set this to True to disable MPI" OFF) option(PARTHENON_ENABLE_HOST_COMM_BUFFERS "CUDA/HIP Only: Allocate communication buffers on host (may be slower)" OFF) option(PARTHENON_DISABLE_HDF5 "HDF5 is enabled by default if found, set this to True to disable HDF5" OFF) option(PARTHENON_DISABLE_HDF5_COMPRESSION "HDF5 compression is enabled by default, set this to True to disable compression in HDF5 output/restart files" OFF) -option(PARTHENON_DISABLE_OPENPMD "OpenPMD is enabled by default if found, set this to True to disable OpenPMD" OFF) +option(PARTHENON_ENABLE_OPENPMD "OpenPMD is enabled by default if found, set this to True to disable OpenPMD" ON) option(PARTHENON_DISABLE_SPARSE "Sparse capability is enabled by default, set this to True to compile-time disable all sparse capability" OFF) option(PARTHENON_ENABLE_ASCENT "Enable Ascent for in situ visualization and analysis" OFF) option(PARTHENON_LINT_DEFAULT "Linting is turned off by default, use the \"lint\" target or set \ @@ -185,7 +185,7 @@ if (NOT PARTHENON_DISABLE_HDF5) install(TARGETS HDF5_C EXPORT parthenonTargets) endif() -if (NOT PARTHENON_DISABLE_OPENPMD) +if (PARTHENON_ENABLE_OPENPMD) #TODO(pgrete) add logic for serial/parallel #TODO(pgrete) add logic for internal/external build include(FetchContent) @@ -201,6 +201,7 @@ if (NOT PARTHENON_DISABLE_OPENPMD) GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git" GIT_TAG "0.15.2") FetchContent_MakeAvailable(openPMD) + install(TARGETS openPMD EXPORT parthenonTargets) endif() # Kokkos recommendatation resulting in not using default GNU extensions diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 35a35279776c..b9e48fd9db92 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -178,6 +178,7 @@ add_library(parthenon outputs/history.cpp outputs/io_wrapper.cpp outputs/io_wrapper.hpp + outputs/openpmd.cpp outputs/output_utils.cpp outputs/output_utils.hpp outputs/outputs.cpp diff --git a/src/config.hpp.in b/src/config.hpp.in index a19033bddbcf..8f67ed63bc5e 100644 --- a/src/config.hpp.in +++ b/src/config.hpp.in @@ -45,7 +45,6 @@ // defne ENABLE_HDF5 or not at all #cmakedefine ENABLE_HDF5 -// defne ENABLE_HDF5 or not at all #cmakedefine PARTHENON_ENABLE_OPENPMD // define PARTHENON_DISABLE_HDF5_COMPRESSION or not at all diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp new file mode 100644 index 000000000000..94d9b7e1365a --- /dev/null +++ b/src/outputs/openpmd.cpp @@ -0,0 +1,213 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2024 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +// (C) (or copyright) 2024. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== +//! \file openpmd.cpp +// \brief Output for OpenPMD https://www.openpmd.org/ (supporting various backends) + +#include +#include +#include +#include +#include +#include +#include + +// Parthenon headers +#include "coordinates/coordinates.hpp" +#include "defs.hpp" +#include "globals.hpp" +#include "interface/variable_state.hpp" +#include "mesh/mesh.hpp" +#include "outputs/output_utils.hpp" +#include "outputs/outputs.hpp" +#include "utils/error_checking.hpp" + +// OpenPMD headers +#ifdef PARTHENON_ENABLE_OPENPMD +#include +#endif // ifdef PARTHENON_ENABLE_OPENPMD + +namespace parthenon { + +using namespace OutputUtils; + +//---------------------------------------------------------------------------------------- +//! \fn void OpenPMDOutput:::WriteOutputFile(Mesh *pm) +// \brief Expose mesh and all Cell variables for processing with Ascent +void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, + const SignalHandler::OutputSignal signal) { +#ifndef PARTHENON_ENABLE_OPENPMD + if (Globals::my_rank == 0) { + PARTHENON_WARN("OpenPMD output requested by input file, but OpenPMD support not " + "compiled in. Skipping this output type."); + } +#else + + using conduit::Node; + + // Ascent needs the MPI communicator we are using + ascent::Ascent ascent; + Node ascent_opts; + ascent_opts["mpi_comm"] = MPI_Comm_c2f(MPI_COMM_WORLD); + ascent_opts["actions_file"] = pin->GetString(output_params.block_name, "actions_file"); + // Only publish fields that are used within actions to reduce memory footprint. + // A user might need to override this, e.g., in a runtime ascent_options.yaml, if + // the required fields cannot be resolved by Ascent. + // See https://ascent.readthedocs.io/en/latest/AscentAPI.html#field-filtering + // TODO(some in mid 2023) Reenable this as this currently only works in develop of + // Ascent and not in published release (expected in 0.9.1), see + // https://github.com/Alpine-DAV/ascent/pull/1109 + // ascent_opts["field_filtering"] = "true"; + ascent.open(ascent_opts); + + // create root node for the whole mesh + Node root; + + for (auto &pmb : pm->block_list) { + // create a unique id for this MeshBlock + const std::string &meshblock_name = "domain_" + std::to_string(pmb->gid); + Node &mesh = root[meshblock_name]; + + // add basic state info + mesh["state/domain_id"] = pmb->gid; + mesh["state/cycle"] = tm->ncycle; + mesh["state/time"] = tm->time; + + auto &bounds = pmb->cellbounds; + auto ib = bounds.GetBoundsI(IndexDomain::entire); + auto jb = bounds.GetBoundsJ(IndexDomain::entire); + auto kb = bounds.GetBoundsK(IndexDomain::entire); + auto ni = ib.e - ib.s + 1; + auto nj = jb.e - jb.s + 1; + auto nk = kb.e - kb.s + 1; + uint64_t ncells = ni * nj * nk; + + auto ib_int = bounds.GetBoundsI(IndexDomain::interior); + auto jb_int = bounds.GetBoundsJ(IndexDomain::interior); + auto kb_int = bounds.GetBoundsK(IndexDomain::interior); + + auto &coords = pmb->coords; + Real dx1 = coords.CellWidth(ib.s, jb.s, kb.s); + Real dx2 = coords.CellWidth(ib.s, jb.s, kb.s); + Real dx3 = coords.CellWidth(ib.s, jb.s, kb.s); + std::array corner = coords.GetXmin(); + + // create the coordinate set + mesh["coordsets/coords/type"] = "uniform"; + PARTHENON_REQUIRE_THROWS(typeid(Coordinates_t) == typeid(UniformCartesian), + "Ascent currently only supports Cartesian coordinates."); + + mesh["coordsets/coords/dims/i"] = ni + 1; + mesh["coordsets/coords/dims/j"] = nj + 1; + if (nk > 1) { + mesh["coordsets/coords/dims/k"] = nk + 1; + } + + // add origin and spacing to the coordset (optional) + mesh["coordsets/coords/origin/x"] = corner[0]; + mesh["coordsets/coords/origin/y"] = corner[1]; + if (nk > 1) { + mesh["coordsets/coords/origin/z"] = corner[2]; + } + + mesh["coordsets/coords/spacing/dx"] = dx1; + mesh["coordsets/coords/spacing/dy"] = dx2; + if (nk > 1) { + mesh["coordsets/coords/spacing/dz"] = dx3; + } + + // add the topology + mesh["topologies/topo/type"] = "uniform"; + mesh["topologies/topo/coordset"] = "coords"; + + // indicate ghost zones with ascent_ghosts set to 1 + Node &n_field = mesh["fields/ascent_ghosts"]; + n_field["association"] = "element"; + n_field["topology"] = "topo"; + + // allocate ghost mask if not already done + if (ghost_mask_.data() == nullptr) { + ghost_mask_ = ParArray1D("Ascent ghost mask", ncells); + + const int njni = nj * ni; + auto &ghost_mask = ghost_mask_; // redef to lambda capture class member + pmb->par_for( + "Set ascent ghost mask", 0, ncells - 1, KOKKOS_LAMBDA(const int &idx) { + const int k = idx / (njni); + const int j = (idx - k * njni) / ni; + const int i = idx - k * njni - j * nj; + + if ((i < ib_int.s) || (ib_int.e < i) || (j < jb_int.s) || (jb_int.e < j) || + ((nk > 1) && ((k < kb_int.s) || (kb_int.e < k)))) { + ghost_mask(idx) = 1; + } else { + ghost_mask(idx) = 0; + } + }); + } + // Set ghost mask + n_field["values"].set_external(ghost_mask_.data(), ncells); + + // create a field for each component of each variable pack + auto &mbd = pmb->meshblock_data.Get(); + + for (const auto &var : mbd->GetVariableVector()) { + // ensure that only cell vars are added (for now) as the topology above is only + // valid for cell centered vars + if (!var->IsSet(Metadata::Cell)) { + continue; + } + const auto var_info = VarInfo(var); + + for (int icomp = 0; icomp < var_info.num_components; ++icomp) { + auto const data = Kokkos::subview(var->data, 0, 0, icomp, Kokkos::ALL(), + Kokkos::ALL(), Kokkos::ALL()); + const std::string varname = var_info.component_labels.at(icomp); + mesh["fields/" + varname + "/association"] = "element"; + mesh["fields/" + varname + "/topology"] = "topo"; + mesh["fields/" + varname + "/values"].set_external(data.data(), ncells); + } + } + } + + // make sure we conform: + Node verify_info; + if (!conduit::blueprint::mesh::verify(root, verify_info)) { + if (parthenon::Globals::my_rank == 0) { + PARTHENON_WARN("Ascent output: blueprint::mesh::verify failed!"); + } + verify_info.print(); + } + ascent.publish(root); + + // Create dummy action as we need to "execute" to override the actions defined in the + // yaml file. + Node actions; + // execute the actions + ascent.execute(actions); + + // close ascent + ascent.close(); +#endif // ifndef PARTHENON_ENABLE_OPENPMD + + // advance output parameters + output_params.file_number++; + output_params.next_time += output_params.dt; + pin->SetInteger(output_params.block_name, "file_number", output_params.file_number); + pin->SetReal(output_params.block_name, "next_time", output_params.next_time); +} + +} // namespace parthenon diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index 4fd64236a19c..bd24a2d95e4c 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -189,6 +189,18 @@ class AscentOutput : public OutputType { ParArray1D ghost_mask_; }; +//---------------------------------------------------------------------------------------- +//! \class OpenPMDOutput +// \brief derived OutputType class for OpenPMD based output + +class OpenPMDOutput : public OutputType { + public: + explicit OpenPMDOutput(const OutputParameters &oparams) : OutputType(oparams) {} + void WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, + const SignalHandler::OutputSignal signal) override; + +}; + #ifdef ENABLE_HDF5 //---------------------------------------------------------------------------------------- //! \class PHDF5Output From 4de250caa11c3115a210216305f59f92e0a18fb3 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 11 Jan 2024 23:41:01 +0100 Subject: [PATCH 03/70] WIP more Open PMD --- src/outputs/openpmd.cpp | 58 +++++++++++++++++++++++++---------------- src/outputs/outputs.cpp | 11 ++++++++ 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 94d9b7e1365a..7943a3e0efe6 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -31,6 +31,8 @@ #include "globals.hpp" #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" +#include "openPMD/IO/Access.hpp" +#include "openPMD/Series.hpp" #include "outputs/output_utils.hpp" #include "outputs/outputs.hpp" #include "utils/error_checking.hpp" @@ -48,38 +50,50 @@ using namespace OutputUtils; //! \fn void OpenPMDOutput:::WriteOutputFile(Mesh *pm) // \brief Expose mesh and all Cell variables for processing with Ascent void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, - const SignalHandler::OutputSignal signal) { + const SignalHandler::OutputSignal signal) { #ifndef PARTHENON_ENABLE_OPENPMD if (Globals::my_rank == 0) { PARTHENON_WARN("OpenPMD output requested by input file, but OpenPMD support not " "compiled in. Skipping this output type."); } #else - - using conduit::Node; - - // Ascent needs the MPI communicator we are using - ascent::Ascent ascent; - Node ascent_opts; - ascent_opts["mpi_comm"] = MPI_Comm_c2f(MPI_COMM_WORLD); - ascent_opts["actions_file"] = pin->GetString(output_params.block_name, "actions_file"); - // Only publish fields that are used within actions to reduce memory footprint. - // A user might need to override this, e.g., in a runtime ascent_options.yaml, if - // the required fields cannot be resolved by Ascent. - // See https://ascent.readthedocs.io/en/latest/AscentAPI.html#field-filtering - // TODO(some in mid 2023) Reenable this as this currently only works in develop of - // Ascent and not in published release (expected in 0.9.1), see - // https://github.com/Alpine-DAV/ascent/pull/1109 - // ascent_opts["field_filtering"] = "true"; - ascent.open(ascent_opts); - - // create root node for the whole mesh - Node root; + using openPMD::Access; + using openPMD::Series; + + // TODO(pgrete) .h5 for hd5 and .bp for ADIOS2 or .json for JSON + // TODO(pgrete) check if CREATE is the correct pattern (for not overwriting the series + // but an interation) This just describes the pattern of the filename. The correct file + // will be accessed through the iteration idx below. The file suffix maps to the chosen + // backend. + Series series = Series("test_%05T.h5", Access::CREATE); + + // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? + // TODO(pgrete) Should we update for restart or only set this once? Or make it per + // iteration? + // ... = pin->GetString(output_params.block_name, "actions_file"); + series.setAuthor("My Name file naming + auto it = series.iterations[42]; + it.open(); // explicit open() is important when run in parallel + + it.setTime(tm->time); + it.setDt(tm->dt); for (auto &pmb : pm->block_list) { // create a unique id for this MeshBlock const std::string &meshblock_name = "domain_" + std::to_string(pmb->gid); - Node &mesh = root[meshblock_name]; + auto mesh = it.meshes[meshblock_name]; // add basic state info mesh["state/domain_id"] = pmb->gid; diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index 33b35a24f284..028dad6513f3 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -246,6 +246,17 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { pnew_type = new VTKOutput(op); } else if (op.file_type == "ascent") { pnew_type = new AscentOutput(op); + } else if (op.file_type == "openpmd") { +#ifdef PARTHENON_ENABLE_OPENPMD + pnew_type = new OpenPMDOutput(op, pin); +#else + msg << "### FATAL ERROR in Outputs constructor" << std::endl + << "Executable not configured for OpenPMD outputs, but OpenPMD file format " + << "is requested in output/restart block '" << op.block_name << "'. " + << "You can disable this block without deleting it by setting a dt < 0." + << std::endl; + PARTHENON_FAIL(msg); +#endif // ifdef PARTHENON_ENABLE_OPENPMD } else if (op.file_type == "histogram") { #ifdef ENABLE_HDF5 pnew_type = new HistogramOutput(op, pin); From b906af7a87c27bb21a4931834644db02633bfe09 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 12 Jan 2024 17:55:04 +0100 Subject: [PATCH 04/70] WIP OpenPMD use file id --- src/outputs/openpmd.cpp | 60 +++++------------------------------------ 1 file changed, 7 insertions(+), 53 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 7943a3e0efe6..01330cb48ec3 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -84,12 +84,11 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // open iteration (corresponding to a timestep in OpenPMD naming) // TODO(pgrete) fix iteration name <-> file naming - auto it = series.iterations[42]; + auto it = series.iterations[output_params.file_number]; it.open(); // explicit open() is important when run in parallel it.setTime(tm->time); it.setDt(tm->dt); - for (auto &pmb : pm->block_list) { // create a unique id for this MeshBlock const std::string &meshblock_name = "domain_" + std::to_string(pmb->gid); @@ -143,38 +142,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, mesh["coordsets/coords/spacing/dz"] = dx3; } - // add the topology - mesh["topologies/topo/type"] = "uniform"; - mesh["topologies/topo/coordset"] = "coords"; - - // indicate ghost zones with ascent_ghosts set to 1 - Node &n_field = mesh["fields/ascent_ghosts"]; - n_field["association"] = "element"; - n_field["topology"] = "topo"; - - // allocate ghost mask if not already done - if (ghost_mask_.data() == nullptr) { - ghost_mask_ = ParArray1D("Ascent ghost mask", ncells); - - const int njni = nj * ni; - auto &ghost_mask = ghost_mask_; // redef to lambda capture class member - pmb->par_for( - "Set ascent ghost mask", 0, ncells - 1, KOKKOS_LAMBDA(const int &idx) { - const int k = idx / (njni); - const int j = (idx - k * njni) / ni; - const int i = idx - k * njni - j * nj; - - if ((i < ib_int.s) || (ib_int.e < i) || (j < jb_int.s) || (jb_int.e < j) || - ((nk > 1) && ((k < kb_int.s) || (kb_int.e < k)))) { - ghost_mask(idx) = 1; - } else { - ghost_mask(idx) = 0; - } - }); - } - // Set ghost mask - n_field["values"].set_external(ghost_mask_.data(), ncells); - // create a field for each component of each variable pack auto &mbd = pmb->meshblock_data.Get(); @@ -196,25 +163,12 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } } } - - // make sure we conform: - Node verify_info; - if (!conduit::blueprint::mesh::verify(root, verify_info)) { - if (parthenon::Globals::my_rank == 0) { - PARTHENON_WARN("Ascent output: blueprint::mesh::verify failed!"); - } - verify_info.print(); - } - ascent.publish(root); - - // Create dummy action as we need to "execute" to override the actions defined in the - // yaml file. - Node actions; - // execute the actions - ascent.execute(actions); - - // close ascent - ascent.close(); + // The iteration can be closed in order to help free up resources. + // The iteration's content will be flushed automatically. + // An iteration once closed cannot (yet) be reopened. + it.close(); + // No need to close series as it's done in the desctructor of the object when it runs + // out of scope. #endif // ifndef PARTHENON_ENABLE_OPENPMD // advance output parameters From 79660a2605cdd8b70afd2bfaa3992006eb9c0898 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 29 Feb 2024 16:48:13 +0100 Subject: [PATCH 05/70] Write blocks --- src/outputs/openpmd.cpp | 120 ++++++++++++++++++++++++---------------- src/outputs/outputs.cpp | 2 +- 2 files changed, 73 insertions(+), 49 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 01330cb48ec3..2868460ef09c 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -17,6 +17,7 @@ //! \file openpmd.cpp // \brief Output for OpenPMD https://www.openpmd.org/ (supporting various backends) +#include #include #include #include @@ -24,6 +25,8 @@ #include #include #include +#include +#include // Parthenon headers #include "coordinates/coordinates.hpp" @@ -31,10 +34,14 @@ #include "globals.hpp" #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" +#include "openPMD/Datatype.hpp" #include "openPMD/IO/Access.hpp" +#include "openPMD/Mesh.hpp" #include "openPMD/Series.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" #include "outputs/output_utils.hpp" #include "outputs/outputs.hpp" +#include "parthenon_array_generic.hpp" #include "utils/error_checking.hpp" // OpenPMD headers @@ -65,7 +72,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // but an interation) This just describes the pattern of the filename. The correct file // will be accessed through the iteration idx below. The file suffix maps to the chosen // backend. - Series series = Series("test_%05T.h5", Access::CREATE); + Series series = Series("adios_test_%05T.bp", Access::CREATE); // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per @@ -75,13 +82,16 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, series.setComment("Hello world!"); series.setMachine("bla"); series.setSoftware("Parthenon + Downstream info"); - series.setDate("today"); + series.setDate("2024-02-29"); // TODO(pgrete) Units? // TODO(pgrete) We probably want this for params! series.setAttribute("bla", true); + // In line with existing outputs, we write one file per iteration/snapshot + series.setIterationEncoding(openPMD::IterationEncoding::fileBased); + // open iteration (corresponding to a timestep in OpenPMD naming) // TODO(pgrete) fix iteration name <-> file naming auto it = series.iterations[output_params.file_number]; @@ -89,64 +99,64 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, it.setTime(tm->time); it.setDt(tm->dt); - for (auto &pmb : pm->block_list) { + + // TODO(pgrete) check var name standard compatiblity + // e.g., description: names of records and their components are only allowed to contain + // the characters a-Z, the numbers 0-9 and the underscore _ + + const int num_blocks_local = static_cast(pm->block_list.size()); + // TODO(pgrete) adjust for single prec output + // openPMD::Datatype dtype = openPMD::determineDatatype(); + // using OutT = typename std::conditional::type; + // Need to create the buffer outside so that it's persistent until the data is flushed. + // Dynamically allocating the inner vector should not be a performance bottleneck given + // that the hdf5 backend also uses an explicit per block and per variable + // GetHostMirrorAndCopy, but still this could be optimized. + std::vector> buffer_list; + + for (size_t b_idx = 0; b_idx < num_blocks_local; ++b_idx) { + const auto &pmb = pm->block_list[b_idx]; // create a unique id for this MeshBlock - const std::string &meshblock_name = "domain_" + std::to_string(pmb->gid); - auto mesh = it.meshes[meshblock_name]; + const std::string &meshblock_name = "block_" + std::to_string(pmb->gid); + auto block_record = it.meshes[meshblock_name]; - // add basic state info - mesh["state/domain_id"] = pmb->gid; - mesh["state/cycle"] = tm->ncycle; - mesh["state/time"] = tm->time; + // TODO(pgrete) check if we should update the logic (e.g., defining axes) for 1D and + // 2D outputs auto &bounds = pmb->cellbounds; - auto ib = bounds.GetBoundsI(IndexDomain::entire); - auto jb = bounds.GetBoundsJ(IndexDomain::entire); - auto kb = bounds.GetBoundsK(IndexDomain::entire); - auto ni = ib.e - ib.s + 1; - auto nj = jb.e - jb.s + 1; - auto nk = kb.e - kb.s + 1; - uint64_t ncells = ni * nj * nk; - - auto ib_int = bounds.GetBoundsI(IndexDomain::interior); - auto jb_int = bounds.GetBoundsJ(IndexDomain::interior); - auto kb_int = bounds.GetBoundsK(IndexDomain::interior); + auto ib = bounds.GetBoundsI(IndexDomain::interior); + auto jb = bounds.GetBoundsJ(IndexDomain::interior); + auto kb = bounds.GetBoundsK(IndexDomain::interior); + uint64_t ni = ib.e - ib.s + 1; + uint64_t nj = jb.e - jb.s + 1; + uint64_t nk = kb.e - kb.s + 1; + uint64_t ncells = ni * nj * nk; auto &coords = pmb->coords; Real dx1 = coords.CellWidth(ib.s, jb.s, kb.s); Real dx2 = coords.CellWidth(ib.s, jb.s, kb.s); Real dx3 = coords.CellWidth(ib.s, jb.s, kb.s); - std::array corner = coords.GetXmin(); - // create the coordinate set - mesh["coordsets/coords/type"] = "uniform"; + // These attributes are shared across all components. + // In general, we should check if there's a performance bottleneck with writing so + // many meshes rather than writing one mesh per level and dump all data there (which + // relies on support for sparse storage by the backend.). + + // TODO(pgrete) check if this should be tied to the MemoryLayout + block_record.setDataOrder(openPMD::Mesh::DataOrder::C); + block_record.setGridSpacing(std::vector{dx3, dx2, dx1}); + block_record.setAxisLabels({"z", "y", "x"}); + std::array corner = coords.GetXmin(); + block_record.setGridGlobalOffset({corner[2], corner[1], corner[0]}); PARTHENON_REQUIRE_THROWS(typeid(Coordinates_t) == typeid(UniformCartesian), "Ascent currently only supports Cartesian coordinates."); - - mesh["coordsets/coords/dims/i"] = ni + 1; - mesh["coordsets/coords/dims/j"] = nj + 1; - if (nk > 1) { - mesh["coordsets/coords/dims/k"] = nk + 1; - } - - // add origin and spacing to the coordset (optional) - mesh["coordsets/coords/origin/x"] = corner[0]; - mesh["coordsets/coords/origin/y"] = corner[1]; - if (nk > 1) { - mesh["coordsets/coords/origin/z"] = corner[2]; - } - - mesh["coordsets/coords/spacing/dx"] = dx1; - mesh["coordsets/coords/spacing/dy"] = dx2; - if (nk > 1) { - mesh["coordsets/coords/spacing/dz"] = dx3; - } + block_record.setGeometry(openPMD::Mesh::Geometry::cartesian); // create a field for each component of each variable pack auto &mbd = pmb->meshblock_data.Get(); for (const auto &var : mbd->GetVariableVector()) { - // ensure that only cell vars are added (for now) as the topology above is only + // ensure that only cell vars are added (for now) as the topology below is only // valid for cell centered vars if (!var->IsSet(Metadata::Cell)) { continue; @@ -154,12 +164,26 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const auto var_info = VarInfo(var); for (int icomp = 0; icomp < var_info.num_components; ++icomp) { - auto const data = Kokkos::subview(var->data, 0, 0, icomp, Kokkos::ALL(), - Kokkos::ALL(), Kokkos::ALL()); + // Pick a subview of the active cells of this component + auto const data = Kokkos::subview( + var->data, 0, 0, icomp, std::make_pair(kb.s, kb.e + 1), + std::make_pair(jb.s, jb.e + 1), std::make_pair(ib.s, ib.e + 1)); + + // Map a view onto a host allocation (so that we can call deep_copy) + auto component_buffer = buffer_list.emplace_back(ncells); + Kokkos::View> + component_buffer_view(component_buffer.data(), nk, nj, ni); + Kokkos::deep_copy(component_buffer_view, data); + const std::string varname = var_info.component_labels.at(icomp); - mesh["fields/" + varname + "/association"] = "element"; - mesh["fields/" + varname + "/topology"] = "topo"; - mesh["fields/" + varname + "/values"].set_external(data.data(), ncells); + auto component_record = block_record[varname]; + // TODO(pgrete) needs to be updated for face and edges etc + component_record.setPosition(std::vector{0.5, 0.5, 0.5}); + auto const dataset = + openPMD::Dataset(openPMD::determineDatatype(), {nk, nj, ni}); + component_record.resetDataset(dataset); + + component_record.storeChunkRaw(component_buffer.data(), {0, 0, 0}, {nk, nj, ni}); } } } diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index ef197177f957..bce7ac4fbffa 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -248,7 +248,7 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { pnew_type = new AscentOutput(op); } else if (op.file_type == "openpmd") { #ifdef PARTHENON_ENABLE_OPENPMD - pnew_type = new OpenPMDOutput(op, pin); + pnew_type = new OpenPMDOutput(op); #else msg << "### FATAL ERROR in Outputs constructor" << std::endl << "Executable not configured for OpenPMD outputs, but OpenPMD file format " From 8d40c91d806504632c05e0aeadeaa483178d642d Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 8 Mar 2024 10:29:44 +0100 Subject: [PATCH 06/70] Centralize getting var info for output --- src/outputs/output_utils.cpp | 28 ++++++++++++++++++++++++++++ src/outputs/output_utils.hpp | 10 ++++++++++ src/outputs/parthenon_hdf5.cpp | 24 +++--------------------- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/src/outputs/output_utils.cpp b/src/outputs/output_utils.cpp index 64c099d85617..baeacf8cbbe2 100644 --- a/src/outputs/output_utils.cpp +++ b/src/outputs/output_utils.cpp @@ -208,5 +208,33 @@ std::size_t MPISum(std::size_t val) { return val; } +VariableVector GetVarsToWrite(const std::shared_ptr pmb, + const bool restart, + const std::vector &variables) { + const auto &var_vec = pmb->meshblock_data.Get()->GetVariableVector(); + auto vars_to_write = GetAnyVariables(var_vec, variables); + if (restart) { + // get all vars with flag Independent OR restart + auto restart_vars = GetAnyVariables( + var_vec, {parthenon::Metadata::Independent, parthenon::Metadata::Restart}); + for (auto restart_var : restart_vars) { + vars_to_write.emplace_back(restart_var); + } + } + return vars_to_write; +} + +std::vector GetAllVarsInfo(const VariableVector &vars) { + std::vector all_vars_info; + for (auto &v : vars) { + all_vars_info.emplace_back(v); + } + + // sort alphabetically + std::sort(all_vars_info.begin(), all_vars_info.end(), + [](const VarInfo &a, const VarInfo &b) { return a.label < b.label; }); + return all_vars_info; +} + } // namespace OutputUtils } // namespace parthenon diff --git a/src/outputs/output_utils.hpp b/src/outputs/output_utils.hpp index db6353090cbf..20c9a8117ff6 100644 --- a/src/outputs/output_utils.hpp +++ b/src/outputs/output_utils.hpp @@ -263,6 +263,16 @@ std::vector ComputeIDsAndFlags(Mesh *pm); std::size_t MPIPrefixSum(std::size_t local, std::size_t &tot_count); std::size_t MPISum(std::size_t local); +// Return all variables to write, i.e., for restarts all indpendent variables and ones +// with explicit Restart flag, but also variables explicitly defined to output in the +// input file. +VariableVector GetVarsToWrite(const std::shared_ptr pmb, + const bool restart, + const std::vector &variables); + +// Returns a sorted vector of VarInfo associated with vars +std::vector GetAllVarsInfo(const VariableVector &vars); + } // namespace OutputUtils } // namespace parthenon diff --git a/src/outputs/parthenon_hdf5.cpp b/src/outputs/parthenon_hdf5.cpp index d7677dfee4fd..21d82a8cfac9 100644 --- a/src/outputs/parthenon_hdf5.cpp +++ b/src/outputs/parthenon_hdf5.cpp @@ -239,28 +239,10 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm // All blocks have the same list of variable metadata that exist in the entire // simulation, but not all variables may be allocated on all blocks - auto get_vars = [=](const std::shared_ptr pmb) { - auto &var_vec = pmb->meshblock_data.Get()->GetVariableVector(); - if (restart_) { - // get all vars with flag Independent OR restart - return GetAnyVariables( - var_vec, {parthenon::Metadata::Independent, parthenon::Metadata::Restart}); - } else { - return GetAnyVariables(var_vec, output_params.variables); - } - }; - // get list of all vars, just use the first block since the list is the same for all // blocks - std::vector all_vars_info; - const auto vars = get_vars(pm->block_list.front()); - for (auto &v : vars) { - all_vars_info.emplace_back(v); - } - - // sort alphabetically - std::sort(all_vars_info.begin(), all_vars_info.end(), - [](const VarInfo &a, const VarInfo &b) { return a.label < b.label; }); + auto all_vars_info = GetAllVarsInfo( + GetVarsToWrite(pm->block_list.front(), restart_, output_params.variables)); // We need to add information about the sparse variables to the HDF5 file, namely: // 1) Which variables are sparse @@ -391,7 +373,7 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm bool is_allocated = false; // for each variable that this local meshblock actually has - const auto vars = get_vars(pmb); + const auto vars = GetVarsToWrite(pmb, restart_, output_params.variables); for (auto &v : vars) { // For reference, if we update the logic here, there's also // a similar block in parthenon_manager.cpp From 4f20c26ebbb922e59d10f105dd653e1b6a1fd769 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 8 Mar 2024 16:36:34 +0100 Subject: [PATCH 07/70] WIP openpmd, chunks don't work yet plus check dimensionality --- src/outputs/openpmd.cpp | 236 +++++++++++++++++++++++---------- src/outputs/parthenon_hdf5.cpp | 30 ++--- 2 files changed, 182 insertions(+), 84 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 2868460ef09c..9d9245a68947 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -17,6 +17,8 @@ //! \file openpmd.cpp // \brief Output for OpenPMD https://www.openpmd.org/ (supporting various backends) +#include +#include #include #include #include @@ -29,11 +31,13 @@ #include // Parthenon headers +#include "basic_types.hpp" #include "coordinates/coordinates.hpp" #include "defs.hpp" #include "globals.hpp" #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" +#include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/IO/Access.hpp" #include "openPMD/Mesh.hpp" @@ -105,65 +109,105 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // the characters a-Z, the numbers 0-9 and the underscore _ const int num_blocks_local = static_cast(pm->block_list.size()); + + // -------------------------------------------------------------------------------- // + // WRITING VARIABLES DATA // + // -------------------------------------------------------------------------------- // + Kokkos::Profiling::pushRegion("write all variable data"); + + // get list of all vars, just use the first block since the list is the same for all + // blocks + // TODO(pgrete) add restart_ var to output + auto all_vars_info = GetAllVarsInfo( + GetVarsToWrite(pm->block_list.front(), true, output_params.variables)); + + // We're currently writing (flushing) one var at a time. This saves host memory but + // results more smaller write. Might be updated in the future. + // Allocate space for largest size variable + int var_size_max = 0; + for (auto &vinfo : all_vars_info) { + const auto var_size = vinfo.Size(); + var_size_max = std::max(var_size_max, var_size); + } + // TODO(pgrete) adjust for single prec output // openPMD::Datatype dtype = openPMD::determineDatatype(); - // using OutT = typename std::conditional::type; - // Need to create the buffer outside so that it's persistent until the data is flushed. - // Dynamically allocating the inner vector should not be a performance bottleneck given - // that the hdf5 backend also uses an explicit per block and per variable - // GetHostMirrorAndCopy, but still this could be optimized. - std::vector> buffer_list; - - for (size_t b_idx = 0; b_idx < num_blocks_local; ++b_idx) { - const auto &pmb = pm->block_list[b_idx]; - // create a unique id for this MeshBlock - const std::string &meshblock_name = "block_" + std::to_string(pmb->gid); - auto block_record = it.meshes[meshblock_name]; - - // TODO(pgrete) check if we should update the logic (e.g., defining axes) for 1D and - // 2D outputs - - auto &bounds = pmb->cellbounds; - auto ib = bounds.GetBoundsI(IndexDomain::interior); - auto jb = bounds.GetBoundsJ(IndexDomain::interior); - auto kb = bounds.GetBoundsK(IndexDomain::interior); - uint64_t ni = ib.e - ib.s + 1; - uint64_t nj = jb.e - jb.s + 1; - uint64_t nk = kb.e - kb.s + 1; - - uint64_t ncells = ni * nj * nk; - auto &coords = pmb->coords; - Real dx1 = coords.CellWidth(ib.s, jb.s, kb.s); - Real dx2 = coords.CellWidth(ib.s, jb.s, kb.s); - Real dx3 = coords.CellWidth(ib.s, jb.s, kb.s); - - // These attributes are shared across all components. - // In general, we should check if there's a performance bottleneck with writing so - // many meshes rather than writing one mesh per level and dump all data there (which - // relies on support for sparse storage by the backend.). - - // TODO(pgrete) check if this should be tied to the MemoryLayout - block_record.setDataOrder(openPMD::Mesh::DataOrder::C); - block_record.setGridSpacing(std::vector{dx3, dx2, dx1}); - block_record.setAxisLabels({"z", "y", "x"}); - std::array corner = coords.GetXmin(); - block_record.setGridGlobalOffset({corner[2], corner[1], corner[0]}); - PARTHENON_REQUIRE_THROWS(typeid(Coordinates_t) == typeid(UniformCartesian), - "Ascent currently only supports Cartesian coordinates."); - block_record.setGeometry(openPMD::Mesh::Geometry::cartesian); - - // create a field for each component of each variable pack - auto &mbd = pmb->meshblock_data.Get(); - - for (const auto &var : mbd->GetVariableVector()) { - // ensure that only cell vars are added (for now) as the topology below is only - // valid for cell centered vars - if (!var->IsSet(Metadata::Cell)) { - continue; + using OutT = + Real; // typename std::conditional::type; + std::vector tmp_data(var_size_max * num_blocks_local); + + // TODO(pgrete) This needs to be in the loop for non-cell-centered vars + auto &bounds = pm->block_list.front()->cellbounds; + auto ib = bounds.GetBoundsI(IndexDomain::interior); + auto jb = bounds.GetBoundsJ(IndexDomain::interior); + auto kb = bounds.GetBoundsK(IndexDomain::interior); + // for each variable we write + for (auto &vinfo : all_vars_info) { + PARTHENON_INSTRUMENT_REGION("Write variable loop") + + // Reset host write bufer. Not really necessary, but doesn't hurt. + memset(tmp_data.data(), 0, tmp_data.size() * sizeof(OutT)); + uint64_t tmp_offset = 0; + + const bool is_scalar = vinfo.nx4 == 1 && vinfo.nx5 == 1 && vinfo.nx6 == 1; + if (vinfo.is_vector) { + // sanity check + PARTHENON_REQUIRE_THROWS( + vinfo.nx4 == pm->ndim && vinfo.nx5 == 1 && vinfo.nx6 == 1, + "A 'standard' vector is expected to only have components matching the " + "dimensionality of the simulation.") + } + + // TODO(pgrete) need to make sure that var names are allowed within standard + const std::string var_name = vinfo.label; + for (auto &pmb : pm->block_list) { + // TODO(pgrete) check if we should skip the suffix for level 0 + const auto level = pmb->loc.level() - pm->GetRootLevel(); + const std::string &mesh_record_name = var_name + "_lvl" + std::to_string(level); + + // Create the mesh_record for this variable at the given level (if it doesn't exist + // yet) + if (!it.meshes.contains(mesh_record_name)) { + auto mesh_record = it.meshes[mesh_record_name]; + + // These following attributes are shared across all components of the record. + + PARTHENON_REQUIRE_THROWS( + typeid(Coordinates_t) == typeid(UniformCartesian), + "OpenPMD in Parthenon currently only supports Cartesian coordinates."); + mesh_record.setGeometry(openPMD::Mesh::Geometry::cartesian); + auto &coords = pmb->coords; + // For uniform Cartesian, all dxN are const across the block so we just pick the + // first index. + Real dx1 = coords.CellWidth(0, 0, 0); + Real dx2 = coords.CellWidth(0, 0, 0); + Real dx3 = coords.CellWidth(0, 0, 0); + + // TODO(pgrete) check if this should be tied to the MemoryLayout + mesh_record.setDataOrder(openPMD::Mesh::DataOrder::C); + mesh_record.setGridSpacing(std::vector{dx3, dx2, dx1}); + mesh_record.setAxisLabels({"z", "y", "x"}); + mesh_record.setGridGlobalOffset({ + pm->mesh_size.xmin(X3DIR), + pm->mesh_size.xmin(X2DIR), + pm->mesh_size.xmin(X1DIR), + }); + + // TODO(pgrete) need unitDimension and timeOffset for this record? } - const auto var_info = VarInfo(var); - for (int icomp = 0; icomp < var_info.num_components; ++icomp) { + auto mesh_record = it.meshes[mesh_record_name]; + + // Now that the mesh record exists, actually write the data + auto out_var = pmb->meshblock_data.Get()->GetVarPtr(var_name); + PARTHENON_REQUIRE_THROWS(out_var->metadata().Where() == + MetadataFlag(Metadata::Cell), + "Currently only cell centered vars are supported."); + + if (out_var->IsAllocated()) { + // TODO(pgrete) check if we can work with a direct copy from a subview to not + // duplicate the memory footprint here +#if 0 // Pick a subview of the active cells of this component auto const data = Kokkos::subview( var->data, 0, 0, icomp, std::make_pair(kb.s, kb.e + 1), @@ -174,25 +218,81 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, Kokkos::View> component_buffer_view(component_buffer.data(), nk, nj, ni); Kokkos::deep_copy(component_buffer_view, data); +#endif + auto out_var_h = out_var->data.GetHostMirrorAndCopy(); + int idx_component = 0; + const auto &Nt = out_var->GetDim(6); + const auto &Nu = out_var->GetDim(5); + const auto &Nv = out_var->GetDim(4); + // loop over all components + for (int t = 0; t < Nt; ++t) { + for (int u = 0; u < Nu; ++u) { + for (int v = 0; v < Nv; ++v) { + // Get the correct record + std::string comp_name; + if (is_scalar) { + comp_name = openPMD::MeshRecordComponent::SCALAR; + } else if (vinfo.is_vector) { + if (v == 0) { + comp_name = "x"; + } else if (v == 1) { + comp_name = "y"; + } else if (v == 2) { + comp_name = "z"; + } else { + PARTHENON_THROW("Expected v index doesn't match vector expectation."); + } + } else { + comp_name = vinfo.component_labels[idx_component]; + } + auto mesh_comp = mesh_record[comp_name]; - const std::string varname = var_info.component_labels.at(icomp); - auto component_record = block_record[varname]; - // TODO(pgrete) needs to be updated for face and edges etc - component_record.setPosition(std::vector{0.5, 0.5, 0.5}); - auto const dataset = - openPMD::Dataset(openPMD::determineDatatype(), {nk, nj, ni}); - component_record.resetDataset(dataset); + // TODO(pgrete) needs to be updated for face and edges etc + mesh_comp.setPosition(std::vector{0.5, 0.5, 0.5}); + // TODO(pgrete) needs to be updated for face and edges etc + // Also this feels wrong for deep hierachies... + auto effective_nx = static_cast(std::pow(2, level)); + openPMD::Extent global_extent = { + static_cast(pm->mesh_size.nx(X3DIR)) * effective_nx, + static_cast(pm->mesh_size.nx(X2DIR)) * effective_nx, + static_cast(pm->mesh_size.nx(X1DIR)) * effective_nx, + }; + auto const dataset = + openPMD::Dataset(openPMD::determineDatatype(), global_extent); + mesh_comp.resetDataset(dataset); + + const auto comp_offset = tmp_offset; + for (int k = kb.s; k <= kb.e; ++k) { + for (int j = jb.s; j <= jb.e; ++j) { + for (int i = ib.s; i <= ib.e; ++i) { + tmp_data[tmp_offset] = static_cast(out_var_h(t, u, v, k, j, i)); + tmp_offset++; + } + } + } + openPMD::Offset chunk_offset = { + pmb->loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), + pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), + pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; + openPMD::Extent chunk_extent = {static_cast(vinfo.nx3), + static_cast(vinfo.nx2), + static_cast(vinfo.nx1)}; + mesh_comp.storeChunkRaw(&tmp_data[comp_offset], chunk_offset, chunk_extent); + idx_component += 1; + } + } + } // loop over components + } // out_var->IsAllocated() + } // loop over blocks + it.seriesFlush(); + } // loop over vars + Kokkos::Profiling::popRegion(); // write all variable data - component_record.storeChunkRaw(component_buffer.data(), {0, 0, 0}, {nk, nj, ni}); - } - } - } // The iteration can be closed in order to help free up resources. // The iteration's content will be flushed automatically. // An iteration once closed cannot (yet) be reopened. it.close(); - // No need to close series as it's done in the desctructor of the object when it runs - // out of scope. + series.close(); #endif // ifndef PARTHENON_ENABLE_OPENPMD // advance output parameters diff --git a/src/outputs/parthenon_hdf5.cpp b/src/outputs/parthenon_hdf5.cpp index 21d82a8cfac9..75c61cc76196 100644 --- a/src/outputs/parthenon_hdf5.cpp +++ b/src/outputs/parthenon_hdf5.cpp @@ -372,22 +372,20 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm const auto &pmb = pm->block_list[b_idx]; bool is_allocated = false; - // for each variable that this local meshblock actually has - const auto vars = GetVarsToWrite(pmb, restart_, output_params.variables); - for (auto &v : vars) { - // For reference, if we update the logic here, there's also - // a similar block in parthenon_manager.cpp - if (v->IsAllocated() && (var_name == v->label())) { - auto v_h = v->data.GetHostMirrorAndCopy(); - OutputUtils::PackOrUnpackVar( - pmb.get(), v.get(), output_params.include_ghost_zones, index, tmpData, - [&](auto index, int t, int u, int v, int k, int j, int i) { - tmpData[index] = static_cast(v_h(t, u, v, k, j, i)); - }); - - is_allocated = true; - break; - } + // TODO(reviewers) Why was the loop originally there? Does the direct Get causes + // issue? + auto v = pmb->meshblock_data.Get()->GetVarPtr(var_name); + // For reference, if we update the logic here, there's also + // a similar block in parthenon_manager.cpp + if (v->IsAllocated() && (var_name == v->label())) { + auto v_h = v->data.GetHostMirrorAndCopy(); + OutputUtils::PackOrUnpackVar( + pmb.get(), v.get(), output_params.include_ghost_zones, index, tmpData, + [&](auto index, int t, int u, int v, int k, int j, int i) { + tmpData[index] = static_cast(v_h(t, u, v, k, j, i)); + }); + + is_allocated = true; } if (vinfo.is_sparse) { From f29e8d1d85bc7f4dca742f80c6f6bf4376748f2f Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 14 Mar 2024 15:23:46 +0100 Subject: [PATCH 08/70] Fix chunk extents --- src/outputs/openpmd.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 9d9245a68947..1702a6503a7e 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -274,9 +274,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, pmb->loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; - openPMD::Extent chunk_extent = {static_cast(vinfo.nx3), - static_cast(vinfo.nx2), - static_cast(vinfo.nx1)}; + openPMD::Extent chunk_extent = { + static_cast(pmb->block_size.nx(X3DIR)), + static_cast(pmb->block_size.nx(X2DIR)), + static_cast(pmb->block_size.nx(X1DIR))}; mesh_comp.storeChunkRaw(&tmp_data[comp_offset], chunk_offset, chunk_extent); idx_component += 1; } From a501a5dc8c2b90d2ebd36923830c0556f3ddd833 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 15 Mar 2024 12:31:12 +0100 Subject: [PATCH 09/70] Write Attributes --- src/outputs/openpmd.cpp | 86 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 3 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 1702a6503a7e..da04b1123c38 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -34,6 +34,7 @@ #include "basic_types.hpp" #include "coordinates/coordinates.hpp" #include "defs.hpp" +#include "driver/driver.hpp" #include "globals.hpp" #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" @@ -47,6 +48,7 @@ #include "outputs/outputs.hpp" #include "parthenon_array_generic.hpp" #include "utils/error_checking.hpp" +#include "utils/instrument.hpp" // OpenPMD headers #ifdef PARTHENON_ENABLE_OPENPMD @@ -76,7 +78,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // but an interation) This just describes the pattern of the filename. The correct file // will be accessed through the iteration idx below. The file suffix maps to the chosen // backend. - Series series = Series("adios_test_%05T.bp", Access::CREATE); + Series series = Series("opmd.%05T.bp", Access::CREATE); // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per @@ -101,8 +103,86 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, auto it = series.iterations[output_params.file_number]; it.open(); // explicit open() is important when run in parallel - it.setTime(tm->time); - it.setDt(tm->dt); + auto const &first_block = *(pm->block_list.front()); + + // TODO(?) in principle, we could abstract this to a more general WriteAttributes place + // and reuse for hdf5 and OpenPMD output with corresponing calls + // -------------------------------------------------------------------------------- // + // WRITING ATTRIBUTES // + // -------------------------------------------------------------------------------- // + + // Note, that profiling is likely skewed as data is actually written to disk/flushed + // only later. + Kokkos::Profiling::pushRegion("write Attributes"); + // First the ones required by the OpenPMD standard + if (tm != nullptr) { + it.setTime(tm->time); + it.setDt(tm->dt); + it.setAttribute("NCycle", tm->ncycle); + } else { + it.setTime(-1.0); + it.setDt(-1.0); + } + // Then our own + { + PARTHENON_INSTRUMENT_REGION("write input"); + // write input key-value pairs + std::ostringstream oss; + pin->ParameterDump(oss); + it.setAttribute("InputFile", oss.str()); + } + + { + it.setAttribute("WallTime", Driver::elapsed_main()); + it.setAttribute("NumDims", pm->ndim); + it.setAttribute("NumMeshBlocks", pm->nbtotal); + it.setAttribute("MaxLevel", pm->GetCurrentLevel() - pm->GetRootLevel()); + // write whether we include ghost cells or not + it.setAttribute("IncludesGhost", output_params.include_ghost_zones ? 1 : 0); + // write number of ghost cells in simulation + it.setAttribute("NGhost", Globals::nghost); + it.setAttribute("Coordinates", std::string(first_block.coords.Name()).c_str()); + + // restart info, write always + it.setAttribute("NBNew", pm->nbnew); + it.setAttribute("NBDel", pm->nbdel); + it.setAttribute("RootLevel", pm->GetRootLevel()); + it.setAttribute("Refine", pm->adaptive ? 1 : 0); + it.setAttribute("Multilevel", pm->multilevel ? 1 : 0); + + it.setAttribute("BlocksPerPE", pm->GetNbList()); + + // Mesh block size + const auto base_block_size = pm->GetBlockSize(); + it.setAttribute("MeshBlockSize", + std::vector{base_block_size.nx(X1DIR), base_block_size.nx(X2DIR), + base_block_size.nx(X3DIR)}); + + // RootGridDomain - float[9] array with xyz mins, maxs, rats (dx(i)/dx(i-1)) + it.setAttribute( + "RootGridDomain", + std::vector{pm->mesh_size.xmin(X1DIR), pm->mesh_size.xmax(X1DIR), + pm->mesh_size.xrat(X1DIR), pm->mesh_size.xmin(X2DIR), + pm->mesh_size.xmax(X2DIR), pm->mesh_size.xrat(X2DIR), + pm->mesh_size.xmin(X3DIR), pm->mesh_size.xmax(X3DIR), + pm->mesh_size.xrat(X3DIR)}); + + // Root grid size (number of cells at root level) + it.setAttribute("RootGridSize", + std::vector{pm->mesh_size.nx(X1DIR), pm->mesh_size.nx(X2DIR), + pm->mesh_size.nx(X3DIR)}); + + // Boundary conditions + std::vector boundary_condition_str(BOUNDARY_NFACES); + for (size_t i = 0; i < boundary_condition_str.size(); i++) { + boundary_condition_str[i] = GetBoundaryString(pm->mesh_bcs[i]); + } + + it.setAttribute("BoundaryConditions", boundary_condition_str); + Kokkos::Profiling::popRegion(); // write Info + } // Info section + + Kokkos::Profiling::popRegion(); // write Attributes // TODO(pgrete) check var name standard compatiblity // e.g., description: names of records and their components are only allowed to contain From c0be75c483d42a9ca386e3b754fe311edc2f6587 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 15 Mar 2024 14:50:52 +0100 Subject: [PATCH 10/70] Rename restart to restart_hdf5 --- src/outputs/{restart.cpp => restart_hdf5.cpp} | 0 src/outputs/{restart.hpp => restart_hdf5.hpp} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/outputs/{restart.cpp => restart_hdf5.cpp} (100%) rename src/outputs/{restart.hpp => restart_hdf5.hpp} (100%) diff --git a/src/outputs/restart.cpp b/src/outputs/restart_hdf5.cpp similarity index 100% rename from src/outputs/restart.cpp rename to src/outputs/restart_hdf5.cpp diff --git a/src/outputs/restart.hpp b/src/outputs/restart_hdf5.hpp similarity index 100% rename from src/outputs/restart.hpp rename to src/outputs/restart_hdf5.hpp From 7f03528da05b04e879fa0309293141b611128079 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 18 Mar 2024 15:47:29 +0100 Subject: [PATCH 11/70] WIP abstract RestartReader --- src/outputs/openpmd.cpp | 10 ++++++++++ src/outputs/restart_hdf5.hpp | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index da04b1123c38..818b1e2fe78f 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -31,6 +31,7 @@ #include // Parthenon headers +#include "Kokkos_Core_fwd.hpp" #include "basic_types.hpp" #include "coordinates/coordinates.hpp" #include "defs.hpp" @@ -123,6 +124,15 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, it.setTime(-1.0); it.setDt(-1.0); } + { // TESTING REMOVE + const auto view_d = Kokkos::View("blub", 5,3); + // Map a view onto a host allocation (so that we can call deep_copy) + auto host_vec = std::vector(view_d.size()); + Kokkos::View> + view_h(host_vec.data(), view_d.extent_int(0), view_d.extent_int(1)); + Kokkos::deep_copy(view_h, view_d); + it.setAttribute("blub", host_vec); + } // Then our own { PARTHENON_INSTRUMENT_REGION("write input"); diff --git a/src/outputs/restart_hdf5.hpp b/src/outputs/restart_hdf5.hpp index 69c301d30702..013800fc5d59 100644 --- a/src/outputs/restart_hdf5.hpp +++ b/src/outputs/restart_hdf5.hpp @@ -14,8 +14,8 @@ // license in this material to reproduce, prepare derivative works, distribute copies to // the public, perform publicly and display publicly, and to permit others to do so. //======================================================================================== -#ifndef OUTPUTS_RESTART_HPP_ -#define OUTPUTS_RESTART_HPP_ +#ifndef OUTPUTS_RESTART_HDF5_HPP_ +#define OUTPUTS_RESTART_HDF5_HPP_ //! \file io_wrapper.hpp // \brief defines a set of small wrapper functions for MPI versus Serial Output. @@ -342,4 +342,4 @@ class RestartReader { }; } // namespace parthenon -#endif // OUTPUTS_RESTART_HPP_ +#endif // OUTPUTS_RESTART_HDF5_HPP_ From 56795f2658f6f6b49088c3add2887bd56dbf1208 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 18 Mar 2024 23:39:15 +0100 Subject: [PATCH 12/70] WIP separating RestartReader --- src/CMakeLists.txt | 4 +- src/mesh/mesh.cpp | 1 + src/outputs/restart.hpp | 115 +++++++++++++++++++++++++++++++++++ src/outputs/restart_hdf5.cpp | 13 ++-- src/outputs/restart_hdf5.hpp | 5 +- src/parthenon_manager.cpp | 11 +++- 6 files changed, 139 insertions(+), 10 deletions(-) create mode 100644 src/outputs/restart.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e54b86d9be58..458fd6fea56e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -192,7 +192,9 @@ add_library(parthenon outputs/parthenon_xdmf.cpp outputs/parthenon_hdf5.hpp outputs/parthenon_xdmf.hpp - outputs/restart.cpp + outputs/restart.hpp + outputs/restart_hdf5.cpp + outputs/restart_hdf5.hpp outputs/vtk.cpp parthenon/driver.hpp diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index cee6af316fc8..2f327628c0d7 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -48,6 +48,7 @@ #include "mesh/meshblock.hpp" #include "mesh/meshblock_tree.hpp" #include "outputs/restart.hpp" +#include "outputs/restart_hdf5.hpp" #include "parameter_input.hpp" #include "parthenon_arrays.hpp" #include "prolong_restrict/prolong_restrict.hpp" diff --git a/src/outputs/restart.hpp b/src/outputs/restart.hpp new file mode 100644 index 000000000000..57cb8f10de48 --- /dev/null +++ b/src/outputs/restart.hpp @@ -0,0 +1,115 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2020-2024 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +// (C) (or copyright) 2020-2021. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== +#ifndef OUTPUTS_RESTART_HPP_ +#define OUTPUTS_RESTART_HPP_ +//! \file io_wrapper.hpp +// \brief defines a set of small wrapper functions for MPI versus Serial Output. + +#include +#include +#include +#include +#include + +#include "mesh/domain.hpp" +#include "utils/error_checking.hpp" + +namespace parthenon { + +class Mesh; +class Param; + +class RestartReader { + public: + RestartReader(); + virtual ~RestartReader() = default; + + struct SparseInfo { + // labels of sparse fields (full label, i.e. base name and sparse id) + std::vector labels; + + // allocation status of sparse fields (2D array outer dimension: block, inner + // dimension: sparse field) + // can't use std::vector here because std::vector is the same as + // std::vector and it doesn't have .data() member + std::unique_ptr allocated; + + int num_blocks = 0; + int num_sparse = 0; + + bool IsAllocated(int block, int sparse_field_idx) const { + PARTHENON_REQUIRE_THROWS(allocated != nullptr, + "Tried to get allocation status but no data present"); + PARTHENON_REQUIRE_THROWS((block >= 0) && (block < num_blocks), + "Invalid block index in SparseInfo::IsAllocated"); + PARTHENON_REQUIRE_THROWS((sparse_field_idx >= 0) && (sparse_field_idx < num_sparse), + "Invalid sparse field index in SparseInfo::IsAllocated"); + + return allocated[block * num_sparse + sparse_field_idx]; + } + }; + + SparseInfo GetSparseInfo() const; + + // Return output format version number. Return -1 if not existent. + int GetOutputFormatVersion() const; + + public: + // Gets data for all blocks on current rank. + // Assumes blocks are contiguous + // fills internal data for given pointer + template + void ReadBlocks(const std::string &name, IndexRange range, std::vector &dataVec, + const std::vector &bsize, int file_output_format_version, + MetadataFlag where, const std::vector &shape = {}) const; + + // Gets the data from a swarm var on current rank. Assumes all + // blocks are contiguous. Fills dataVec based on shape from swarmvar + // metadata. + template + void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, const Metadata &m, + std::vector &dataVec); + + // Reads an array dataset from file as a 1D vector. + template + std::vector ReadDataset(const std::string &name) const; + + template + std::vector GetAttrVec(const std::string &location, const std::string &name) const; + + template + T GetAttr(const std::string &location, const std::string &name) const; + + // Gets the counts and offsets for MPI ranks for the meshblocks set + // by the indexrange. Returns the total count on this rank. + std::size_t GetSwarmCounts(const std::string &swarm, const IndexRange &range, + std::vector &counts, + std::vector &offsets); + + void ReadParams(const std::string &name, Params &p); + + // closes out the restart file + // perhaps belongs in a destructor? + void Close(); + + // Does file have ghost cells? + int hasGhost; +}; + +} // namespace parthenon +#endif // OUTPUTS_RESTART_HDF5_HPP_ diff --git a/src/outputs/restart_hdf5.cpp b/src/outputs/restart_hdf5.cpp index d986d233bd1f..67eb99b1df85 100644 --- a/src/outputs/restart_hdf5.cpp +++ b/src/outputs/restart_hdf5.cpp @@ -1,6 +1,6 @@ //======================================================================================== // Parthenon performance portable AMR framework -// Copyright(C) 2020-2022 The Parthenon collaboration +// Copyright(C) 2020-2024 The Parthenon collaboration // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== // (C) (or copyright) 2020-2021. Triad National Security, LLC. All rights reserved. @@ -31,6 +31,7 @@ #include "outputs/parthenon_hdf5.hpp" #endif #include "outputs/restart.hpp" +#include "outputs/restart_hdf5.hpp" #include "utils/error_checking.hpp" namespace parthenon { @@ -38,7 +39,7 @@ namespace parthenon { //---------------------------------------------------------------------------------------- //! \fn void RestartReader::RestartReader(const std::string filename) // \brief Opens the restart file and stores appropriate file handle in fh_ -RestartReader::RestartReader(const char *filename) : filename_(filename) { +RestartReaderHDF5::RestartReaderHDF5(const char *filename) : filename_(filename) { #ifndef ENABLE_HDF5 std::stringstream msg; msg << "### FATAL ERROR in Restart (Reader) constructor" << std::endl @@ -54,7 +55,7 @@ RestartReader::RestartReader(const char *filename) : filename_(filename) { #endif // ENABLE_HDF5 } -int RestartReader::GetOutputFormatVersion() const { +int RestartReaderHDF5::GetOutputFormatVersion() const { #ifndef ENABLE_HDF5 PARTHENON_FAIL("Restart functionality is not available because HDF5 is disabled"); #else // HDF5 enabled @@ -69,7 +70,7 @@ int RestartReader::GetOutputFormatVersion() const { #endif // ENABLE_HDF5 } -RestartReader::SparseInfo RestartReader::GetSparseInfo() const { +RestartReaderHDF5::SparseInfo RestartReaderHDF5::GetSparseInfo() const { #ifndef ENABLE_HDF5 PARTHENON_FAIL("Restart functionality is not available because HDF5 is disabled"); #else // HDF5 enabled @@ -107,7 +108,7 @@ RestartReader::SparseInfo RestartReader::GetSparseInfo() const { // Gets the counts and offsets for MPI ranks for the meshblocks set // by the indexrange. Returns the total count on this rank. -std::size_t RestartReader::GetSwarmCounts(const std::string &swarm, +std::size_t RestartReaderHDF5::GetSwarmCounts(const std::string &swarm, const IndexRange &range, std::vector &counts, std::vector &offsets) { @@ -141,7 +142,7 @@ std::size_t RestartReader::GetSwarmCounts(const std::string &swarm, #endif // ENABLE_HDF5 } -void RestartReader::ReadParams(const std::string &name, Params &p) { +void RestartReaderHDF5::ReadParams(const std::string &name, Params &p) { #ifdef ENABLE_HDF5 p.ReadFromRestart(name, params_group_); #endif // ENABLE_HDF5 diff --git a/src/outputs/restart_hdf5.hpp b/src/outputs/restart_hdf5.hpp index 013800fc5d59..5b0f43fa4091 100644 --- a/src/outputs/restart_hdf5.hpp +++ b/src/outputs/restart_hdf5.hpp @@ -26,6 +26,7 @@ #include #include "config.hpp" +#include "outputs/restart.hpp" #ifdef ENABLE_HDF5 #include @@ -54,9 +55,9 @@ namespace parthenon { class Mesh; class Param; -class RestartReader { +class RestartReaderHDF5 : public RestartReader{ public: - explicit RestartReader(const char *theFile); + RestartReaderHDF5(const char *theFile); struct SparseInfo { // labels of sparse fields (full label, i.e. base name and sparse id) diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index 814a8c6f094f..853eb989d69e 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -29,6 +29,9 @@ #include "amr_criteria/refinement_package.hpp" #include "config.hpp" #include "driver/driver.hpp" +#include "outputs/restart.hpp" +#include "outputs/restart_hdf5.hpp" +#include FS_HEADER #include "globals.hpp" #include "interface/update.hpp" #include "mesh/domain.hpp" @@ -38,6 +41,8 @@ #include "utils/error_checking.hpp" #include "utils/utils.hpp" +namespace fs = FS_NAMESPACE; + namespace parthenon { ParthenonStatus ParthenonManager::ParthenonInitEnv(int argc, char *argv[]) { @@ -100,7 +105,11 @@ ParthenonStatus ParthenonManager::ParthenonInitEnv(int argc, char *argv[]) { pinput = std::make_unique(arg.input_filename); } else if (arg.res_flag != 0) { // Read input from restart file - restartReader = std::make_unique(arg.restart_filename); + if (fs::path(arg.restart_filename).extension() == ".rhdf") { + restartReader = std::make_unique(arg.restart_filename); + } else { + PARTHENON_FAIL("HELP!"); + } // Load input stream pinput = std::make_unique(); From 8587303c06426c2448730c09dfa22c45cf0ac689 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 19 Mar 2024 16:29:12 +0100 Subject: [PATCH 13/70] Make RestartReader abstract --- src/mesh/mesh.cpp | 35 +++++++++---------- src/outputs/restart.hpp | 62 +++++++++++++++++++-------------- src/outputs/restart_hdf5.cpp | 37 ++++++++++++++++++-- src/outputs/restart_hdf5.hpp | 67 ++++++++++++++++-------------------- src/parthenon_manager.cpp | 9 ++--- 5 files changed, 121 insertions(+), 89 deletions(-) diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index 2f327628c0d7..edd010ef72df 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -104,8 +104,8 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, // private members: num_mesh_threads_(pin->GetOrAddInteger("parthenon/mesh", "num_threads", 1)), tree(this), use_uniform_meshgen_fn_{true, true, true, true}, lb_flag_(true), - lb_automatic_(), - lb_manual_(), MeshBndryFnctn{nullptr, nullptr, nullptr, nullptr, nullptr, nullptr} { + lb_automatic_(), lb_manual_(), + MeshBndryFnctn{nullptr, nullptr, nullptr, nullptr, nullptr, nullptr} { std::stringstream msg; RegionSize block_size; BoundaryFlag block_bcs[6]; @@ -486,8 +486,8 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr, // private members: num_mesh_threads_(pin->GetOrAddInteger("parthenon/mesh", "num_threads", 1)), tree(this), use_uniform_meshgen_fn_{true, true, true, true}, lb_flag_(true), - lb_automatic_(), - lb_manual_(), MeshBndryFnctn{nullptr, nullptr, nullptr, nullptr, nullptr, nullptr} { + lb_automatic_(), lb_manual_(), + MeshBndryFnctn{nullptr, nullptr, nullptr, nullptr, nullptr, nullptr} { std::stringstream msg; RegionSize block_size; BoundaryFlag block_bcs[6]; @@ -506,13 +506,14 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr, // read the restart file // the file is already open and the pointer is set to after + auto mesh_info = rr.GetMeshInfo(); // All ranks read HDF file - nbnew = rr.GetAttr("Info", "NBNew"); - nbdel = rr.GetAttr("Info", "NBDel"); - nbtotal = rr.GetAttr("Info", "NumMeshBlocks"); - root_level = rr.GetAttr("Info", "RootLevel"); + nbnew = mesh_info.nbnew; + nbdel = mesh_info.nbdel; + nbtotal = mesh_info.nbtotal; + root_level = mesh_info.root_level; - const auto bc = rr.GetAttrVec("Info", "BoundaryConditions"); + const auto bc = mesh_info.bound_cond; for (int i = 0; i < 6; i++) { block_bcs[i] = GetBoundaryFlag(bc[i]); } @@ -538,7 +539,7 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr, } EnrollBndryFncts_(app_in); - const auto grid_dim = rr.GetAttrVec("Info", "RootGridDomain"); + const auto grid_dim = mesh_info.grid_dim; mesh_size.xmin(X1DIR) = grid_dim[0]; mesh_size.xmax(X1DIR) = grid_dim[1]; mesh_size.xrat(X1DIR) = grid_dim[2]; @@ -554,14 +555,11 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr, // initialize loclist = std::vector(nbtotal); - const auto blockSize = rr.GetAttrVec("Info", "MeshBlockSize"); - const auto includesGhost = rr.GetAttr("Info", "IncludesGhost"); - const auto nGhost = rr.GetAttr("Info", "NGhost"); - for (auto &dir : {X1DIR, X2DIR, X3DIR}) { block_size.xrat(dir) = mesh_size.xrat(dir); - block_size.nx(dir) = - blockSize[dir - 1] - (blockSize[dir - 1] > 1) * includesGhost * 2 * nGhost; + block_size.nx(dir) = mesh_info.block_size[dir - 1] - + (mesh_info.block_size[dir - 1] > 1) * mesh_info.includes_ghost * + 2 * mesh_info.n_ghost; if (block_size.nx(dir) == 1) { block_size.symmetry(dir) = true; mesh_size.symmetry(dir) = true; @@ -595,9 +593,8 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr, InitUserMeshData(this, pin); // Populate logical locations - auto lx123 = rr.ReadDataset("/Blocks/loc.lx123"); - auto locLevelGidLidCnghostGflag = - rr.ReadDataset("/Blocks/loc.level-gid-lid-cnghost-gflag"); + auto lx123 = mesh_info.lx123; + auto locLevelGidLidCnghostGflag = mesh_info.level_gid_lid_cnghost_gflag; current_level = -1; for (int i = 0; i < nbtotal; i++) { loclist[i] = LogicalLocation(locLevelGidLidCnghostGflag[5 * i], lx123[3 * i], diff --git a/src/outputs/restart.hpp b/src/outputs/restart.hpp index 57cb8f10de48..2ee20afc01dc 100644 --- a/src/outputs/restart.hpp +++ b/src/outputs/restart.hpp @@ -35,7 +35,7 @@ class Param; class RestartReader { public: - RestartReader(); + RestartReader() = default; virtual ~RestartReader() = default; struct SparseInfo { @@ -63,45 +63,55 @@ class RestartReader { } }; - SparseInfo GetSparseInfo() const; + [[nodiscard]] virtual SparseInfo GetSparseInfo() const = 0; + + struct MeshInfo { + int nbnew, nbdel, nbtotal, root_level, includes_ghost, n_ghost; + std::vector bound_cond; + std::vector block_size; + std::vector grid_dim; + std::vector lx123; + std::vector level_gid_lid_cnghost_gflag; // what's this?! + }; + [[nodiscard]] virtual MeshInfo GetMeshInfo() const = 0; + + struct TimeInfo { + Real time, dt; + int ncycle; + }; + [[nodiscard]] virtual TimeInfo GetTimeInfo() const = 0; + + [[nodiscard]] virtual std::string GetInputString() const = 0; // Return output format version number. Return -1 if not existent. - int GetOutputFormatVersion() const; + [[nodiscard]] virtual int GetOutputFormatVersion() const = 0; - public: // Gets data for all blocks on current rank. // Assumes blocks are contiguous // fills internal data for given pointer - template - void ReadBlocks(const std::string &name, IndexRange range, std::vector &dataVec, - const std::vector &bsize, int file_output_format_version, - MetadataFlag where, const std::vector &shape = {}) const; + virtual void ReadBlocks(const std::string &name, IndexRange range, + std::vector &dataVec, const std::vector &bsize, + int file_output_format_version, MetadataFlag where, + const std::vector &shape = {}) const = 0; // Gets the data from a swarm var on current rank. Assumes all // blocks are contiguous. Fills dataVec based on shape from swarmvar // metadata. - template - void ReadSwarmVar(const std::string &swarmname, const std::string &varname, - const std::size_t count, const std::size_t offset, const Metadata &m, - std::vector &dataVec); - - // Reads an array dataset from file as a 1D vector. - template - std::vector ReadDataset(const std::string &name) const; - - template - std::vector GetAttrVec(const std::string &location, const std::string &name) const; - - template - T GetAttr(const std::string &location, const std::string &name) const; + virtual void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, + const Metadata &m, std::vector &dataVec) = 0; + virtual void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, + const Metadata &m, std::vector &dataVec) = 0; // Gets the counts and offsets for MPI ranks for the meshblocks set // by the indexrange. Returns the total count on this rank. - std::size_t GetSwarmCounts(const std::string &swarm, const IndexRange &range, - std::vector &counts, - std::vector &offsets); + [[nodiscard]] virtual std::size_t GetSwarmCounts(const std::string &swarm, + const IndexRange &range, + std::vector &counts, + std::vector &offsets) = 0; - void ReadParams(const std::string &name, Params &p); + virtual void ReadParams(const std::string &name, Params &p) = 0; // closes out the restart file // perhaps belongs in a destructor? diff --git a/src/outputs/restart_hdf5.cpp b/src/outputs/restart_hdf5.cpp index 67eb99b1df85..ceba14d72309 100644 --- a/src/outputs/restart_hdf5.cpp +++ b/src/outputs/restart_hdf5.cpp @@ -106,12 +106,43 @@ RestartReaderHDF5::SparseInfo RestartReaderHDF5::GetSparseInfo() const { #endif // ENABLE_HDF5 } +RestartReaderHDF5::MeshInfo RestartReaderHDF5::GetMeshInfo() const { + RestartReaderHDF5::MeshInfo mesh_info; + mesh_info.nbnew = GetAttr("Info", "NBNew"); + mesh_info.nbdel = GetAttr("Info", "NBDel"); + mesh_info.nbtotal = GetAttr("Info", "NumMeshBlocks"); + mesh_info.root_level = GetAttr("Info", "RootLevel"); + + mesh_info.bound_cond = GetAttrVec("Info", "BoundaryConditions"); + + mesh_info.block_size = GetAttrVec("Info", "MeshBlockSize"); + mesh_info.includes_ghost = GetAttr("Info", "IncludesGhost"); + mesh_info.n_ghost = GetAttr("Info", "NGhost"); + + mesh_info.grid_dim = GetAttrVec("Info", "RootGridDomain"); + + mesh_info.lx123 = ReadDataset("/Blocks/loc.lx123"); + mesh_info.level_gid_lid_cnghost_gflag = + ReadDataset("/Blocks/loc.level-gid-lid-cnghost-gflag"); + + return mesh_info; +} + +RestartReaderHDF5::TimeInfo RestartReaderHDF5::GetTimeInfo() const { + RestartReaderHDF5::TimeInfo time_info; + + time_info.time = GetAttr("Info", "Time"); + time_info.dt = GetAttr("Info", "dt"); + time_info.ncycle = GetAttr("Info", "NCycle"); + + return time_info; +} // Gets the counts and offsets for MPI ranks for the meshblocks set // by the indexrange. Returns the total count on this rank. std::size_t RestartReaderHDF5::GetSwarmCounts(const std::string &swarm, - const IndexRange &range, - std::vector &counts, - std::vector &offsets) { + const IndexRange &range, + std::vector &counts, + std::vector &offsets) { #ifndef ENABLE_HDF5 PARTHENON_FAIL("Restart functionality is not available because HDF5 is disabled"); return 0; diff --git a/src/outputs/restart_hdf5.hpp b/src/outputs/restart_hdf5.hpp index 5b0f43fa4091..65eafdec1c57 100644 --- a/src/outputs/restart_hdf5.hpp +++ b/src/outputs/restart_hdf5.hpp @@ -55,39 +55,22 @@ namespace parthenon { class Mesh; class Param; -class RestartReaderHDF5 : public RestartReader{ +class RestartReaderHDF5 : public RestartReader { public: - RestartReaderHDF5(const char *theFile); - - struct SparseInfo { - // labels of sparse fields (full label, i.e. base name and sparse id) - std::vector labels; - - // allocation status of sparse fields (2D array outer dimension: block, inner - // dimension: sparse field) - // can't use std::vector here because std::vector is the same as - // std::vector and it doesn't have .data() member - std::unique_ptr allocated; - - int num_blocks = 0; - int num_sparse = 0; - - bool IsAllocated(int block, int sparse_field_idx) const { - PARTHENON_REQUIRE_THROWS(allocated != nullptr, - "Tried to get allocation status but no data present"); - PARTHENON_REQUIRE_THROWS((block >= 0) && (block < num_blocks), - "Invalid block index in SparseInfo::IsAllocated"); - PARTHENON_REQUIRE_THROWS((sparse_field_idx >= 0) && (sparse_field_idx < num_sparse), - "Invalid sparse field index in SparseInfo::IsAllocated"); - - return allocated[block * num_sparse + sparse_field_idx]; - } - }; + RestartReaderHDF5(const char *filename); + + [[nodiscard]] SparseInfo GetSparseInfo() const override; + + [[nodiscard]] MeshInfo GetMeshInfo() const override; - SparseInfo GetSparseInfo() const; + [[nodiscard]] TimeInfo GetTimeInfo() const override; + + [[nodiscard]] std::string GetInputString() const override { + return GetAttr("Input", "File"); + }; // Return output format version number. Return -1 if not existent. - int GetOutputFormatVersion() const; + [[nodiscard]] int GetOutputFormatVersion() const override; private: struct DatasetHandle { @@ -144,14 +127,13 @@ class RestartReaderHDF5 : public RestartReader{ // Gets data for all blocks on current rank. // Assumes blocks are contiguous // fills internal data for given pointer - template - void ReadBlocks(const std::string &name, IndexRange range, std::vector &dataVec, + void ReadBlocks(const std::string &name, IndexRange range, std::vector &dataVec, const std::vector &bsize, int file_output_format_version, - MetadataFlag where, const std::vector &shape = {}) const { + MetadataFlag where, const std::vector &shape = {}) const override { #ifndef ENABLE_HDF5 PARTHENON_FAIL("Restart functionality is not available because HDF5 is disabled"); #else // HDF5 enabled - auto hdl = OpenDataset(name); + auto hdl = OpenDataset(name); constexpr int CHUNK_MAX_DIM = 7; @@ -315,14 +297,25 @@ class RestartReaderHDF5 : public RestartReader{ return res[0]; } + void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, const Metadata &m, + std::vector &dataVec) override { + ReadSwarmVar<>(swarmname, varname, count, offset, m, dataVec); + }; + void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, const Metadata &m, + std::vector &dataVec) override { + ReadSwarmVar<>(swarmname, varname, count, offset, m, dataVec); + }; // Gets the counts and offsets for MPI ranks for the meshblocks set // by the indexrange. Returns the total count on this rank. - std::size_t GetSwarmCounts(const std::string &swarm, const IndexRange &range, - std::vector &counts, - std::vector &offsets); + [[nodiscard]] std::size_t GetSwarmCounts(const std::string &swarm, + const IndexRange &range, + std::vector &counts, + std::vector &offsets) override; - void ReadParams(const std::string &name, Params &p); + void ReadParams(const std::string &name, Params &p) override; // closes out the restart file // perhaps belongs in a destructor? diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index 853eb989d69e..dadfde12dcab 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -113,7 +113,7 @@ ParthenonStatus ParthenonManager::ParthenonInitEnv(int argc, char *argv[]) { // Load input stream pinput = std::make_unique(); - auto inputString = restartReader->GetAttr("Input", "File"); + auto inputString = restartReader->GetInputString(); std::istringstream is(inputString); pinput->LoadFromStream(is); } @@ -176,13 +176,14 @@ void ParthenonManager::ParthenonInitPackagesAndMesh() { std::make_unique(pinput.get(), app_input.get(), *restartReader, packages); // Read simulation time and cycle from restart file and set in input - Real tNow = restartReader->GetAttr("Info", "Time"); + const auto time_info = restartReader->GetTimeInfo(); + Real tNow = time_info.time; pinput->SetReal("parthenon/time", "start_time", tNow); - Real dt = restartReader->GetAttr("Info", "dt"); + Real dt = time_info.dt; pinput->SetReal("parthenon/time", "dt", dt); - int ncycle = restartReader->GetAttr("Info", "NCycle"); + int ncycle = time_info.ncycle; pinput->SetInteger("parthenon/time", "ncycle", ncycle); // Read package data from restart file From e3ea8d796307e0b6b9e113f120fc14db77d03978 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 20 Mar 2024 14:46:19 +0100 Subject: [PATCH 14/70] Add OpenPMD restart skeleton --- src/CMakeLists.txt | 2 + src/outputs/restart_opmd.cpp | 94 ++++++++++++++++++++++++++++++++++++ src/outputs/restart_opmd.hpp | 79 ++++++++++++++++++++++++++++++ src/parthenon_manager.cpp | 3 ++ 4 files changed, 178 insertions(+) create mode 100644 src/outputs/restart_opmd.cpp create mode 100644 src/outputs/restart_opmd.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 458fd6fea56e..9174a5dd0be6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -195,6 +195,8 @@ add_library(parthenon outputs/restart.hpp outputs/restart_hdf5.cpp outputs/restart_hdf5.hpp + outputs/restart_opmd.cpp + outputs/restart_opmd.hpp outputs/vtk.cpp parthenon/driver.hpp diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp new file mode 100644 index 000000000000..48b27cc4e46c --- /dev/null +++ b/src/outputs/restart_opmd.cpp @@ -0,0 +1,94 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2024 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file restart_opmd.cpp +// \brief Restarts a simulation from an OpenPMD output with ADIOS2 backend + +#include +#include +#include + +#include "interface/params.hpp" +#include "openPMD/Iteration.hpp" +#include "openPMD/Series.hpp" +#include "outputs/restart.hpp" +#include "outputs/restart_opmd.hpp" +#include "utils/error_checking.hpp" + +namespace parthenon { + +//---------------------------------------------------------------------------------------- +//! \fn void RestartReader::RestartReader(const std::string filename) +// \brief Opens the restart file and stores appropriate file handle in fh_ +RestartReaderOPMD::RestartReaderOPMD(const char *filename) : filename_(filename) { + auto series = openPMD::Series(filename, openPMD::Access::READ_ONLY, MPI_COMM_WORLD); + PARTHENON_REQUIRE_THROWS( + series.iterations.size() == 1, + "Parthenon restarts should only contain one iteration/timestep."); + unsigned long idx; + for (const auto &i : series.iterations) { + idx = i.first; + } + it = std::make_unique(series.iterations[idx]); +} + +int RestartReaderOPMD::GetOutputFormatVersion() const { + // TODO(pgrete) move info to shared header and introduce constexpr var + if (it->containsAttribute("OutputFormatVersion")) { + return it->getAttribute("OutputFormatVersion").get(); + } else { + return -1; + } +} + +RestartReaderOPMD::SparseInfo RestartReaderOPMD::GetSparseInfo() const {} + +RestartReaderOPMD::MeshInfo RestartReaderOPMD::GetMeshInfo() const { + RestartReaderOPMD::MeshInfo mesh_info; + mesh_info.nbnew = it->getAttribute("NBNew").get(); + mesh_info.nbdel = it->getAttribute("NBDel").get(); + mesh_info.nbtotal = it->getAttribute("NumMeshBlocks").get(); + mesh_info.root_level = it->getAttribute("RootLevel").get(); + + mesh_info.bound_cond = + it->getAttribute("BoundaryConditions").get>(); + + mesh_info.block_size = it->getAttribute("MeshBlockSize").get>(); + mesh_info.includes_ghost = it->getAttribute("IncludesGhost").get(); + mesh_info.n_ghost = it->getAttribute("NGhost").get(); + + mesh_info.grid_dim = it->getAttribute("RootGridDomain").get>(); + // TODO(pgrete) need impl + // mesh_info.lx123 = ReadDataset("/Blocks/loc.lx123"); + // mesh_info.level_gid_lid_cnghost_gflag = + // ReadDataset("/Blocks/loc.level-gid-lid-cnghost-gflag"); + + return mesh_info; +} + +RestartReaderOPMD::TimeInfo RestartReaderOPMD::GetTimeInfo() const { + RestartReaderOPMD::TimeInfo time_info; + + time_info.time = it->time(); + time_info.dt = it->dt(); + time_info.ncycle = it->getAttribute("NCycle").get(); + + return time_info; +} +// Gets the counts and offsets for MPI ranks for the meshblocks set +// by the indexrange. Returns the total count on this rank. +std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, + const IndexRange &range, + std::vector &counts, + std::vector &offsets) {} + +void RestartReaderOPMD::ReadParams(const std::string &name, Params &p) {} +void RestartReaderOPMD::ReadBlocks(const std::string &name, IndexRange range, + std::vector &dataVec, + const std::vector &bsize, + int file_output_format_version, MetadataFlag where, + const std::vector &shape) const {} + +} // namespace parthenon diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp new file mode 100644 index 000000000000..ef4d5e9251c6 --- /dev/null +++ b/src/outputs/restart_opmd.hpp @@ -0,0 +1,79 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2024 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +#ifndef OUTPUTS_RESTART_OPMD_HPP_ +#define OUTPUTS_RESTART_OPMD_HPP_ +//! \file restart_opmd.hpp +// \brief Provides support for restarting from OpenPMD output + +#include +#include +#include + +#include "openPMD/Iteration.hpp" +#include "outputs/restart.hpp" + +#include "mesh/domain.hpp" + +namespace parthenon { + +class Mesh; +class Param; + +class RestartReaderOPMD : public RestartReader { + public: + explicit RestartReaderOPMD(const char *filename); + + [[nodiscard]] SparseInfo GetSparseInfo() const override; + + [[nodiscard]] MeshInfo GetMeshInfo() const override; + + [[nodiscard]] TimeInfo GetTimeInfo() const override; + + [[nodiscard]] std::string GetInputString() const override { + return it->getAttribute("InputFile").get(); + }; + + // Return output format version number. Return -1 if not existent. + [[nodiscard]] int GetOutputFormatVersion() const override; + + public: + // Gets data for all blocks on current rank. + // Assumes blocks are contiguous + // fills internal data for given pointer + void ReadBlocks(const std::string &name, IndexRange range, std::vector &dataVec, + const std::vector &bsize, int file_output_format_version, + MetadataFlag where, const std::vector &shape = {}) const override; + + void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, const Metadata &m, + std::vector &dataVec) override{}; + void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, const Metadata &m, + std::vector &dataVec) override{}; + + // Gets the counts and offsets for MPI ranks for the meshblocks set + // by the indexrange. Returns the total count on this rank. + [[nodiscard]] std::size_t GetSwarmCounts(const std::string &swarm, + const IndexRange &range, + std::vector &counts, + std::vector &offsets) override; + + void ReadParams(const std::string &name, Params &p) override; + + // closes out the restart file + // perhaps belongs in a destructor? + void Close(); + + // Does file have ghost cells? + int hasGhost; + + private: + const std::string filename_; + std::unique_ptr it; +}; + +} // namespace parthenon +#endif // OUTPUTS_RESTART_OPMD_HPP_ diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index 28a084bcd5e0..8507acd3d3fe 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -35,6 +35,7 @@ #include "outputs/output_utils.hpp" #include "outputs/restart.hpp" #include "outputs/restart_hdf5.hpp" +#include "outputs/restart_opmd.hpp" #include "utils/error_checking.hpp" #include "utils/utils.hpp" @@ -104,6 +105,8 @@ ParthenonStatus ParthenonManager::ParthenonInitEnv(int argc, char *argv[]) { // Read input from restart file if (fs::path(arg.restart_filename).extension() == ".rhdf") { restartReader = std::make_unique(arg.restart_filename); + } else if (fs::path(arg.restart_filename).extension() == ".bp") { + restartReader = std::make_unique(arg.restart_filename); } else { PARTHENON_FAIL("Unsupported restart file format."); } From 33b6261ed2f148d9df69cf2eb8d5c57094386d29 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 20 Mar 2024 16:32:25 +0100 Subject: [PATCH 15/70] WIP updating loc logic --- src/outputs/openpmd.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 818b1e2fe78f..253c9579e39f 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -124,8 +124,9 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, it.setTime(-1.0); it.setDt(-1.0); } - { // TESTING REMOVE - const auto view_d = Kokkos::View("blub", 5,3); + { // FIXME move this to dump params + const auto view_d = + Kokkos::View("blub", 5, 3); // Map a view onto a host allocation (so that we can call deep_copy) auto host_vec = std::vector(view_d.size()); Kokkos::View> @@ -194,6 +195,15 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, Kokkos::Profiling::popRegion(); // write Attributes + // Write block metadata + { + // TODO(pgrete) FIXME this is collective, so should be a dataset! + const int n_blocks_global = pm->nbtotal; + const int n_blocks_local = static_cast(pm->block_list.size()); + std::vector loc = OutputUtils::ComputeLocs(pm); + it.setAttribute("loc.lx123", loc); + } + // TODO(pgrete) check var name standard compatiblity // e.g., description: names of records and their components are only allowed to contain // the characters a-Z, the numbers 0-9 and the underscore _ From 62e54da614cbb39f3b849b29e7f8b47b25e1b1e4 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 11 Apr 2024 10:50:29 +0200 Subject: [PATCH 16/70] Fix interface from recent changes in develop --- src/outputs/openpmd.cpp | 13 ++++++++----- src/outputs/output_utils.cpp | 5 +++-- src/outputs/output_utils.hpp | 3 ++- src/outputs/restart_opmd.cpp | 4 ++-- src/outputs/restart_opmd.hpp | 3 ++- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 555df89cee66..b31d63148348 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -144,6 +144,8 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } { + // It's not clear we need all these attributes, but they mirror what's done in the + // hdf5 output. it.setAttribute("WallTime", Driver::elapsed_main()); it.setAttribute("NumDims", pm->ndim); it.setAttribute("NumMeshBlocks", pm->nbtotal); @@ -157,7 +159,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // restart info, write always it.setAttribute("NBNew", pm->nbnew); it.setAttribute("NBDel", pm->nbdel); - it.setAttribute("RootLevel", pm->GetRootLevel()); + it.setAttribute("RootLevel", pm->GetLegacyTreeRootLevel()); it.setAttribute("Refine", pm->adaptive ? 1 : 0); it.setAttribute("Multilevel", pm->multilevel ? 1 : 0); @@ -215,12 +217,13 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // -------------------------------------------------------------------------------- // Kokkos::Profiling::pushRegion("write all variable data"); + auto &bounds = pm->block_list.front()->cellbounds; // get list of all vars, just use the first block since the list is the same for all // blocks // TODO(pgrete) add restart_ var to output // TODO(pgrete) check if this needs to be updated/unifed with get_var logic in hdf5 auto all_vars_info = GetAllVarsInfo( - GetVarsToWrite(pm->block_list.front(), true, output_params.variables)); + GetVarsToWrite(pm->block_list.front(), true, output_params.variables), bounds); // We're currently writing (flushing) one var at a time. This saves host memory but // results more smaller write. Might be updated in the future. @@ -238,7 +241,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, std::vector tmp_data(var_size_max * num_blocks_local); // TODO(pgrete) This needs to be in the loop for non-cell-centered vars - auto &bounds = pm->block_list.front()->cellbounds; auto ib = bounds.GetBoundsI(IndexDomain::interior); auto jb = bounds.GetBoundsJ(IndexDomain::interior); auto kb = bounds.GetBoundsK(IndexDomain::interior); @@ -250,11 +252,12 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, memset(tmp_data.data(), 0, tmp_data.size() * sizeof(OutT)); uint64_t tmp_offset = 0; - const bool is_scalar = vinfo.nx4 == 1 && vinfo.nx5 == 1 && vinfo.nx6 == 1; + const bool is_scalar = + vinfo.GetDim(4) == 1 && vinfo.GetDim(5) == 1 && vinfo.GetDim(6) == 1; if (vinfo.is_vector) { // sanity check PARTHENON_REQUIRE_THROWS( - vinfo.nx4 == pm->ndim && vinfo.nx5 == 1 && vinfo.nx6 == 1, + vinfo.GetDim(4) == pm->ndim && vinfo.GetDim(5) == 1 && vinfo.GetDim(6) == 1, "A 'standard' vector is expected to only have components matching the " "dimensionality of the simulation.") } diff --git a/src/outputs/output_utils.cpp b/src/outputs/output_utils.cpp index a33dc03395a1..8b5d2e1c3cfe 100644 --- a/src/outputs/output_utils.cpp +++ b/src/outputs/output_utils.cpp @@ -329,10 +329,11 @@ VariableVector GetVarsToWrite(const std::shared_ptr pmb, return vars_to_write; } -std::vector GetAllVarsInfo(const VariableVector &vars) { +std::vector GetAllVarsInfo(const VariableVector &vars, + const IndexShape &cellbounds) { std::vector all_vars_info; for (auto &v : vars) { - all_vars_info.emplace_back(v); + all_vars_info.emplace_back(v, cellbounds); } // sort alphabetically diff --git a/src/outputs/output_utils.hpp b/src/outputs/output_utils.hpp index 2199edbc2e45..a6b4929e453d 100644 --- a/src/outputs/output_utils.hpp +++ b/src/outputs/output_utils.hpp @@ -354,7 +354,8 @@ VariableVector GetVarsToWrite(const std::shared_ptr pmb, const std::vector &variables); // Returns a sorted vector of VarInfo associated with vars -std::vector GetAllVarsInfo(const VariableVector &vars); +std::vector GetAllVarsInfo(const VariableVector &vars, + const IndexShape &cellbounds); } // namespace OutputUtils } // namespace parthenon diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 48b27cc4e46c..7e1732ea6743 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -68,8 +68,8 @@ RestartReaderOPMD::MeshInfo RestartReaderOPMD::GetMeshInfo() const { return mesh_info; } -RestartReaderOPMD::TimeInfo RestartReaderOPMD::GetTimeInfo() const { - RestartReaderOPMD::TimeInfo time_info; +SimTime RestartReaderOPMD::GetTimeInfo() const { + SimTime time_info{}; time_info.time = it->time(); time_info.dt = it->dt(); diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index ef4d5e9251c6..e795fe5123bd 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -12,6 +12,7 @@ #include #include +#include "basic_types.hpp" #include "openPMD/Iteration.hpp" #include "outputs/restart.hpp" @@ -30,7 +31,7 @@ class RestartReaderOPMD : public RestartReader { [[nodiscard]] MeshInfo GetMeshInfo() const override; - [[nodiscard]] TimeInfo GetTimeInfo() const override; + [[nodiscard]] SimTime GetTimeInfo() const override; [[nodiscard]] std::string GetInputString() const override { return it->getAttribute("InputFile").get(); From 63096633e52ac9cc561805150dae3a1bb5c1db9a Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 11 Apr 2024 13:52:43 +0200 Subject: [PATCH 17/70] Read and Write loc --- src/outputs/openpmd.cpp | 15 ++++++++++----- src/outputs/output_utils.cpp | 34 ++++++++++++++++++++++++++++++++++ src/outputs/output_utils.hpp | 5 +++++ src/outputs/restart_opmd.cpp | 8 ++++---- src/utils/mpi_types.hpp | 9 +++++++++ 5 files changed, 62 insertions(+), 9 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index b31d63148348..36eb4e84fbda 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -199,11 +199,16 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // Write block metadata { - // TODO(pgrete) FIXME this is collective, so should be a dataset! - const int n_blocks_global = pm->nbtotal; - const int n_blocks_local = static_cast(pm->block_list.size()); - std::vector loc = OutputUtils::ComputeLocs(pm); - it.setAttribute("loc.lx123", loc); + // Manually gather all block data first as it allows to use the (simpler) + // Attribute interface rather than writing a distributed dataset -- especially as all + // data is being read on restart by every rank anyway. + std::vector loc_local = OutputUtils::ComputeLocs(pm); + auto loc_global = FlattendedLocalToGlobal(pm, loc_local); + it.setAttribute("loc.lx123", loc_global); + + std::vector id_local = OutputUtils::ComputeIDsAndFlags(pm); + auto id_global = FlattendedLocalToGlobal(pm, id_local); + it.setAttribute("loc.level-gid-lid-cnghost-gflag", id_global); } // TODO(pgrete) check var name standard compatiblity diff --git a/src/outputs/output_utils.cpp b/src/outputs/output_utils.cpp index 8b5d2e1c3cfe..0e081744cd43 100644 --- a/src/outputs/output_utils.cpp +++ b/src/outputs/output_utils.cpp @@ -29,6 +29,7 @@ #include "mesh/mesh.hpp" #include "mesh/meshblock.hpp" #include "outputs/output_utils.hpp" +#include "utils/mpi_types.hpp" namespace parthenon { namespace OutputUtils { @@ -241,6 +242,39 @@ std::vector ComputeIDsAndFlags(Mesh *pm) { }); } +template +std::vector FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_local) { + const int n_blocks_global = pm->nbtotal; + const int n_blocks_local = static_cast(pm->block_list.size()); + + const int n_elem = data_local.size() / n_blocks_local; + PARTHENON_REQUIRE_THROWS(data_local.size() % n_blocks_local == 0, + "Results from flattened input vector does not evenly divide " + "into number of local blocks."); + std::vector data_global(n_elem * n_blocks_global); + + std::vector counts(Globals::nranks); + std::vector offsets(Globals::nranks); + + const auto &nblist = pm->GetNbList(); + counts[0] = n_elem * nblist[0]; + offsets[0] = 0; + for (int r = 1; r < Globals::nranks; r++) { + counts[r] = n_elem * nblist[r]; + offsets[r] = offsets[r - 1] + counts[r - 1]; + } + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allgatherv(data_local.data(), counts[Globals::my_rank], + MPITypeMap::type(), data_global.data(), + counts.data(), offsets.data(), MPITypeMap::type(), + MPI_COMM_WORLD)); +#else + return data_local; +#endif + return data_global; +} + // TODO(JMM): I could make this use the other loop // functionality/high-order functions. but it was more code than this // for, I think, little benefit. diff --git a/src/outputs/output_utils.hpp b/src/outputs/output_utils.hpp index a6b4929e453d..4f5c920da920 100644 --- a/src/outputs/output_utils.hpp +++ b/src/outputs/output_utils.hpp @@ -339,6 +339,11 @@ std::vector ComputeXminBlocks(Mesh *pm); std::vector ComputeLocs(Mesh *pm); std::vector ComputeIDsAndFlags(Mesh *pm); +// Takes a vector containing flattened data of all rank local blocks and returns the +// flattened data over all blocks. +template +std::vector FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_local); + // TODO(JMM): Potentially unsafe if MPI_UNSIGNED_LONG_LONG isn't a size_t // however I think it's probably safe to assume we'll be on systems // where this is the case? diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 7e1732ea6743..f49bdf47b56d 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include "interface/params.hpp" #include "openPMD/Iteration.hpp" @@ -60,10 +61,9 @@ RestartReaderOPMD::MeshInfo RestartReaderOPMD::GetMeshInfo() const { mesh_info.n_ghost = it->getAttribute("NGhost").get(); mesh_info.grid_dim = it->getAttribute("RootGridDomain").get>(); - // TODO(pgrete) need impl - // mesh_info.lx123 = ReadDataset("/Blocks/loc.lx123"); - // mesh_info.level_gid_lid_cnghost_gflag = - // ReadDataset("/Blocks/loc.level-gid-lid-cnghost-gflag"); + mesh_info.lx123 = it->getAttribute("loc.lx123").get>(); + mesh_info.level_gid_lid_cnghost_gflag = + it->getAttribute("loc.level-gid-lid-cnghost-gflag").get>(); return mesh_info; } diff --git a/src/utils/mpi_types.hpp b/src/utils/mpi_types.hpp index a990cecc7e3c..164ab66f1115 100644 --- a/src/utils/mpi_types.hpp +++ b/src/utils/mpi_types.hpp @@ -1,4 +1,8 @@ //======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2021-2024 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== // (C) (or copyright) 2021. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los @@ -34,6 +38,11 @@ inline MPI_Datatype MPITypeMap::type() { return MPI_PARTHENON_REAL; } +template <> +inline MPI_Datatype MPITypeMap::type() { + return MPI_INT64_T; +} + template <> inline MPI_Datatype MPITypeMap::type() { return MPI_INT; From 7224f42b703d8c2c2a2cb1f55c6bab9d16382178 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 11 Apr 2024 14:11:47 +0200 Subject: [PATCH 18/70] Houston, we have a build --- src/outputs/openpmd.cpp | 4 ++-- src/outputs/output_utils.cpp | 7 +++++++ src/outputs/restart_opmd.cpp | 15 ++++++++++----- src/outputs/restart_opmd.hpp | 6 +++--- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 36eb4e84fbda..7e5b88f022a8 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -203,11 +203,11 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // Attribute interface rather than writing a distributed dataset -- especially as all // data is being read on restart by every rank anyway. std::vector loc_local = OutputUtils::ComputeLocs(pm); - auto loc_global = FlattendedLocalToGlobal(pm, loc_local); + auto loc_global = FlattendedLocalToGlobal(pm, loc_local); it.setAttribute("loc.lx123", loc_global); std::vector id_local = OutputUtils::ComputeIDsAndFlags(pm); - auto id_global = FlattendedLocalToGlobal(pm, id_local); + auto id_global = FlattendedLocalToGlobal(pm, id_local); it.setAttribute("loc.level-gid-lid-cnghost-gflag", id_global); } diff --git a/src/outputs/output_utils.cpp b/src/outputs/output_utils.cpp index 0e081744cd43..a94a4160d789 100644 --- a/src/outputs/output_utils.cpp +++ b/src/outputs/output_utils.cpp @@ -15,6 +15,7 @@ // the public, perform publicly and display publicly, and to permit others to do so. //======================================================================================== +#include #include #include #include @@ -275,6 +276,12 @@ std::vector FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_loca return data_global; } +// explicit template instantiation +template std::vector +FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_local); +template std::vector FlattendedLocalToGlobal(Mesh *pm, + const std::vector &data_local); + // TODO(JMM): I could make this use the other loop // functionality/high-order functions. but it was more code than this // for, I think, little benefit. diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index f49bdf47b56d..d94da08e3332 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -44,7 +44,10 @@ int RestartReaderOPMD::GetOutputFormatVersion() const { } } -RestartReaderOPMD::SparseInfo RestartReaderOPMD::GetSparseInfo() const {} +RestartReaderOPMD::SparseInfo RestartReaderOPMD::GetSparseInfo() const { + // TODO(pgrete) needs impl + return {}; +} RestartReaderOPMD::MeshInfo RestartReaderOPMD::GetMeshInfo() const { RestartReaderOPMD::MeshInfo mesh_info; @@ -82,13 +85,15 @@ SimTime RestartReaderOPMD::GetTimeInfo() const { std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, const IndexRange &range, std::vector &counts, - std::vector &offsets) {} + std::vector &offsets) { + // TODO(pgrete) needs impl + return 0; +} void RestartReaderOPMD::ReadParams(const std::string &name, Params &p) {} void RestartReaderOPMD::ReadBlocks(const std::string &name, IndexRange range, + const OutputUtils::VarInfo &info, std::vector &dataVec, - const std::vector &bsize, - int file_output_format_version, MetadataFlag where, - const std::vector &shape) const {} + int file_output_format_version) const {}; } // namespace parthenon diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index e795fe5123bd..46a5658eb031 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -44,9 +44,9 @@ class RestartReaderOPMD : public RestartReader { // Gets data for all blocks on current rank. // Assumes blocks are contiguous // fills internal data for given pointer - void ReadBlocks(const std::string &name, IndexRange range, std::vector &dataVec, - const std::vector &bsize, int file_output_format_version, - MetadataFlag where, const std::vector &shape = {}) const override; + void ReadBlocks(const std::string &name, IndexRange range, + const OutputUtils::VarInfo &info, std::vector &dataVec, + int file_output_format_version) const override; void ReadSwarmVar(const std::string &swarmname, const std::string &varname, const std::size_t count, const std::size_t offset, const Metadata &m, From ae1f2417f92188d050f2d92a65278ea5833512f8 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 12 Apr 2024 09:34:06 +0200 Subject: [PATCH 19/70] Added OpenPMD restart ReadBlocks --- src/outputs/restart.hpp | 2 +- src/outputs/restart_hdf5.cpp | 4 +- src/outputs/restart_hdf5.hpp | 2 +- src/outputs/restart_opmd.cpp | 73 ++++++++++++++++++++++++++++++++++-- src/outputs/restart_opmd.hpp | 2 +- src/parthenon_manager.cpp | 2 +- 6 files changed, 76 insertions(+), 9 deletions(-) diff --git a/src/outputs/restart.hpp b/src/outputs/restart.hpp index e986acbaf028..9d452a786e07 100644 --- a/src/outputs/restart.hpp +++ b/src/outputs/restart.hpp @@ -89,7 +89,7 @@ class RestartReader { // fills internal data for given pointer virtual void ReadBlocks(const std::string &name, IndexRange range, const OutputUtils::VarInfo &info, std::vector &dataVec, - int file_output_format_version) const = 0; + int file_output_format_version, Mesh *pmesh) const = 0; // Gets the data from a swarm var on current rank. Assumes all // blocks are contiguous. Fills dataVec based on shape from swarmvar diff --git a/src/outputs/restart_hdf5.cpp b/src/outputs/restart_hdf5.cpp index a5016bc68e8d..8d83cbdf78fe 100644 --- a/src/outputs/restart_hdf5.cpp +++ b/src/outputs/restart_hdf5.cpp @@ -183,7 +183,8 @@ void RestartReaderHDF5::ReadParams(const std::string &name, Params &p) { void RestartReaderHDF5::ReadBlocks(const std::string &name, IndexRange range, const OutputUtils::VarInfo &info, std::vector &dataVec, - int file_output_format_version) const { + int file_output_format_version, + Mesh * /*pmesh*/) const { #ifndef ENABLE_HDF5 PARTHENON_FAIL("Restart functionality is not available because HDF5 is disabled"); #else // HDF5 enabled @@ -224,6 +225,7 @@ void RestartReaderHDF5::ReadBlocks(const std::string &name, IndexRange range, H5Sselect_hyperslab(hdl.dataspace, H5S_SELECT_SET, offset, NULL, count, NULL)); const H5S memspace = H5S::FromHIDCheck(H5Screate_simple(total_dim, count, NULL)); + // TODO(reviewer) What's going on here? The follow line is identical to the one above. PARTHENON_HDF5_CHECK( H5Sselect_hyperslab(hdl.dataspace, H5S_SELECT_SET, offset, NULL, count, NULL)); diff --git a/src/outputs/restart_hdf5.hpp b/src/outputs/restart_hdf5.hpp index 562b67daf78b..f43fbd58df84 100644 --- a/src/outputs/restart_hdf5.hpp +++ b/src/outputs/restart_hdf5.hpp @@ -113,7 +113,7 @@ class RestartReaderHDF5 : public RestartReader { // fills internal data for given pointer void ReadBlocks(const std::string &name, IndexRange range, const OutputUtils::VarInfo &info, std::vector &dataVec, - int file_output_format_version) const override; + int file_output_format_version, Mesh *pmesh) const override; // Gets the data from a swarm var on current rank. Assumes all // blocks are contiguous. Fills dataVec based on shape from swarmvar diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index d94da08e3332..e7bbbaec533f 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -11,6 +11,7 @@ #include #include +#include "basic_types.hpp" #include "interface/params.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" @@ -91,9 +92,73 @@ std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, } void RestartReaderOPMD::ReadParams(const std::string &name, Params &p) {} -void RestartReaderOPMD::ReadBlocks(const std::string &name, IndexRange range, - const OutputUtils::VarInfo &info, - std::vector &dataVec, - int file_output_format_version) const {}; +void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, + const OutputUtils::VarInfo &vinfo, + std::vector &data_vec, + int file_output_format_version, Mesh *pm) const { + for (auto &pmb : pm->block_list) { + // TODO(pgrete) check if we should skip the suffix for level 0 + const auto level = pmb->loc.level() - pm->GetRootLevel(); + const std::string &mesh_record_name = var_name + "_lvl" + std::to_string(level); + + PARTHENON_REQUIRE_THROWS(it->meshes.contains(mesh_record_name), + "Missing mesh record in restart file."); + auto mesh_record = it->meshes[mesh_record_name]; + + int64_t comp_offset = 0; // offset data_vector to store component data + int idx_component = 0; // used in label for non-vector variables + const bool is_scalar = + vinfo.GetDim(4) == 1 && vinfo.GetDim(5) == 1 && vinfo.GetDim(6) == 1; + const auto &Nt = vinfo.GetDim(6); + const auto &Nu = vinfo.GetDim(5); + const auto &Nv = vinfo.GetDim(4); + // loop over all components + for (int t = 0; t < Nt; ++t) { + for (int u = 0; u < Nu; ++u) { + for (int v = 0; v < Nv; ++v) { + // Get the correct record + std::string comp_name; + if (is_scalar) { + comp_name = openPMD::MeshRecordComponent::SCALAR; + } else if (vinfo.is_vector) { + if (v == 0) { + comp_name = "x"; + } else if (v == 1) { + comp_name = "y"; + } else if (v == 2) { + comp_name = "z"; + } else { + PARTHENON_THROW("Expected v index doesn't match vector expectation."); + } + } else { + comp_name = vinfo.component_labels[idx_component]; + } + PARTHENON_REQUIRE_THROWS(mesh_record.contains(comp_name), + "Missing component in mesh record of restart file."); + auto mesh_comp = mesh_record[comp_name]; + + openPMD::Offset chunk_offset = { + pmb->loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), + pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), + pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; + openPMD::Extent chunk_extent = { + static_cast(pmb->block_size.nx(X3DIR)), + static_cast(pmb->block_size.nx(X2DIR)), + static_cast(pmb->block_size.nx(X1DIR))}; + mesh_comp.loadChunkRaw(&data_vec[comp_offset], chunk_extent, chunk_extent); + // TODO(pgrete) check if output utils machinery can be used for non-cell + // centered fields, which might not be that straightforward as a global mesh is + // stored rather than individual blocks. + comp_offset += pmb->block_size.nx(X1DIR) * pmb->block_size.nx(X2DIR) * + pmb->block_size.nx(X3DIR); + idx_component += 1; + } + } + } // loop over components + } // loop over blocks + + // Now actually read the registered chunks form disk + it->seriesFlush(); +} } // namespace parthenon diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index 46a5658eb031..b1a41699a609 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -46,7 +46,7 @@ class RestartReaderOPMD : public RestartReader { // fills internal data for given pointer void ReadBlocks(const std::string &name, IndexRange range, const OutputUtils::VarInfo &info, std::vector &dataVec, - int file_output_format_version) const override; + int file_output_format_version, Mesh *pmesh) const override; void ReadSwarmVar(const std::string &swarmname, const std::string &varname, const std::size_t count, const std::size_t offset, const Metadata &m, diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index ed35c2afcfbf..edb1e4c2dcce 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -326,7 +326,7 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { } // Read relevant data from the hdf file, this works for dense and sparse variables try { - resfile.ReadBlocks(label, myBlocks, v_info, tmp, file_output_format_ver); + resfile.ReadBlocks(label, myBlocks, v_info, tmp, file_output_format_ver, &rm); } catch (std::exception &ex) { std::cout << "[" << Globals::my_rank << "] WARNING: Failed to read variable " << label << " from restart file:" << std::endl From 19863d73fe9efc8d831f5d5479b0c7ef447c51df Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 12 Apr 2024 16:55:53 +0200 Subject: [PATCH 20/70] Fix loc level --- src/outputs/restart_opmd.cpp | 36 +++++++++++++++++++++++++++++++----- src/parthenon_manager.cpp | 3 ++- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index e7bbbaec533f..c5d98fb0479b 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -91,18 +91,42 @@ std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, return 0; } -void RestartReaderOPMD::ReadParams(const std::string &name, Params &p) {} +void RestartReaderOPMD::ReadParams(const std::string &name, Params &p) { +#if 0 + // views and vecs of scalar types + ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); + ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); + ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); + ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); + ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); + ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); + ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); + + // strings + ReadFromHDF5AllParamsOfType(prefix, group); + ReadFromHDF5AllParamsOfType>(prefix, group); + + template +void call_my_func(my_list ) +{ + (myFunc(), ...); +} +#endif +} void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, const OutputUtils::VarInfo &vinfo, std::vector &data_vec, int file_output_format_version, Mesh *pm) const { for (auto &pmb : pm->block_list) { // TODO(pgrete) check if we should skip the suffix for level 0 - const auto level = pmb->loc.level() - pm->GetRootLevel(); + // TODO(pgrete) ask LR why this is not mirrored from writing + // const auto level = pmb->loc.level() - pm->GetRootLevel(); + const auto level = pmb->loc.level(); const std::string &mesh_record_name = var_name + "_lvl" + std::to_string(level); PARTHENON_REQUIRE_THROWS(it->meshes.contains(mesh_record_name), - "Missing mesh record in restart file."); + "Missing mesh record '" + mesh_record_name + + "' in restart file."); auto mesh_record = it->meshes[mesh_record_name]; int64_t comp_offset = 0; // offset data_vector to store component data @@ -134,7 +158,9 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block comp_name = vinfo.component_labels[idx_component]; } PARTHENON_REQUIRE_THROWS(mesh_record.contains(comp_name), - "Missing component in mesh record of restart file."); + "Missing component'" + comp_name + + "' in mesh record '" + mesh_record_name + + "' of restart file."); auto mesh_comp = mesh_record[comp_name]; openPMD::Offset chunk_offset = { @@ -145,7 +171,7 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block static_cast(pmb->block_size.nx(X3DIR)), static_cast(pmb->block_size.nx(X2DIR)), static_cast(pmb->block_size.nx(X1DIR))}; - mesh_comp.loadChunkRaw(&data_vec[comp_offset], chunk_extent, chunk_extent); + mesh_comp.loadChunkRaw(&data_vec[comp_offset], chunk_offset, chunk_extent); // TODO(pgrete) check if output utils machinery can be used for non-cell // centered fields, which might not be that straightforward as a global mesh is // stored rather than individual blocks. diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index edb1e4c2dcce..d77ae9377b15 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -257,7 +257,8 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { // Currently supports versions 3 and 4. const auto file_output_format_ver = resfile.GetOutputFormatVersion(); - if (file_output_format_ver < HDF5::OUTPUT_VERSION_FORMAT - 1) { + // TODO(pgrete) figure out what to do about versions of different outputs + if (false && file_output_format_ver < HDF5::OUTPUT_VERSION_FORMAT - 1) { std::stringstream msg; msg << "File format version " << file_output_format_ver << " not supported. " << "Current format is " << HDF5::OUTPUT_VERSION_FORMAT << std::endl; From e2b2bd19420277b677fc9d3f6e88434d90696c06 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 15 Apr 2024 12:17:34 +0200 Subject: [PATCH 21/70] Make Series persistent and fix rootlevel typo --- src/outputs/openpmd.cpp | 2 +- src/outputs/restart_opmd.cpp | 9 ++++----- src/outputs/restart_opmd.hpp | 4 ++++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 7e5b88f022a8..27dc4d6627a7 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -159,7 +159,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // restart info, write always it.setAttribute("NBNew", pm->nbnew); it.setAttribute("NBDel", pm->nbdel); - it.setAttribute("RootLevel", pm->GetLegacyTreeRootLevel()); + it.setAttribute("RootLevel", pm->GetRootLevel()); it.setAttribute("Refine", pm->adaptive ? 1 : 0); it.setAttribute("Multilevel", pm->multilevel ? 1 : 0); diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index c5d98fb0479b..d14c5710ee39 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -6,6 +6,7 @@ //! \file restart_opmd.cpp // \brief Restarts a simulation from an OpenPMD output with ADIOS2 backend +#include #include #include #include @@ -24,8 +25,8 @@ namespace parthenon { //---------------------------------------------------------------------------------------- //! \fn void RestartReader::RestartReader(const std::string filename) // \brief Opens the restart file and stores appropriate file handle in fh_ -RestartReaderOPMD::RestartReaderOPMD(const char *filename) : filename_(filename) { - auto series = openPMD::Series(filename, openPMD::Access::READ_ONLY, MPI_COMM_WORLD); +RestartReaderOPMD::RestartReaderOPMD(const char *filename) + : filename_(filename), series(filename, openPMD::Access::READ_ONLY, MPI_COMM_WORLD) { PARTHENON_REQUIRE_THROWS( series.iterations.size() == 1, "Parthenon restarts should only contain one iteration/timestep."); @@ -119,9 +120,7 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block int file_output_format_version, Mesh *pm) const { for (auto &pmb : pm->block_list) { // TODO(pgrete) check if we should skip the suffix for level 0 - // TODO(pgrete) ask LR why this is not mirrored from writing - // const auto level = pmb->loc.level() - pm->GetRootLevel(); - const auto level = pmb->loc.level(); + const auto level = pmb->loc.level() - pm->GetRootLevel(); const std::string &mesh_record_name = var_name + "_lvl" + std::to_string(level); PARTHENON_REQUIRE_THROWS(it->meshes.contains(mesh_record_name), diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index b1a41699a609..47913cf460fd 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -14,6 +14,7 @@ #include "basic_types.hpp" #include "openPMD/Iteration.hpp" +#include "openPMD/Series.hpp" #include "outputs/restart.hpp" #include "mesh/domain.hpp" @@ -73,6 +74,9 @@ class RestartReaderOPMD : public RestartReader { private: const std::string filename_; + openPMD::Series series; + // Iteration is a pointer because it cannot be default constructed (it depends on the + // Series). std::unique_ptr it; }; From f9373e8a4fe0073a64d46134b1e4a3f6821455d7 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 15 Apr 2024 15:50:23 +0200 Subject: [PATCH 22/70] WIP Read/Write Params --- src/interface/params.hpp | 6 ++++ src/outputs/openpmd.cpp | 44 +++++++++++++++++++++++++++ src/outputs/outputs.hpp | 1 + src/outputs/restart_opmd.cpp | 58 ++++++++++++++++++++++-------------- src/parthenon_manager.cpp | 3 +- 5 files changed, 89 insertions(+), 23 deletions(-) diff --git a/src/interface/params.hpp b/src/interface/params.hpp index 4b2f4a09d0cd..e99f8561d0ae 100644 --- a/src/interface/params.hpp +++ b/src/interface/params.hpp @@ -118,6 +118,12 @@ class Params { return it->second; } + const Mutability &GetMutability(const std::string &key) const { + auto const it = myMutable_.find(key); + PARTHENON_REQUIRE_THROWS(it != myMutable_.end(), "Key " + key + " doesn't exist"); + return it->second; + } + std::vector GetKeys() const { std::vector keys; for (auto &x : myParams_) { diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 27dc4d6627a7..69f7885c026e 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -37,16 +38,19 @@ #include "defs.hpp" #include "driver/driver.hpp" #include "globals.hpp" +#include "interface/state_descriptor.hpp" #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" #include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/IO/Access.hpp" +#include "openPMD/Iteration.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/Series.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" #include "outputs/output_utils.hpp" #include "outputs/outputs.hpp" +#include "outputs/parthenon_hdf5.hpp" // needd for VALId_VEC_TYPES -> move #include "parthenon_array_generic.hpp" #include "utils/error_checking.hpp" #include "utils/instrument.hpp" @@ -60,6 +64,32 @@ namespace parthenon { using namespace OutputUtils; +template +void WriteAllParamsOfType(std::shared_ptr pkg, openPMD::Iteration *it) { + const std::string prefix = "Params/" + pkg->label() + "/"; + const auto ¶ms = pkg->AllParams(); + for (const auto &key : params.GetKeys()) { + const auto type = params.GetType(key); + if (type == std::type_index(typeid(T))) { + // auto typed_ptr = dynamic_cast *>((p.second).get()); + it->setAttribute(prefix + key, params.Get(key)); + } + } +} + +template +void WriteAllParamsOfMultipleTypes(std::shared_ptr pkg, + openPMD::Iteration *it) { + ([&] { WriteAllParamsOfType(pkg, it); }(), ...); +} + +template +void WriteAllParams(std::shared_ptr pkg, openPMD::Iteration *it) { + WriteAllParamsOfMultipleTypes>(pkg, it); + // TODO(pgrete) check why this doens't work, i.e., which type is causing problems + // WriteAllParamsOfMultipleTypes(pkg, it); +} + //---------------------------------------------------------------------------------------- //! \fn void OpenPMDOutput:::WriteOutputFile(Mesh *pm) // \brief Expose mesh and all Cell variables for processing with Ascent @@ -125,6 +155,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, it.setDt(-1.0); } { // FIXME move this to dump params + PARTHENON_INSTRUMENT_REGION("Dump Params"); const auto view_d = Kokkos::View("blub", 5, 3); // Map a view onto a host allocation (so that we can call deep_copy) @@ -133,6 +164,19 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, view_h(host_vec.data(), view_d.extent_int(0), view_d.extent_int(1)); Kokkos::deep_copy(view_h, view_d); it.setAttribute("blub", host_vec); + + for (const auto &[key, pkg] : pm->packages.AllPackages()) { + // WriteAllParams(pkg, &it); // check why this (vector of bool) doesn't work + WriteAllParams(pkg, &it); + WriteAllParams(pkg, &it); + WriteAllParams(pkg, &it); + WriteAllParams(pkg, &it); + WriteAllParams(pkg, &it); + WriteAllParams(pkg, &it); + WriteAllParams(pkg, &it); + WriteAllParamsOfType(pkg, &it); + // WriteAllParamsOfType>(pkg, &it); + } } // Then our own { diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index 59cb03610374..972a35b36490 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -32,6 +32,7 @@ #include "interface/mesh_data.hpp" #include "io_wrapper.hpp" #include "kokkos_abstraction.hpp" +#include "openPMD/Iteration.hpp" #include "parthenon_arrays.hpp" #include "utils/error_checking.hpp" diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index d14c5710ee39..30a2aafc5ed8 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -92,28 +92,42 @@ std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, return 0; } -void RestartReaderOPMD::ReadParams(const std::string &name, Params &p) { -#if 0 - // views and vecs of scalar types - ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); - ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); - ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); - ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); - ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); - ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); - ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); - - // strings - ReadFromHDF5AllParamsOfType(prefix, group); - ReadFromHDF5AllParamsOfType>(prefix, group); - - template -void call_my_func(my_list ) -{ - (myFunc(), ...); +template +void ReadAllParamsOfType(const std::string &pkg_name, openPMD::Iteration *it, + Params ¶ms) { + for (const auto &key : params.GetKeys()) { + const auto type = params.GetType(key); + auto mutability = params.GetMutability(key); + if (type == std::type_index(typeid(T)) && mutability == Params::Mutability::Restart) { + auto val = it->getAttribute("Params/" + pkg_name + "/" + key).get(); + params.Update(key, val); + } + } +} + +template +void ReadAllParamsOfMultipleTypes(const std::string &pkg_name, openPMD::Iteration *it, + Params &p) { + ([&] { ReadAllParamsOfType(pkg_name, it, p); }(), ...); } -#endif + +template +void ReadAllParams(const std::string &pkg_name, openPMD::Iteration *it, Params &p) { + ReadAllParamsOfMultipleTypes>(pkg_name, it, p); + // TODO(pgrete) check why this doens't work, i.e., which type is causing problems + // ReadAllParamsOfMultipleTypes(pkg, it); +} +void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { + ReadAllParams(pkg_name, it, p); + ReadAllParams(pkg_name, it, p); + ReadAllParams(pkg_name, it, p); + ReadAllParams(pkg_name, it, p); + ReadAllParams(pkg_name, it, p); + ReadAllParams(pkg_name, it, p); + ReadAllParams(pkg_name, it, p); + ReadAllParamsOfType(pkg_name, it, p); } + void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, const OutputUtils::VarInfo &vinfo, std::vector &data_vec, @@ -172,8 +186,8 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block static_cast(pmb->block_size.nx(X1DIR))}; mesh_comp.loadChunkRaw(&data_vec[comp_offset], chunk_offset, chunk_extent); // TODO(pgrete) check if output utils machinery can be used for non-cell - // centered fields, which might not be that straightforward as a global mesh is - // stored rather than individual blocks. + // centered fields, which might not be that straightforward as a global mesh + // is stored rather than individual blocks. comp_offset += pmb->block_size.nx(X1DIR) * pmb->block_size.nx(X2DIR) * pmb->block_size.nx(X3DIR); idx_component += 1; diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index d77ae9377b15..71e108b012f1 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -353,7 +353,8 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { // Double note that this also needs to be update in case // we update the HDF5 infrastructure! - if (file_output_format_ver >= HDF5::OUTPUT_VERSION_FORMAT - 1) { + // TODO(pgrete) figure out what to do about versions of different outputs + if (true || file_output_format_ver >= HDF5::OUTPUT_VERSION_FORMAT - 1) { OutputUtils::PackOrUnpackVar( v_info, resfile.hasGhost, index, [&](auto index, int topo, int t, int u, int v, int k, int j, int i) { From 96a3f4cc41f81c524680ebe979c37023da8680a5 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 15 Apr 2024 19:02:14 +0200 Subject: [PATCH 23/70] Make ReadParams private member --- src/outputs/restart_opmd.cpp | 29 ++++++++++++++--------------- src/outputs/restart_opmd.hpp | 7 +++++++ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 30a2aafc5ed8..23ba83bbf4af 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -93,8 +93,7 @@ std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, } template -void ReadAllParamsOfType(const std::string &pkg_name, openPMD::Iteration *it, - Params ¶ms) { +void RestartReaderOPMD::ReadAllParamsOfType(const std::string &pkg_name, Params ¶ms) { for (const auto &key : params.GetKeys()) { const auto type = params.GetType(key); auto mutability = params.GetMutability(key); @@ -106,26 +105,26 @@ void ReadAllParamsOfType(const std::string &pkg_name, openPMD::Iteration *it, } template -void ReadAllParamsOfMultipleTypes(const std::string &pkg_name, openPMD::Iteration *it, - Params &p) { - ([&] { ReadAllParamsOfType(pkg_name, it, p); }(), ...); +void RestartReaderOPMD::ReadAllParamsOfMultipleTypes(const std::string &pkg_name, + Params &p) { + ([&] { ReadAllParamsOfType(pkg_name, p); }(), ...); } template -void ReadAllParams(const std::string &pkg_name, openPMD::Iteration *it, Params &p) { - ReadAllParamsOfMultipleTypes>(pkg_name, it, p); +void RestartReaderOPMD::ReadAllParams(const std::string &pkg_name, Params &p) { + ReadAllParamsOfMultipleTypes>(pkg_name, p); // TODO(pgrete) check why this doens't work, i.e., which type is causing problems // ReadAllParamsOfMultipleTypes(pkg, it); } void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { - ReadAllParams(pkg_name, it, p); - ReadAllParams(pkg_name, it, p); - ReadAllParams(pkg_name, it, p); - ReadAllParams(pkg_name, it, p); - ReadAllParams(pkg_name, it, p); - ReadAllParams(pkg_name, it, p); - ReadAllParams(pkg_name, it, p); - ReadAllParamsOfType(pkg_name, it, p); + ReadAllParams(pkg_name, p); + ReadAllParams(pkg_name, p); + ReadAllParams(pkg_name, p); + ReadAllParams(pkg_name, p); + ReadAllParams(pkg_name, p); + ReadAllParams(pkg_name, p); + ReadAllParams(pkg_name, p); + ReadAllParamsOfType(pkg_name, p); } void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index 47913cf460fd..f1c60d3efb7f 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -78,6 +78,13 @@ class RestartReaderOPMD : public RestartReader { // Iteration is a pointer because it cannot be default constructed (it depends on the // Series). std::unique_ptr it; + + template + void ReadAllParamsOfType(const std::string &pkg_name, Params ¶ms); + template + void ReadAllParamsOfMultipleTypes(const std::string &pkg_name, Params &p); + template + void ReadAllParams(const std::string &pkg_name, Params &p); }; } // namespace parthenon From 56976a8514c0c54dfb02fa03261a350876e51d9e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 18 Apr 2024 11:04:29 +0200 Subject: [PATCH 24/70] Fix root level in output --- src/outputs/openpmd.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 69f7885c026e..19c264034a73 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -203,14 +203,14 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // restart info, write always it.setAttribute("NBNew", pm->nbnew); it.setAttribute("NBDel", pm->nbdel); - it.setAttribute("RootLevel", pm->GetRootLevel()); + it.setAttribute("RootLevel", pm->GetLegacyTreeRootLevel()); it.setAttribute("Refine", pm->adaptive ? 1 : 0); it.setAttribute("Multilevel", pm->multilevel ? 1 : 0); it.setAttribute("BlocksPerPE", pm->GetNbList()); // Mesh block size - const auto base_block_size = pm->GetBlockSize(); + const auto base_block_size = pm->GetDefaultBlockSize(); it.setAttribute("MeshBlockSize", std::vector{base_block_size.nx(X1DIR), base_block_size.nx(X2DIR), base_block_size.nx(X3DIR)}); From 6199843c2241a1fcaadd0c682f3e3bb46f7be78f Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 22 Apr 2024 16:32:43 +0200 Subject: [PATCH 25/70] Move to mesh per record standard for writing --- src/outputs/openpmd.cpp | 76 +++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 19c264034a73..3b3b35b4eea2 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -316,40 +316,43 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, for (auto &pmb : pm->block_list) { // TODO(pgrete) check if we should skip the suffix for level 0 const auto level = pmb->loc.level() - pm->GetRootLevel(); - const std::string &mesh_record_name = var_name + "_lvl" + std::to_string(level); - - // Create the mesh_record for this variable at the given level (if it doesn't exist - // yet) - if (!it.meshes.contains(mesh_record_name)) { - auto mesh_record = it.meshes[mesh_record_name]; - - // These following attributes are shared across all components of the record. - - PARTHENON_REQUIRE_THROWS( - typeid(Coordinates_t) == typeid(UniformCartesian), - "OpenPMD in Parthenon currently only supports Cartesian coordinates."); - mesh_record.setGeometry(openPMD::Mesh::Geometry::cartesian); - auto &coords = pmb->coords; - // For uniform Cartesian, all dxN are const across the block so we just pick the - // first index. - Real dx1 = coords.CellWidth(0, 0, 0); - Real dx2 = coords.CellWidth(0, 0, 0); - Real dx3 = coords.CellWidth(0, 0, 0); - - // TODO(pgrete) check if this should be tied to the MemoryLayout - mesh_record.setDataOrder(openPMD::Mesh::DataOrder::C); - mesh_record.setGridSpacing(std::vector{dx3, dx2, dx1}); - mesh_record.setAxisLabels({"z", "y", "x"}); - mesh_record.setGridGlobalOffset({ - pm->mesh_size.xmin(X3DIR), - pm->mesh_size.xmin(X2DIR), - pm->mesh_size.xmin(X1DIR), - }); - - // TODO(pgrete) need unitDimension and timeOffset for this record? - } - auto mesh_record = it.meshes[mesh_record_name]; + for (const auto &comp_lbl : vinfo.component_labels) { + + const std::string &mesh_record_name = + var_name + "_" + comp_lbl + "_lvl" + std::to_string(level); + + // Create the mesh_record for this variable at the given level (if it doesn't + // exist yet) + if (!it.meshes.contains(mesh_record_name)) { + auto mesh_record = it.meshes[mesh_record_name]; + + // These following attributes are shared across all components of the record. + + PARTHENON_REQUIRE_THROWS( + typeid(Coordinates_t) == typeid(UniformCartesian), + "OpenPMD in Parthenon currently only supports Cartesian coordinates."); + mesh_record.setGeometry(openPMD::Mesh::Geometry::cartesian); + auto &coords = pmb->coords; + // For uniform Cartesian, all dxN are const across the block so we just pick the + // first index. + Real dx1 = coords.CellWidth(0, 0, 0); + Real dx2 = coords.CellWidth(0, 0, 0); + Real dx3 = coords.CellWidth(0, 0, 0); + + // TODO(pgrete) check if this should be tied to the MemoryLayout + mesh_record.setDataOrder(openPMD::Mesh::DataOrder::C); + mesh_record.setGridSpacing(std::vector{dx3, dx2, dx1}); + mesh_record.setAxisLabels({"z", "y", "x"}); + mesh_record.setGridGlobalOffset({ + pm->mesh_size.xmin(X3DIR), + pm->mesh_size.xmin(X2DIR), + pm->mesh_size.xmin(X1DIR), + }); + + // TODO(pgrete) need unitDimension and timeOffset for this record? + } + } // Now that the mesh record exists, actually write the data auto out_var = pmb->meshblock_data.Get()->GetVarPtr(var_name); @@ -396,8 +399,13 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, PARTHENON_THROW("Expected v index doesn't match vector expectation."); } } else { - comp_name = vinfo.component_labels[idx_component]; + comp_name = openPMD::MeshRecordComponent::SCALAR; + // comp_name = vinfo.component_labels[idx_component]; } + const std::string &mesh_record_name = + var_name + "_" + vinfo.component_labels[idx_component] + "_lvl" + + std::to_string(level); + auto mesh_record = it.meshes[mesh_record_name]; auto mesh_comp = mesh_record[comp_name]; // TODO(pgrete) needs to be updated for face and edges etc From 684b7ab63f2a74a98b6c60dfddd483dc16929f94 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 23 Apr 2024 10:17:43 +0200 Subject: [PATCH 26/70] Allow for 2D and 3D output. Fix single dataset reset. --- src/outputs/openpmd.cpp | 99 +++++++++++++++++++++++++++++------------ 1 file changed, 70 insertions(+), 29 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 3b3b35b4eea2..efd0284db1bd 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -342,13 +342,51 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // TODO(pgrete) check if this should be tied to the MemoryLayout mesh_record.setDataOrder(openPMD::Mesh::DataOrder::C); - mesh_record.setGridSpacing(std::vector{dx3, dx2, dx1}); - mesh_record.setAxisLabels({"z", "y", "x"}); - mesh_record.setGridGlobalOffset({ - pm->mesh_size.xmin(X3DIR), - pm->mesh_size.xmin(X2DIR), - pm->mesh_size.xmin(X1DIR), - }); + + // TODO(pgrete) allwo for proper vectors/tensors + auto mesh_comp = mesh_record[openPMD::MeshRecordComponent::SCALAR]; + // TODO(pgrete) needs to be updated for face and edges etc + // Also this feels wrong for deep hierachies... + auto effective_nx = static_cast(std::pow(2, level)); + openPMD::Extent global_extent; + if (pm->ndim == 3) { + mesh_record.setGridSpacing(std::vector{dx3, dx2, dx1}); + mesh_record.setAxisLabels({"z", "y", "x"}); + mesh_record.setGridGlobalOffset({ + pm->mesh_size.xmin(X3DIR), + pm->mesh_size.xmin(X2DIR), + pm->mesh_size.xmin(X1DIR), + }); + // TODO(pgrete) needs to be updated for face and edges etc + mesh_comp.setPosition(std::vector{0.5, 0.5, 0.5}); + global_extent = { + static_cast(pm->mesh_size.nx(X3DIR)) * effective_nx, + static_cast(pm->mesh_size.nx(X2DIR)) * effective_nx, + static_cast(pm->mesh_size.nx(X1DIR)) * effective_nx, + }; + } else if (pm->ndim == 2) { + mesh_record.setGridSpacing(std::vector{dx2, dx1}); + mesh_record.setAxisLabels({"y", "x"}); + mesh_record.setGridGlobalOffset({ + pm->mesh_size.xmin(X2DIR), + pm->mesh_size.xmin(X1DIR), + }); + + // TODO(pgrete) needs to be updated for face and edges etc + mesh_comp.setPosition(std::vector{0.5, 0.5}); + global_extent = { + static_cast(pm->mesh_size.nx(X2DIR)) * effective_nx, + static_cast(pm->mesh_size.nx(X1DIR)) * effective_nx, + }; + + } else { + PARTHENON_THROW("1D output for openpmd not yet supported."); + } + // Handling this here to now re-reset dataset later when iterating through the + // blocks + auto const dataset = + openPMD::Dataset(openPMD::determineDatatype(), global_extent); + mesh_comp.resetDataset(dataset); // TODO(pgrete) need unitDimension and timeOffset for this record? } @@ -408,20 +446,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, auto mesh_record = it.meshes[mesh_record_name]; auto mesh_comp = mesh_record[comp_name]; - // TODO(pgrete) needs to be updated for face and edges etc - mesh_comp.setPosition(std::vector{0.5, 0.5, 0.5}); - // TODO(pgrete) needs to be updated for face and edges etc - // Also this feels wrong for deep hierachies... - auto effective_nx = static_cast(std::pow(2, level)); - openPMD::Extent global_extent = { - static_cast(pm->mesh_size.nx(X3DIR)) * effective_nx, - static_cast(pm->mesh_size.nx(X2DIR)) * effective_nx, - static_cast(pm->mesh_size.nx(X1DIR)) * effective_nx, - }; - auto const dataset = - openPMD::Dataset(openPMD::determineDatatype(), global_extent); - mesh_comp.resetDataset(dataset); - const auto comp_offset = tmp_offset; for (int k = kb.s; k <= kb.e; ++k) { for (int j = jb.s; j <= jb.e; ++j) { @@ -431,14 +455,31 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } } } - openPMD::Offset chunk_offset = { - pmb->loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), - pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), - pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; - openPMD::Extent chunk_extent = { - static_cast(pmb->block_size.nx(X3DIR)), - static_cast(pmb->block_size.nx(X2DIR)), - static_cast(pmb->block_size.nx(X1DIR))}; + openPMD::Offset chunk_offset; + openPMD::Extent chunk_extent; + if (pm->ndim == 3) { + chunk_offset = { + pmb->loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), + pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), + pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; + chunk_extent = {static_cast(pmb->block_size.nx(X3DIR)), + static_cast(pmb->block_size.nx(X2DIR)), + static_cast(pmb->block_size.nx(X1DIR))}; + } else if (pm->ndim == 2) { + chunk_offset = { + pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), + pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; + chunk_extent = {static_cast(pmb->block_size.nx(X2DIR)), + static_cast(pmb->block_size.nx(X1DIR))}; + } else { + PARTHENON_THROW("1D output for openpmd not yet supported."); + } + std::cout << "Block " << pmb->gid << " writes chunk of [" << chunk_extent[0] + << " " << chunk_extent[1] << " " + << "] with offset [" << chunk_offset[0] << " " << chunk_offset[1] + << "] and logical locs [" << pmb->loc.lx2() << " " + << pmb->loc.lx1() << "]\n"; + mesh_comp.storeChunkRaw(&tmp_data[comp_offset], chunk_offset, chunk_extent); idx_component += 1; } From 251c6ea7660fbcd2bd3c7539b95bc1e4e4e959b7 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 23 Apr 2024 17:42:00 +0200 Subject: [PATCH 27/70] Fix logical loc --- src/outputs/openpmd.cpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index efd0284db1bd..d611bc7095b6 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -457,28 +457,24 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } openPMD::Offset chunk_offset; openPMD::Extent chunk_extent; + const auto loc = pm->Forest().GetLegacyTreeLocation(pmb->loc); if (pm->ndim == 3) { chunk_offset = { - pmb->loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), - pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), - pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; + loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), + loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), + loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; chunk_extent = {static_cast(pmb->block_size.nx(X3DIR)), static_cast(pmb->block_size.nx(X2DIR)), static_cast(pmb->block_size.nx(X1DIR))}; } else if (pm->ndim == 2) { chunk_offset = { - pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), - pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; + loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), + loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; chunk_extent = {static_cast(pmb->block_size.nx(X2DIR)), static_cast(pmb->block_size.nx(X1DIR))}; } else { PARTHENON_THROW("1D output for openpmd not yet supported."); } - std::cout << "Block " << pmb->gid << " writes chunk of [" << chunk_extent[0] - << " " << chunk_extent[1] << " " - << "] with offset [" << chunk_offset[0] << " " << chunk_offset[1] - << "] and logical locs [" << pmb->loc.lx2() << " " - << pmb->loc.lx1() << "]\n"; mesh_comp.storeChunkRaw(&tmp_data[comp_offset], chunk_offset, chunk_extent); idx_component += 1; From 03f80c716db8e395e96781e87aad4459b0b27557 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 24 Apr 2024 09:38:25 +0200 Subject: [PATCH 28/70] Rename opmd files --- src/CMakeLists.txt | 2 +- src/outputs/{openpmd.cpp => parthenon_opmd.cpp} | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) rename src/outputs/{openpmd.cpp => parthenon_opmd.cpp} (99%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 140ea9733f47..81f7c4c00bcf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -182,7 +182,6 @@ add_library(parthenon outputs/history.cpp outputs/io_wrapper.cpp outputs/io_wrapper.hpp - outputs/openpmd.cpp outputs/output_utils.cpp outputs/output_utils.hpp outputs/outputs.cpp @@ -194,6 +193,7 @@ add_library(parthenon outputs/parthenon_hdf5_types.hpp outputs/parthenon_xdmf.cpp outputs/parthenon_hdf5.hpp + outputs/parthenon_opmd.cpp outputs/parthenon_xdmf.hpp outputs/restart.hpp outputs/restart_hdf5.cpp diff --git a/src/outputs/openpmd.cpp b/src/outputs/parthenon_opmd.cpp similarity index 99% rename from src/outputs/openpmd.cpp rename to src/outputs/parthenon_opmd.cpp index d611bc7095b6..cc6fd1fd03b0 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -14,7 +14,7 @@ // license in this material to reproduce, prepare derivative works, distribute copies to // the public, perform publicly and display publicly, and to permit others to do so. //======================================================================================== -//! \file openpmd.cpp +//! \file parthenon_openpmd.cpp // \brief Output for OpenPMD https://www.openpmd.org/ (supporting various backends) #include @@ -90,6 +90,13 @@ void WriteAllParams(std::shared_ptr pkg, openPMD::Iteration *it // WriteAllParamsOfMultipleTypes(pkg, it); } +namespace OpenPMDUtils { + + auto GetMeshRecordAndComponentNames() { + + } +} + //---------------------------------------------------------------------------------------- //! \fn void OpenPMDOutput:::WriteOutputFile(Mesh *pm) // \brief Expose mesh and all Cell variables for processing with Ascent @@ -422,7 +429,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, for (int t = 0; t < Nt; ++t) { for (int u = 0; u < Nu; ++u) { for (int v = 0; v < Nv; ++v) { - // Get the correct record std::string comp_name; if (is_scalar) { comp_name = openPMD::MeshRecordComponent::SCALAR; From 890fffe334a0c142425a7bb8e278ad291349460e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 24 Apr 2024 10:14:23 +0200 Subject: [PATCH 29/70] Separate common calls to chunks and names --- src/outputs/parthenon_opmd.cpp | 126 ++++++++++++++++++--------------- 1 file changed, 67 insertions(+), 59 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index cc6fd1fd03b0..ae1b1f40bcb2 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include // Parthenon headers @@ -41,6 +42,7 @@ #include "interface/state_descriptor.hpp" #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" +#include "mesh/meshblock.hpp" #include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/IO/Access.hpp" @@ -92,10 +94,60 @@ void WriteAllParams(std::shared_ptr pkg, openPMD::Iteration *it namespace OpenPMDUtils { - auto GetMeshRecordAndComponentNames() { +// Construct OpenPMD Mesh "record" name and comonnent identifier. +// - comp_idx is a flattended index over all components of the vectors and tensors, i.e., +// the typical v,u,t indices. +// - level is the current effective level of the Mesh record +auto GetMeshRecordAndComponentNames(const VarInfo &vinfo, const int comp_idx, + const int level) { + std::string comp_name; + if (vinfo.is_vector) { + if (comp_idx == 0) { + comp_name = "x"; + } else if (comp_idx == 1) { + comp_name = "y"; + } else if (comp_idx == 2) { + comp_name = "z"; + } else { + PARTHENON_THROW("Expected component index doesn't match vector expectation."); + } + // Current unclear how to properly handle other vectors and tensors, so everything + // that not's a proper vector is a a scalar for now. + } else { + comp_name = openPMD::MeshRecordComponent::SCALAR; + } + // TODO(pgrete) need to make sure that var names are allowed within standard + const std::string &mesh_record_name = vinfo.label + "_" + + vinfo.component_labels[comp_idx] + "_lvl" + + std::to_string(level); + return std::make_tuple(mesh_record_name, comp_name); +} +// Calculate logical location on effective mesh (i.e., a mesh with size that matches full +// coverage at given resolution on a particular level) +// TODO(pgrete) needs to be updated to properly work with Forests +auto GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb) { + openPMD::Offset chunk_offset; + openPMD::Extent chunk_extent; + const auto loc = pm->Forest().GetLegacyTreeLocation(pmb->loc); + if (pm->ndim == 3) { + chunk_offset = {loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), + loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), + loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; + chunk_extent = {static_cast(pmb->block_size.nx(X3DIR)), + static_cast(pmb->block_size.nx(X2DIR)), + static_cast(pmb->block_size.nx(X1DIR))}; + } else if (pm->ndim == 2) { + chunk_offset = {loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), + loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; + chunk_extent = {static_cast(pmb->block_size.nx(X2DIR)), + static_cast(pmb->block_size.nx(X1DIR))}; + } else { + PARTHENON_THROW("1D output for openpmd not yet supported."); } + return std::make_tuple(chunk_offset, chunk_extent); } +} // namespace OpenPMDUtils //---------------------------------------------------------------------------------------- //! \fn void OpenPMDOutput:::WriteOutputFile(Mesh *pm) @@ -308,8 +360,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, memset(tmp_data.data(), 0, tmp_data.size() * sizeof(OutT)); uint64_t tmp_offset = 0; - const bool is_scalar = - vinfo.GetDim(4) == 1 && vinfo.GetDim(5) == 1 && vinfo.GetDim(6) == 1; if (vinfo.is_vector) { // sanity check PARTHENON_REQUIRE_THROWS( @@ -318,21 +368,18 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, "dimensionality of the simulation.") } - // TODO(pgrete) need to make sure that var names are allowed within standard - const std::string var_name = vinfo.label; for (auto &pmb : pm->block_list) { // TODO(pgrete) check if we should skip the suffix for level 0 const auto level = pmb->loc.level() - pm->GetRootLevel(); - for (const auto &comp_lbl : vinfo.component_labels) { - - const std::string &mesh_record_name = - var_name + "_" + comp_lbl + "_lvl" + std::to_string(level); + for (int comp_idx = 0; comp_idx < vinfo.component_labels.size(); comp_idx++) { + const auto [record_name, comp_name] = + OpenPMDUtils::GetMeshRecordAndComponentNames(vinfo, comp_idx, level); // Create the mesh_record for this variable at the given level (if it doesn't // exist yet) - if (!it.meshes.contains(mesh_record_name)) { - auto mesh_record = it.meshes[mesh_record_name]; + if (!it.meshes.contains(record_name)) { + auto mesh_record = it.meshes[record_name]; // These following attributes are shared across all components of the record. @@ -350,8 +397,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // TODO(pgrete) check if this should be tied to the MemoryLayout mesh_record.setDataOrder(openPMD::Mesh::DataOrder::C); - // TODO(pgrete) allwo for proper vectors/tensors - auto mesh_comp = mesh_record[openPMD::MeshRecordComponent::SCALAR]; + auto mesh_comp = mesh_record[comp_name]; // TODO(pgrete) needs to be updated for face and edges etc // Also this feels wrong for deep hierachies... auto effective_nx = static_cast(std::pow(2, level)); @@ -400,7 +446,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } // Now that the mesh record exists, actually write the data - auto out_var = pmb->meshblock_data.Get()->GetVarPtr(var_name); + auto out_var = pmb->meshblock_data.Get()->GetVarPtr(vinfo.label); PARTHENON_REQUIRE_THROWS(out_var->metadata().Where() == MetadataFlag(Metadata::Cell), "Currently only cell centered vars are supported."); @@ -421,7 +467,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, Kokkos::deep_copy(component_buffer_view, data); #endif auto out_var_h = out_var->data.GetHostMirrorAndCopy(); - int idx_component = 0; + int comp_idx = 0; const auto &Nt = out_var->GetDim(6); const auto &Nu = out_var->GetDim(5); const auto &Nv = out_var->GetDim(4); @@ -429,28 +475,9 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, for (int t = 0; t < Nt; ++t) { for (int u = 0; u < Nu; ++u) { for (int v = 0; v < Nv; ++v) { - std::string comp_name; - if (is_scalar) { - comp_name = openPMD::MeshRecordComponent::SCALAR; - } else if (vinfo.is_vector) { - if (v == 0) { - comp_name = "x"; - } else if (v == 1) { - comp_name = "y"; - } else if (v == 2) { - comp_name = "z"; - } else { - PARTHENON_THROW("Expected v index doesn't match vector expectation."); - } - } else { - comp_name = openPMD::MeshRecordComponent::SCALAR; - // comp_name = vinfo.component_labels[idx_component]; - } - const std::string &mesh_record_name = - var_name + "_" + vinfo.component_labels[idx_component] + "_lvl" + - std::to_string(level); - auto mesh_record = it.meshes[mesh_record_name]; - auto mesh_comp = mesh_record[comp_name]; + const auto [record_name, comp_name] = + OpenPMDUtils::GetMeshRecordAndComponentNames(vinfo, comp_idx, level); + auto mesh_comp = it.meshes[record_name][comp_name]; const auto comp_offset = tmp_offset; for (int k = kb.s; k <= kb.e; ++k) { @@ -461,29 +488,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } } } - openPMD::Offset chunk_offset; - openPMD::Extent chunk_extent; - const auto loc = pm->Forest().GetLegacyTreeLocation(pmb->loc); - if (pm->ndim == 3) { - chunk_offset = { - loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), - loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), - loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; - chunk_extent = {static_cast(pmb->block_size.nx(X3DIR)), - static_cast(pmb->block_size.nx(X2DIR)), - static_cast(pmb->block_size.nx(X1DIR))}; - } else if (pm->ndim == 2) { - chunk_offset = { - loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), - loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; - chunk_extent = {static_cast(pmb->block_size.nx(X2DIR)), - static_cast(pmb->block_size.nx(X1DIR))}; - } else { - PARTHENON_THROW("1D output for openpmd not yet supported."); - } - + const auto [chunk_offset, chunk_extent] = + OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb); mesh_comp.storeChunkRaw(&tmp_data[comp_offset], chunk_offset, chunk_extent); - idx_component += 1; + comp_idx += 1; } } } // loop over components From 04359d37d462f49c894bef1ea651949972497156 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 24 Apr 2024 10:33:51 +0200 Subject: [PATCH 30/70] Reuse shared chunk and name for restarts --- src/CMakeLists.txt | 1 + src/outputs/parthenon_opmd.cpp | 20 ++++++--------- src/outputs/parthenon_opmd.hpp | 37 +++++++++++++++++++++++++++ src/outputs/restart_opmd.cpp | 46 ++++++++++------------------------ 4 files changed, 59 insertions(+), 45 deletions(-) create mode 100644 src/outputs/parthenon_opmd.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 81f7c4c00bcf..0b61ea712035 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -194,6 +194,7 @@ add_library(parthenon outputs/parthenon_xdmf.cpp outputs/parthenon_hdf5.hpp outputs/parthenon_opmd.cpp + outputs/parthenon_opmd.hpp outputs/parthenon_xdmf.hpp outputs/restart.hpp outputs/restart_hdf5.cpp diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index ae1b1f40bcb2..c7fb88453ecc 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -94,12 +94,9 @@ void WriteAllParams(std::shared_ptr pkg, openPMD::Iteration *it namespace OpenPMDUtils { -// Construct OpenPMD Mesh "record" name and comonnent identifier. -// - comp_idx is a flattended index over all components of the vectors and tensors, i.e., -// the typical v,u,t indices. -// - level is the current effective level of the Mesh record -auto GetMeshRecordAndComponentNames(const VarInfo &vinfo, const int comp_idx, - const int level) { +std::tuple GetMeshRecordAndComponentNames(const VarInfo &vinfo, + const int comp_idx, + const int level) { std::string comp_name; if (vinfo.is_vector) { if (comp_idx == 0) { @@ -120,13 +117,12 @@ auto GetMeshRecordAndComponentNames(const VarInfo &vinfo, const int comp_idx, const std::string &mesh_record_name = vinfo.label + "_" + vinfo.component_labels[comp_idx] + "_lvl" + std::to_string(level); - return std::make_tuple(mesh_record_name, comp_name); + // return std::make_tuple(mesh_record_name, comp_name); + return {mesh_record_name, comp_name}; } -// Calculate logical location on effective mesh (i.e., a mesh with size that matches full -// coverage at given resolution on a particular level) -// TODO(pgrete) needs to be updated to properly work with Forests -auto GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb) { +std::tuple +GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb) { openPMD::Offset chunk_offset; openPMD::Extent chunk_extent; const auto loc = pm->Forest().GetLegacyTreeLocation(pmb->loc); @@ -145,7 +141,7 @@ auto GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb) { } else { PARTHENON_THROW("1D output for openpmd not yet supported."); } - return std::make_tuple(chunk_offset, chunk_extent); + return {chunk_offset, chunk_extent}; } } // namespace OpenPMDUtils diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp new file mode 100644 index 000000000000..31c606ef4e61 --- /dev/null +++ b/src/outputs/parthenon_opmd.hpp @@ -0,0 +1,37 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2024 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +#ifndef OUTPUTS_PARTHENON_OPMD_HPP_ +#define OUTPUTS_PARTHENON_OPMD_HPP_ +//! \file restart_opmd.hpp +// \brief Provides support for restarting from OpenPMD output + +#include + +#include "mesh/meshblock.hpp" +#include "openPMD/Dataset.hpp" +#include "outputs/output_utils.hpp" + +namespace parthenon { + +namespace OpenPMDUtils { + +// Construct OpenPMD Mesh "record" name and comonnent identifier. +// - comp_idx is a flattended index over all components of the vectors and tensors, i.e., +// the typical v,u,t indices. +// - level is the current effective level of the Mesh record +std::tuple +GetMeshRecordAndComponentNames(const OutputUtils::VarInfo &vinfo, const int comp_idx, + const int level); + +// Calculate logical location on effective mesh (i.e., a mesh with size that matches full +// coverage at given resolution on a particular level) +// TODO(pgrete) needs to be updated to properly work with Forests +std::tuple +GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb); + +} // namespace OpenPMDUtils +} // namespace parthenon +#endif // OUTPUTS_PARTHENON_OPMD_HPP_ diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 23ba83bbf4af..66f4ea555507 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -16,6 +16,7 @@ #include "interface/params.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" +#include "outputs/parthenon_opmd.hpp" #include "outputs/restart.hpp" #include "outputs/restart_opmd.hpp" #include "utils/error_checking.hpp" @@ -134,15 +135,9 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block for (auto &pmb : pm->block_list) { // TODO(pgrete) check if we should skip the suffix for level 0 const auto level = pmb->loc.level() - pm->GetRootLevel(); - const std::string &mesh_record_name = var_name + "_lvl" + std::to_string(level); - - PARTHENON_REQUIRE_THROWS(it->meshes.contains(mesh_record_name), - "Missing mesh record '" + mesh_record_name + - "' in restart file."); - auto mesh_record = it->meshes[mesh_record_name]; int64_t comp_offset = 0; // offset data_vector to store component data - int idx_component = 0; // used in label for non-vector variables + int comp_idx = 0; // used in label for non-vector variables const bool is_scalar = vinfo.GetDim(4) == 1 && vinfo.GetDim(5) == 1 && vinfo.GetDim(6) == 1; const auto &Nt = vinfo.GetDim(6); @@ -153,43 +148,28 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block for (int u = 0; u < Nu; ++u) { for (int v = 0; v < Nv; ++v) { // Get the correct record - std::string comp_name; - if (is_scalar) { - comp_name = openPMD::MeshRecordComponent::SCALAR; - } else if (vinfo.is_vector) { - if (v == 0) { - comp_name = "x"; - } else if (v == 1) { - comp_name = "y"; - } else if (v == 2) { - comp_name = "z"; - } else { - PARTHENON_THROW("Expected v index doesn't match vector expectation."); - } - } else { - comp_name = vinfo.component_labels[idx_component]; - } + const auto [record_name, comp_name] = + OpenPMDUtils::GetMeshRecordAndComponentNames(vinfo, comp_idx, level); + + PARTHENON_REQUIRE_THROWS(it->meshes.contains(record_name), + "Missing mesh record '" + record_name + + "' in restart file."); + auto mesh_record = it->meshes[record_name]; PARTHENON_REQUIRE_THROWS(mesh_record.contains(comp_name), "Missing component'" + comp_name + - "' in mesh record '" + mesh_record_name + + "' in mesh record '" + record_name + "' of restart file."); auto mesh_comp = mesh_record[comp_name]; - openPMD::Offset chunk_offset = { - pmb->loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), - pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), - pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; - openPMD::Extent chunk_extent = { - static_cast(pmb->block_size.nx(X3DIR)), - static_cast(pmb->block_size.nx(X2DIR)), - static_cast(pmb->block_size.nx(X1DIR))}; + const auto [chunk_offset, chunk_extent] = + OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb); mesh_comp.loadChunkRaw(&data_vec[comp_offset], chunk_offset, chunk_extent); // TODO(pgrete) check if output utils machinery can be used for non-cell // centered fields, which might not be that straightforward as a global mesh // is stored rather than individual blocks. comp_offset += pmb->block_size.nx(X1DIR) * pmb->block_size.nx(X2DIR) * pmb->block_size.nx(X3DIR); - idx_component += 1; + comp_idx += 1; } } } // loop over components From 2b89659d605a5c600b976fb3e3b3a0aade180e75 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 24 Apr 2024 12:14:22 +0200 Subject: [PATCH 31/70] Add regression test --- src/outputs/parthenon_opmd.cpp | 5 +- tst/regression/CMakeLists.txt | 15 +++ .../test_suites/restart_opmd/__init__.py | 0 .../restart_opmd/parthinput.restart | 60 ++++++++++++ .../restart_opmd/parthinput_override.restart | 9 ++ .../test_suites/restart_opmd/restart_opmd.py | 97 +++++++++++++++++++ 6 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 tst/regression/test_suites/restart_opmd/__init__.py create mode 100644 tst/regression/test_suites/restart_opmd/parthinput.restart create mode 100644 tst/regression/test_suites/restart_opmd/parthinput_override.restart create mode 100644 tst/regression/test_suites/restart_opmd/restart_opmd.py diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index c7fb88453ecc..5423c4f3757e 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -164,7 +164,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // but an interation) This just describes the pattern of the filename. The correct file // will be accessed through the iteration idx below. The file suffix maps to the chosen // backend. - Series series = Series("opmd.%05T.bp", Access::CREATE); + // TODO(pgrete) add final and now logic + Series series = + Series(output_params.file_basename + "." + output_params.file_id + ".%05T.bp", + Access::CREATE); // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per diff --git a/tst/regression/CMakeLists.txt b/tst/regression/CMakeLists.txt index 96a54da6e427..78b1089acf1f 100644 --- a/tst/regression/CMakeLists.txt +++ b/tst/regression/CMakeLists.txt @@ -130,6 +130,21 @@ if (ENABLE_HDF5) endif() +if (PARTHENON_ENABLE_OPENPMD) + + # h5py is needed for restart and hdf5 test + list(APPEND REQUIRED_PYTHON_MODULES openpmd_api) + + # Restart + list(APPEND TEST_DIRS restart_opmd) + list(APPEND TEST_PROCS ${NUM_MPI_PROC_TESTING}) + list(APPEND TEST_ARGS "--driver ${PROJECT_BINARY_DIR}/example/advection/advection-example \ + --driver_input ${CMAKE_CURRENT_SOURCE_DIR}/test_suites/restart_opmd/parthinput.restart \ + --num_steps 2") + list(APPEND EXTRA_TEST_LABELS "") + + endif() + # Any external modules that are required by python can be added to REQUIRED_PYTHON_MODULES # list variable, before including TestSetup.cmake. list(APPEND REQUIRED_PYTHON_MODULES numpy) diff --git a/tst/regression/test_suites/restart_opmd/__init__.py b/tst/regression/test_suites/restart_opmd/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tst/regression/test_suites/restart_opmd/parthinput.restart b/tst/regression/test_suites/restart_opmd/parthinput.restart new file mode 100644 index 000000000000..a30112475502 --- /dev/null +++ b/tst/regression/test_suites/restart_opmd/parthinput.restart @@ -0,0 +1,60 @@ +# ======================================================================================== +# Parthenon performance portable AMR framework +# Copyright(C) 2024 The Parthenon collaboration +# Licensed under the 3-clause BSD License, see LICENSE file for details +# ======================================================================================== + + +problem_id = advection + + +refinement = adaptive +numlevel = 3 + +nx1 = 64 +x1min = -0.5 +x1max = 0.5 +ix1_bc = periodic +ox1_bc = periodic + +nx2 = 64 +x2min = -0.5 +x2max = 0.5 +ix2_bc = periodic +ox2_bc = periodic + +nx3 = 1 +x3min = -0.5 +x3max = 0.5 +ix3_bc = periodic +ox3_bc = periodic + + +nx1 = 16 +nx2 = 16 +nx3 = 1 + + +nlim = -1 +tlim = 0.1 +integrator = rk2 +ncycle_out_mesh = -10000 + + +cfl = 0.45 +vx = 1.0 +vy = 1.0 +vz = 1.0 +profile = hard_sphere + +refine_tol = 0.3 # control the package specific refinement tagging function +derefine_tol = 0.03 +compute_error = false +num_vars = 1 # number of variables +vec_size = 1 # size of each variable +fill_derived = false # whether to fill one-copy test vars + + +file_type = openpmd +dt = 0.050 + diff --git a/tst/regression/test_suites/restart_opmd/parthinput_override.restart b/tst/regression/test_suites/restart_opmd/parthinput_override.restart new file mode 100644 index 000000000000..c5b368aebcbd --- /dev/null +++ b/tst/regression/test_suites/restart_opmd/parthinput_override.restart @@ -0,0 +1,9 @@ +# ======================================================================================== +# Parthenon performance portable AMR framework +# Copyright(C) 2024 The Parthenon collaboration +# Licensed under the 3-clause BSD License, see LICENSE file for details +# ======================================================================================== + +# Testing to override parameters in a restart file from an input file + +problem_id=silver diff --git a/tst/regression/test_suites/restart_opmd/restart_opmd.py b/tst/regression/test_suites/restart_opmd/restart_opmd.py new file mode 100644 index 000000000000..50aabe99b6ab --- /dev/null +++ b/tst/regression/test_suites/restart_opmd/restart_opmd.py @@ -0,0 +1,97 @@ +# ======================================================================================== +# Parthenon performance portable AMR framework +# Copyright(C) 2024 The Parthenon collaboration +# Licensed under the 3-clause BSD License, see LICENSE file for details +# ======================================================================================== + +# Modules +import sys +import utils.test_case + + +# To prevent littering up imported folders with .pyc files or __pycache_ folder +sys.dont_write_bytecode = True + + +class TestCase(utils.test_case.TestCaseAbs): + def Prepare(self, parameters, step): + # enable coverage testing on pass where restart + # files are both read and written + parameters.coverage_status = "both" + + # run baseline (to the very end) + if step == 1: + parameters.driver_cmd_line_args = ["parthenon/job/problem_id=gold"] + # restart from an early snapshot + elif step == 2: + parameters.driver_cmd_line_args = [ + "-r", + "gold.out1.00001.bp", + "-i", + f"{parameters.parthenon_path}/tst/regression/test_suites/restart_opmd/parthinput_override.restart", + ] + + return parameters + + def Analyse(self, parameters): + try: + import openpmd_api as ompd + except ModuleNotFoundError: + print("Couldn't find required openpmd_api module to compare test results.") + return False + success = True + + def compare_attributes(series_a, series_b): + all_equal = True + for attr in series_a.attributes: + if series_b.contains_attribute(attr): + attr_a = series_a.get_attribute(attr) + attr_b = series_b.get_attribute(attr) + if attr_a != attr_b: + print(f"Mismatch in attribute '{attr}'. " + f"'{attr_a}' versus '{attr_b}'\n" + ) + all_equal = False + else: + print(f"Missing attribute '{attr}' in second file.") + all_equal = False + return all_equal + + + + + def compare_files(name): + series_gold = opmd.Series("gold.out1.%T.bp/", opmd.Access.read_only) + series_silver = opmd.Series("silver.out1.%T.bp/", opmd.Access.read_only) + delta = compare( + [ + "gold.out0.%s.rhdf" % name, + "silver.out0.%s.rhdf" % name, + ], + one=True, + ) + + if delta != 0: + print( + "ERROR: Found difference between gold and silver output '%s'." + % name + ) + return False + + return True + + # comapre a few files throughout the simulations + success &= compare_files("00002") + success &= compare_files("00005") + success &= compare_files("00009") + success &= compare_files("final") + + found_line = False + for line in parameters.stdouts[1].decode("utf-8").split("\n"): + if "Terminating on wall-time limit" in line: + found_line = True + if not found_line: + print("ERROR: wall-time limit based termination not triggered.") + success = False + + return success From 804e60ddcf7d0f5220d8c138cf22b6755a662a97 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 24 Apr 2024 18:29:16 +0200 Subject: [PATCH 32/70] Somewhat make restarts working --- src/outputs/restart_opmd.cpp | 6 +- .../restart_opmd/parthinput.restart | 2 +- .../test_suites/restart_opmd/restart_opmd.py | 97 ++++++++++++------- 3 files changed, 65 insertions(+), 40 deletions(-) diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 66f4ea555507..953d1205e0ae 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -132,14 +132,12 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block const OutputUtils::VarInfo &vinfo, std::vector &data_vec, int file_output_format_version, Mesh *pm) const { + int64_t comp_offset = 0; // offset data_vector to store component data for (auto &pmb : pm->block_list) { // TODO(pgrete) check if we should skip the suffix for level 0 const auto level = pmb->loc.level() - pm->GetRootLevel(); - int64_t comp_offset = 0; // offset data_vector to store component data - int comp_idx = 0; // used in label for non-vector variables - const bool is_scalar = - vinfo.GetDim(4) == 1 && vinfo.GetDim(5) == 1 && vinfo.GetDim(6) == 1; + int comp_idx = 0; // used in label for non-vector variables const auto &Nt = vinfo.GetDim(6); const auto &Nu = vinfo.GetDim(5); const auto &Nv = vinfo.GetDim(4); diff --git a/tst/regression/test_suites/restart_opmd/parthinput.restart b/tst/regression/test_suites/restart_opmd/parthinput.restart index a30112475502..768453160a7b 100644 --- a/tst/regression/test_suites/restart_opmd/parthinput.restart +++ b/tst/regression/test_suites/restart_opmd/parthinput.restart @@ -36,7 +36,7 @@ nx3 = 1 nlim = -1 -tlim = 0.1 +tlim = 0.2 integrator = rk2 ncycle_out_mesh = -10000 diff --git a/tst/regression/test_suites/restart_opmd/restart_opmd.py b/tst/regression/test_suites/restart_opmd/restart_opmd.py index 50aabe99b6ab..89156f003fd6 100644 --- a/tst/regression/test_suites/restart_opmd/restart_opmd.py +++ b/tst/regression/test_suites/restart_opmd/restart_opmd.py @@ -35,63 +35,90 @@ def Prepare(self, parameters, step): def Analyse(self, parameters): try: - import openpmd_api as ompd + import openpmd_api as opmd except ModuleNotFoundError: print("Couldn't find required openpmd_api module to compare test results.") return False success = True def compare_attributes(series_a, series_b): + skip_attributes = [ + "iterationFormat", # Stores the file name format. Expected to differ. + "WallTime", + "InputFile", # Is updated during runtime, e.g., startime and thus differs + ] all_equal = True for attr in series_a.attributes: if series_b.contains_attribute(attr): attr_a = series_a.get_attribute(attr) attr_b = series_b.get_attribute(attr) - if attr_a != attr_b: - print(f"Mismatch in attribute '{attr}'. " - f"'{attr_a}' versus '{attr_b}'\n" - ) + if attr not in skip_attributes and attr_a != attr_b: + print( + f"Mismatch in attribute '{attr}'. " + f"'{attr_a}' versus '{attr_b}'\n" + ) all_equal = False else: print(f"Missing attribute '{attr}' in second file.") all_equal = False return all_equal - + # need series in order to flush + def compare_data(it_a, it_b, series_a, series_b): + all_equal = True + for mesh_name, mesh_a in it_a.meshes.items(): + if mesh_name not in it_b.meshes: + print(f"Missing mesh '{mesh_name}' in second file.") + all_equal = False + continue + mesh_b = it_b.meshes[mesh_name] + + for comp_name, comp_a in mesh_a.items(): + if comp_name not in mesh_b: + print( + f"Missing component '{comp_name}' in mesh '{mesh_name}' of second file." + ) + all_equal = False + continue + comp_b = mesh_b[comp_name] + data_a = comp_a.load_chunk() + series_a.flush() + data_b = comp_b.load_chunk() + series_b.flush() + + if (data_a != data_b).any(): + print( + f"Data of component '{comp_name}' in mesh '{mesh_name}' does not match." + ) + all_equal = False + continue + return all_equal - def compare_files(name): + def compare_files(idx_it): + all_good = True series_gold = opmd.Series("gold.out1.%T.bp/", opmd.Access.read_only) series_silver = opmd.Series("silver.out1.%T.bp/", opmd.Access.read_only) - delta = compare( - [ - "gold.out0.%s.rhdf" % name, - "silver.out0.%s.rhdf" % name, - ], - one=True, - ) - - if delta != 0: - print( - "ERROR: Found difference between gold and silver output '%s'." - % name - ) - return False - - return True + + # PG: yes, this is inefficient but keeps the logic simple + all_good &= compare_attributes(series_gold, series_silver) + all_good &= compare_attributes(series_silver, series_gold) + + it_gold = series_gold.iterations[idx_it] + it_silver = series_silver.iterations[idx_it] + all_good &= compare_attributes(it_gold, it_silver) + all_good &= compare_attributes(it_silver, it_gold) + + all_good &= compare_data(it_silver, it_gold, series_gold, series_silver) + all_good &= compare_data(it_gold, it_silver, series_gold, series_silver) + + return all_good # comapre a few files throughout the simulations - success &= compare_files("00002") - success &= compare_files("00005") - success &= compare_files("00009") - success &= compare_files("final") - - found_line = False - for line in parameters.stdouts[1].decode("utf-8").split("\n"): - if "Terminating on wall-time limit" in line: - found_line = True - if not found_line: - print("ERROR: wall-time limit based termination not triggered.") - success = False + success &= compare_files(1) + success &= compare_files(2) + success &= compare_files(3) + success &= compare_files(4) + # success &= compare_files("final") return success From a436f5523a9efe9de967714dc0cf2aa569eae801 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 26 Apr 2024 10:54:12 +0200 Subject: [PATCH 33/70] Fix order of arguments for correct flush --- .../test_suites/restart_opmd/restart_opmd.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tst/regression/test_suites/restart_opmd/restart_opmd.py b/tst/regression/test_suites/restart_opmd/restart_opmd.py index 89156f003fd6..f1a6117053cd 100644 --- a/tst/regression/test_suites/restart_opmd/restart_opmd.py +++ b/tst/regression/test_suites/restart_opmd/restart_opmd.py @@ -7,6 +7,7 @@ # Modules import sys import utils.test_case +import numpy as np # To prevent littering up imported folders with .pyc files or __pycache_ folder @@ -86,9 +87,12 @@ def compare_data(it_a, it_b, series_a, series_b): data_b = comp_b.load_chunk() series_b.flush() - if (data_a != data_b).any(): + try: + np.testing.assert_array_max_ulp(data_a, data_b) + except AssertionError as err: print( - f"Data of component '{comp_name}' in mesh '{mesh_name}' does not match." + f"Data of component '{comp_name}' in mesh '{mesh_name}' does not match:\n" + f"{err}\n" ) all_equal = False continue @@ -109,7 +113,7 @@ def compare_files(idx_it): all_good &= compare_attributes(it_gold, it_silver) all_good &= compare_attributes(it_silver, it_gold) - all_good &= compare_data(it_silver, it_gold, series_gold, series_silver) + all_good &= compare_data(it_silver, it_gold, series_silver, series_gold) all_good &= compare_data(it_gold, it_silver, series_gold, series_silver) return all_good From 28d725d097c43151fd547e0c38530f2432bb94bf Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 26 Apr 2024 11:29:45 +0200 Subject: [PATCH 34/70] Fix handling of output variable names --- src/outputs/outputs.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index 031664c92895..302778e694d1 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -212,8 +212,13 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { // set output variable and optional data format string used in formatted writes if ((op.file_type != "hst") && (op.file_type != "rst") && (op.file_type != "ascent") && (op.file_type != "histogram")) { - op.variables = pin->GetOrAddVector(pib->block_name, "variables", - std::vector()); + // differentiating here whether a block exists or not to not add an empty + // parameter to the input file (which might interfere with restarts) + if (pin->DoesParameterExist(pib->block_name, "variables")) { + op.variables = pin->GetVector(pib->block_name, "variables"); + } else { + op.variables = std::vector(); + } // JMM: If the requested var isn't present for a given swarm, // it is simply not output. op.swarms.clear(); // Not sure this is needed From a039ea1d0a8f750a799526197851266a66f9e617 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 26 Apr 2024 15:06:41 +0200 Subject: [PATCH 35/70] Fix reading chunks for sparsely populated output files --- src/outputs/restart_opmd.cpp | 2 + .../test_suites/restart_opmd/restart_opmd.py | 43 ++++++++++++++++--- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 953d1205e0ae..2f114c6a4287 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -36,6 +36,8 @@ RestartReaderOPMD::RestartReaderOPMD(const char *filename) idx = i.first; } it = std::make_unique(series.iterations[idx]); + // Explicitly open (important for parallel execution) + it->open(); } int RestartReaderOPMD::GetOutputFormatVersion() const { diff --git a/tst/regression/test_suites/restart_opmd/restart_opmd.py b/tst/regression/test_suites/restart_opmd/restart_opmd.py index f1a6117053cd..3548418f61d4 100644 --- a/tst/regression/test_suites/restart_opmd/restart_opmd.py +++ b/tst/regression/test_suites/restart_opmd/restart_opmd.py @@ -25,9 +25,13 @@ def Prepare(self, parameters, step): parameters.driver_cmd_line_args = ["parthenon/job/problem_id=gold"] # restart from an early snapshot elif step == 2: + # TODO(pgrete or someone else) ideally we want to restart from a later snapshot + # BUT results are not bitwise identical for AMR runs. PG thinks this is + # related to not storing the deref counter (and similar) and also thinks + # it's worth fixing. parameters.driver_cmd_line_args = [ "-r", - "gold.out1.00001.bp", + "gold.out1.00000.bp", "-i", f"{parameters.parthenon_path}/tst/regression/test_suites/restart_opmd/parthinput_override.restart", ] @@ -82,10 +86,39 @@ def compare_data(it_a, it_b, series_a, series_b): all_equal = False continue comp_b = mesh_b[comp_name] - data_a = comp_a.load_chunk() - series_a.flush() - data_b = comp_b.load_chunk() - series_b.flush() + + if comp_a.shape != comp_b.shape: + print( + f"Mismatch is mech record component shapes of " + " compontent '{comp_name}' in mesh '{mesh_name}': " + f"{comp_a.shape} versus {comp_b.shape}\n" + ) + all_equal = False + continue + + # Given that the shapes are guaranteed to match (follow the check above) + # we can load chunks from both files. + # Note that we have to go over chunks as data might be sparse on disk so + # loading the entire record will contain gargabe in sparse places. + data_a = np.empty(comp_a.shape) + data_a[:] = np.nan + data_b = np.copy(data_a) + for chunk in comp_a.available_chunks(): + # Following OpenPMD-viewer `chunk_to_slice` here + # https://github.com/openPMD/openPMD-viewer/blob/6eccb608893d2c9b8d158d950c3f0451898a80f6/openpmd_viewer/openpmd_timeseries/data_reader/io_reader/utilities.py#L14 + stops = [a + b for a, b in zip(chunk.offset, chunk.extent)] + indices_per_dim = zip(chunk.offset, stops) + sl = tuple( + map(lambda s: slice(s[0], s[1], None), indices_per_dim) + ) + + tmp = comp_a[sl] + series_a.flush() + data_a[sl] = tmp + + tmp = comp_b[sl] + series_b.flush() + data_b[sl] = tmp try: np.testing.assert_array_max_ulp(data_a, data_b) From 7476641aeb88892b81818969b031ad280ac81142 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 3 May 2024 17:04:08 +0200 Subject: [PATCH 36/70] Dont tell anyone I spent days on this... --- src/outputs/parthenon_opmd.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 5423c4f3757e..9b0adbd5f7d4 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -167,7 +167,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // TODO(pgrete) add final and now logic Series series = Series(output_params.file_basename + "." + output_params.file_id + ".%05T.bp", - Access::CREATE); + Access::CREATE, MPI_COMM_WORLD); // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per From 39d4b99f02b353369bbbed92df4cf037ad0632c6 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 3 May 2024 17:37:20 +0200 Subject: [PATCH 37/70] Temp disable dumping Views from device --- CMakeLists.txt | 2 +- src/outputs/outputs.hpp | 1 - src/outputs/parthenon_opmd.cpp | 27 +++++++++++++++++---------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a902e0e785e..a9a343008d4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,7 +31,7 @@ include(CTest) # Compile time constants # Compile Options -option(PARTHENON_SINGLE_PRENSION "Run in single precision" OFF) +option(PARTHENON_SINGLE_PRECISION "Run in single precision" OFF) option(PARTHENON_DISABLE_MPI "MPI is enabled by default if found, set this to True to disable MPI" OFF) option(PARTHENON_ENABLE_HOST_COMM_BUFFERS "CUDA/HIP Only: Allocate communication buffers on host (may be slower)" OFF) option(PARTHENON_DISABLE_HDF5 "HDF5 is enabled by default if found, set this to True to disable HDF5" OFF) diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index e0a578a8e249..14a3df4eb66f 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -214,7 +214,6 @@ class OpenPMDOutput : public OutputType { explicit OpenPMDOutput(const OutputParameters &oparams) : OutputType(oparams) {} void WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const SignalHandler::OutputSignal signal) override; - }; #ifdef ENABLE_HDF5 diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 9b0adbd5f7d4..9b669d70de8a 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -177,7 +177,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, series.setComment("Hello world!"); series.setMachine("bla"); series.setSoftware("Parthenon + Downstream info"); - series.setDate("2024-02-29"); + series.setDate("2024-02-29 17:48:42 +0100"); // TODO(pgrete) Units? @@ -213,15 +213,19 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, it.setDt(-1.0); } { // FIXME move this to dump params - PARTHENON_INSTRUMENT_REGION("Dump Params"); - const auto view_d = - Kokkos::View("blub", 5, 3); - // Map a view onto a host allocation (so that we can call deep_copy) - auto host_vec = std::vector(view_d.size()); - Kokkos::View> - view_h(host_vec.data(), view_d.extent_int(0), view_d.extent_int(1)); - Kokkos::deep_copy(view_h, view_d); - it.setAttribute("blub", host_vec); + + // TODO(pgrete) Make this test piece work on devices + if constexpr (false) { + PARTHENON_INSTRUMENT_REGION("Dump Params"); + const auto view_d = + Kokkos::View("blub", 5, 3); + // Map a view onto a host allocation (so that we can call deep_copy) + auto host_vec = std::vector(view_d.size()); + Kokkos::View> + view_h(host_vec.data(), view_d.extent_int(0), view_d.extent_int(1)); + Kokkos::deep_copy(view_h, view_d); + it.setAttribute("blub", host_vec); + } for (const auto &[key, pkg] : pm->packages.AllPackages()) { // WriteAllParams(pkg, &it); // check why this (vector of bool) doesn't work @@ -438,6 +442,9 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // blocks auto const dataset = openPMD::Dataset(openPMD::determineDatatype(), global_extent); + // TODO(pgrete) check whether this should/need to be a collective so that the + // mesh generation should be done across all ranks prior to writing data, rather + // than in-situ for the local blocks only mesh_comp.resetDataset(dataset); // TODO(pgrete) need unitDimension and timeOffset for this record? From 4241198d9fb02352bf4f00ed04262a283d67ad08 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 12 Jun 2024 13:47:22 +0200 Subject: [PATCH 38/70] Dump deref cnt in opmd restart --- src/outputs/parthenon_opmd.cpp | 5 +++++ src/outputs/restart_opmd.cpp | 2 ++ src/outputs/restart_opmd.hpp | 7 ++++--- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 9b669d70de8a..02b761363174 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -315,6 +315,11 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, std::vector id_local = OutputUtils::ComputeIDsAndFlags(pm); auto id_global = FlattendedLocalToGlobal(pm, id_local); it.setAttribute("loc.level-gid-lid-cnghost-gflag", id_global); + + // derefinement count + std::vector derefcnt_local = OutputUtils::ComputeDerefinementCount(pm); + auto derefcnt_global = FlattendedLocalToGlobal(pm, derefcnt_local); + it.setAttribute("derefinement_count", derefcnt_global); } // TODO(pgrete) check var name standard compatiblity diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 2f114c6a4287..0bd3ba5d3768 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -72,6 +72,8 @@ RestartReaderOPMD::MeshInfo RestartReaderOPMD::GetMeshInfo() const { mesh_info.lx123 = it->getAttribute("loc.lx123").get>(); mesh_info.level_gid_lid_cnghost_gflag = it->getAttribute("loc.level-gid-lid-cnghost-gflag").get>(); + mesh_info.derefinement_count = + it->getAttribute("derefinement_count").get>(); return mesh_info; } diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index f1c60d3efb7f..e8fad77b6efc 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -41,6 +41,9 @@ class RestartReaderOPMD : public RestartReader { // Return output format version number. Return -1 if not existent. [[nodiscard]] int GetOutputFormatVersion() const override; + // Current not supported + [[nodiscard]] int HasGhost() const override { return 0; }; + public: // Gets data for all blocks on current rank. // Assumes blocks are contiguous @@ -69,11 +72,9 @@ class RestartReaderOPMD : public RestartReader { // perhaps belongs in a destructor? void Close(); - // Does file have ghost cells? - int hasGhost; - private: const std::string filename_; + openPMD::Series series; // Iteration is a pointer because it cannot be default constructed (it depends on the // Series). From 60a38b2a9ffde83363f0a5153d98daa4ea08a608 Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Tue, 18 Jun 2024 14:31:49 -0400 Subject: [PATCH 39/70] install openpmd in macOS CI --- .github/workflows/ci-macos.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index a1666b5a9520..45edb719c746 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -28,10 +28,14 @@ jobs: cache-dependency-path: '**/requirements.txt' - run: pip install -r requirements.txt - - name: Install dependencies + - name: Install dependencies (Homebrew) run: | brew install openmpi hdf5-mpi adios2 || true + - name: Install OpenPMD + run: | + openPMD_USE_MPI=ON python3 -m pip install openpmd-api --no-binary openpmd-api + - name: Configure run: cmake -B build -DCMAKE_BUILD_TYPE=Release From 28020db6c13541b4e8d133f6cb49dbf0ea41d9e4 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 25 Jun 2024 13:16:04 +0200 Subject: [PATCH 40/70] Remove extraneous popRegion --- src/outputs/parthenon_opmd.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 02b761363174..29aabf310705 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -298,7 +298,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } it.setAttribute("BoundaryConditions", boundary_condition_str); - Kokkos::Profiling::popRegion(); // write Info } // Info section Kokkos::Profiling::popRegion(); // write Attributes From af4b966a8025e0588dedda9b684c07ec2e88b2fc Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 11 Jul 2024 10:14:57 +0200 Subject: [PATCH 41/70] Fix formatting --- src/outputs/parthenon_opmd.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 29aabf310705..55f19155053d 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -298,7 +298,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } it.setAttribute("BoundaryConditions", boundary_condition_str); - } // Info section + } // Info section Kokkos::Profiling::popRegion(); // write Attributes @@ -505,10 +505,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } } } // loop over components - } // out_var->IsAllocated() - } // loop over blocks + } // out_var->IsAllocated() + } // loop over blocks it.seriesFlush(); - } // loop over vars + } // loop over vars Kokkos::Profiling::popRegion(); // write all variable data // The iteration can be closed in order to help free up resources. From 0e015d10aa8486f9a00a8232f6e9a0c89a35789e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 11 Jul 2024 11:16:47 +0200 Subject: [PATCH 42/70] Make format clang16 compatible --- src/outputs/parthenon_opmd.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 55f19155053d..33cb1c3110cc 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -505,10 +505,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } } } // loop over components - } // out_var->IsAllocated() - } // loop over blocks + } // out_var->IsAllocated() + } // loop over blocks it.seriesFlush(); - } // loop over vars + } // loop over vars Kokkos::Profiling::popRegion(); // write all variable data // The iteration can be closed in order to help free up resources. From 5135aea1966b4d5e78dcf64f959d4f2a61cbe723 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 12 Jul 2024 08:51:13 +0200 Subject: [PATCH 43/70] Fix default backend_config parsing --- src/outputs/outputs.cpp | 5 ++++- src/outputs/outputs.hpp | 9 ++++++++- src/outputs/parthenon_opmd.cpp | 7 ++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index c63b9f19a4b5..2d449d4d940a 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -269,7 +269,10 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { pnew_type = new AscentOutput(op); } else if (op.file_type == "openpmd") { #ifdef PARTHENON_ENABLE_OPENPMD - pnew_type = new OpenPMDOutput(op); + const auto backend_config = + pin->GetOrAddString(op.block_name, "backend_config", "{}"); + + pnew_type = new OpenPMDOutput(op, backend_config); #else msg << "### FATAL ERROR in Outputs constructor" << std::endl << "Executable not configured for OpenPMD outputs, but OpenPMD file format " diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index 7dc2607cd3fc..e4d6738579e4 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "Kokkos_ScatterView.hpp" @@ -177,9 +178,15 @@ class AscentOutput : public OutputType { class OpenPMDOutput : public OutputType { public: - explicit OpenPMDOutput(const OutputParameters &oparams) : OutputType(oparams) {} + explicit OpenPMDOutput(const OutputParameters &oparams, + std::string backend_config) + : OutputType(oparams), backend_config_(std::move(backend_config)) {} void WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const SignalHandler::OutputSignal signal) override; + + private: + // path to file containing config passed to backend + std::string backend_config_; }; #ifdef ENABLE_HDF5 diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 33cb1c3110cc..7883347a8f22 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -165,9 +165,14 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // will be accessed through the iteration idx below. The file suffix maps to the chosen // backend. // TODO(pgrete) add final and now logic + if (backend_config_ != "{}") { + // the prepending @ indicates that the config is a file to be read and parsed + backend_config_.insert(0, "@"); + } + Series series = Series(output_params.file_basename + "." + output_params.file_id + ".%05T.bp", - Access::CREATE, MPI_COMM_WORLD); + Access::CREATE, MPI_COMM_WORLD, backend_config_); // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per From b344291a4cfc4ff2bc064f207fc6a268a57ff91d Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 12 Jul 2024 09:04:39 +0200 Subject: [PATCH 44/70] another attempt --- src/outputs/outputs.cpp | 2 +- src/outputs/parthenon_opmd.cpp | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index 2d449d4d940a..dad81a799175 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -270,7 +270,7 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { } else if (op.file_type == "openpmd") { #ifdef PARTHENON_ENABLE_OPENPMD const auto backend_config = - pin->GetOrAddString(op.block_name, "backend_config", "{}"); + pin->GetOrAddString(op.block_name, "backend_config", "default"); pnew_type = new OpenPMDOutput(op, backend_config); #else diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 7883347a8f22..61372fb48db0 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -165,14 +165,13 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // will be accessed through the iteration idx below. The file suffix maps to the chosen // backend. // TODO(pgrete) add final and now logic - if (backend_config_ != "{}") { - // the prepending @ indicates that the config is a file to be read and parsed - backend_config_.insert(0, "@"); - } + // Prepending @ indicates that the config is a file to be read and parsed. + std::string backend_config = + backend_config_ == "default" ? "{}" : "@" + backend_config_; Series series = Series(output_params.file_basename + "." + output_params.file_id + ".%05T.bp", - Access::CREATE, MPI_COMM_WORLD, backend_config_); + Access::CREATE, MPI_COMM_WORLD, backend_config); // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per From 7486915ccc8e00bca984154a5c8ece1865e304d6 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 12 Jul 2024 10:06:15 +0200 Subject: [PATCH 45/70] Bump OpenPMD version --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a90fe04a3ee..f6642ccb2451 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,7 +204,8 @@ if (PARTHENON_ENABLE_OPENPMD) set(openPMD_USE_PYTHON OFF) FetchContent_Declare(openPMD GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git" - GIT_TAG "0.15.2") + # we need newer than the latest 0.15.2 release to support writing attriutes from a subset of ranks + GIT_TAG "bda3544") # develop as of 2024-07-12 FetchContent_MakeAvailable(openPMD) install(TARGETS openPMD EXPORT parthenonTargets) endif() From 0e3f758490ee362184714dd92aed09277614f07e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 25 Jul 2024 20:23:53 +0200 Subject: [PATCH 46/70] pmd: Write scalar particle data --- src/outputs/parthenon_opmd.cpp | 61 ++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 61372fb48db0..7f1956250b9e 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -515,6 +515,67 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } // loop over vars Kokkos::Profiling::popRegion(); // write all variable data + // -------------------------------------------------------------------------------- // + // WRITING PARTICLE DATA // + // -------------------------------------------------------------------------------- // + + Kokkos::Profiling::pushRegion("write particle data"); + // TODO(pgrete) as above, first wrt differentiating between restart_ (last arg) + AllSwarmInfo all_swarm_info(pm->block_list, output_params.swarms, true); + for (auto &[swname, swinfo] : all_swarm_info.all_info) { + openPMD::ParticleSpecies swm = it.particles[swname]; + // These indicate particles/meshblock and location in global index + // space where each meshblock starts + swm.setAttribute("counts", swinfo.counts); + swm.setAttribute("offsets", swinfo.offsets); + + if (swinfo.global_count == 0) { + continue; + } + + // TODO(pgrete) dedup code (figure out why FillHostBuffer cannot be called from within + // auto lambda) + auto &int_vars = std::get>(swinfo.vars); + for (auto &[vname, swmvarvec] : int_vars) { + const auto &vinfo = swinfo.var_info.at(vname); + PARTHENON_REQUIRE_THROWS(vinfo.tensor_rank == 0, + "Only scalar particles supported at the moment."); + auto host_data = swinfo.FillHostBuffer(vname, swmvarvec); + openPMD::RecordComponent rc = swm[vname][openPMD::MeshRecordComponent::SCALAR]; + + auto const dataset = openPMD::Dataset(openPMD::determineDatatype(host_data.data()), + {swinfo.global_count}); + rc.resetDataset(dataset); + rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); + // Flush because the host buffer is temporary + it.seriesFlush(); + } + auto &real_vars = std::get>(swinfo.vars); + for (auto &[vname, swmvarvec] : real_vars) { + const auto &vinfo = swinfo.var_info.at(vname); + PARTHENON_REQUIRE_THROWS(vinfo.tensor_rank == 0, + "Only scalar particles supported at the moment."); + auto host_data = swinfo.FillHostBuffer(vname, swmvarvec); + openPMD::RecordComponent rc = swm[vname][openPMD::MeshRecordComponent::SCALAR]; + + auto const dataset = openPMD::Dataset(openPMD::determineDatatype(host_data.data()), + {swinfo.global_count}); + rc.resetDataset(dataset); + rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); + // Flush because the host buffer is temporary + it.seriesFlush(); + } + + // From the HDF5 output: + // If swarm does not contain an "id" object, generate a sequential + // one for vis. + // BUT PG: this may break things in unpredicable ways + // I'm in favor of enforcing a global id somehow. We shold discuss. + PARTHENON_REQUIRE_THROWS(swinfo.var_info.count("id") != 0, + "Particles should always carry a unique, persistent id!"); + } + Kokkos::Profiling::popRegion(); // write particle data + // The iteration can be closed in order to help free up resources. // The iteration's content will be flushed automatically. // An iteration once closed cannot (yet) be reopened. From e39ef6135ae5ef8a9baaddb3b36f3c94fe77a4cf Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 25 Jul 2024 21:29:58 +0200 Subject: [PATCH 47/70] Code dedup --- src/outputs/output_utils.hpp | 2 +- src/outputs/parthenon_opmd.cpp | 57 +++++++++++++++------------------- 2 files changed, 26 insertions(+), 33 deletions(-) diff --git a/src/outputs/output_utils.hpp b/src/outputs/output_utils.hpp index e1b411ccf480..88fd7629fd37 100644 --- a/src/outputs/output_utils.hpp +++ b/src/outputs/output_utils.hpp @@ -233,7 +233,7 @@ struct SwarmInfo { // Copies swarmvar to host in prep for output template std::vector FillHostBuffer(const std::string vname, - ParticleVariableVector &swmvarvec) { + const ParticleVariableVector &swmvarvec) const { const auto &vinfo = var_info.at(vname); std::vector host_data(count_on_rank * vinfo.nvar); std::size_t ivec = 0; diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 7f1956250b9e..8e104f8a5453 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -48,6 +48,7 @@ #include "openPMD/IO/Access.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Mesh.hpp" +#include "openPMD/ParticleSpecies.hpp" #include "openPMD/Series.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" #include "outputs/output_utils.hpp" @@ -94,6 +95,28 @@ void WriteAllParams(std::shared_ptr pkg, openPMD::Iteration *it namespace OpenPMDUtils { +template +void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, + openPMD::Iteration it) { + auto &vars_of_type_T = std::get>(swinfo.vars); + for (const auto &[vname, swmvarvec] : vars_of_type_T) { + const auto &vinfo = swinfo.var_info.at(vname); + if (vinfo.tensor_rank != 0) { + continue; + } + PARTHENON_REQUIRE_THROWS(vinfo.tensor_rank == 0, + "Only scalar particles supported at the moment."); + auto host_data = swinfo.FillHostBuffer(vname, swmvarvec); + openPMD::RecordComponent rc = swm[vname][openPMD::MeshRecordComponent::SCALAR]; + + auto const dataset = openPMD::Dataset(openPMD::determineDatatype(host_data.data()), + {swinfo.global_count}); + rc.resetDataset(dataset); + rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); + // Flush because the host buffer is temporary + it.seriesFlush(); + } +} std::tuple GetMeshRecordAndComponentNames(const VarInfo &vinfo, const int comp_idx, const int level) { @@ -533,38 +556,8 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, continue; } - // TODO(pgrete) dedup code (figure out why FillHostBuffer cannot be called from within - // auto lambda) - auto &int_vars = std::get>(swinfo.vars); - for (auto &[vname, swmvarvec] : int_vars) { - const auto &vinfo = swinfo.var_info.at(vname); - PARTHENON_REQUIRE_THROWS(vinfo.tensor_rank == 0, - "Only scalar particles supported at the moment."); - auto host_data = swinfo.FillHostBuffer(vname, swmvarvec); - openPMD::RecordComponent rc = swm[vname][openPMD::MeshRecordComponent::SCALAR]; - - auto const dataset = openPMD::Dataset(openPMD::determineDatatype(host_data.data()), - {swinfo.global_count}); - rc.resetDataset(dataset); - rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); - // Flush because the host buffer is temporary - it.seriesFlush(); - } - auto &real_vars = std::get>(swinfo.vars); - for (auto &[vname, swmvarvec] : real_vars) { - const auto &vinfo = swinfo.var_info.at(vname); - PARTHENON_REQUIRE_THROWS(vinfo.tensor_rank == 0, - "Only scalar particles supported at the moment."); - auto host_data = swinfo.FillHostBuffer(vname, swmvarvec); - openPMD::RecordComponent rc = swm[vname][openPMD::MeshRecordComponent::SCALAR]; - - auto const dataset = openPMD::Dataset(openPMD::determineDatatype(host_data.data()), - {swinfo.global_count}); - rc.resetDataset(dataset); - rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); - // Flush because the host buffer is temporary - it.seriesFlush(); - } + OpenPMDUtils::WriteSwarmVar(swinfo, swm, it); + OpenPMDUtils::WriteSwarmVar(swinfo, swm, it); // From the HDF5 output: // If swarm does not contain an "id" object, generate a sequential From b938511d246616616afec86e3dd26827b200f563 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 25 Jul 2024 22:00:50 +0200 Subject: [PATCH 48/70] Allow writing non-scalar particles --- src/outputs/output_utils.hpp | 1 + src/outputs/parthenon_opmd.cpp | 24 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/outputs/output_utils.hpp b/src/outputs/output_utils.hpp index 88fd7629fd37..8041ec96df02 100644 --- a/src/outputs/output_utils.hpp +++ b/src/outputs/output_utils.hpp @@ -242,6 +242,7 @@ struct SwarmInfo { for (int n4 = 0; n4 < vinfo.GetN(4); ++n4) { for (int n3 = 0; n3 < vinfo.GetN(3); ++n3) { for (int n2 = 0; n2 < vinfo.GetN(2); ++n2) { + // TODO(pgrete) understand what's doing on with the blocks here... std::size_t block_idx = 0; for (auto &swmvar : swmvarvec) { // Copied extra times. JMM: If we defrag, unneeded? diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 8e104f8a5453..d24deab9e8a6 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -101,18 +101,26 @@ void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, auto &vars_of_type_T = std::get>(swinfo.vars); for (const auto &[vname, swmvarvec] : vars_of_type_T) { const auto &vinfo = swinfo.var_info.at(vname); - if (vinfo.tensor_rank != 0) { - continue; - } - PARTHENON_REQUIRE_THROWS(vinfo.tensor_rank == 0, - "Only scalar particles supported at the moment."); auto host_data = swinfo.FillHostBuffer(vname, swmvarvec); - openPMD::RecordComponent rc = swm[vname][openPMD::MeshRecordComponent::SCALAR]; auto const dataset = openPMD::Dataset(openPMD::determineDatatype(host_data.data()), {swinfo.global_count}); - rc.resetDataset(dataset); - rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); + // TODO(pgrete) ask OpenPMD group if this is the right approach of if our non-scalar + // partices should be a multi-D `dataset` if is scalar + if (vinfo.tensor_rank == 0) { + openPMD::RecordComponent rc = swm[vname][openPMD::MeshRecordComponent::SCALAR]; + rc.resetDataset(dataset); + rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); + + // else flatten components + } else { + for (auto n = 0; n < vinfo.nvar; n++) { + openPMD::RecordComponent rc = swm[vname][std::to_string(n)]; + rc.resetDataset(dataset); + rc.storeChunkRaw(&host_data[n * swinfo.count_on_rank], + {swinfo.offsets[Globals::my_rank]}, {swinfo.count_on_rank}); + } + } // Flush because the host buffer is temporary it.seriesFlush(); } From 08d6b41add94f7a43622331c3465a6f82b765af7 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 25 Jul 2024 22:16:46 +0200 Subject: [PATCH 49/70] Make positions standard compliant --- src/outputs/parthenon_opmd.cpp | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index d24deab9e8a6..8ea920b55604 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -40,6 +40,7 @@ #include "driver/driver.hpp" #include "globals.hpp" #include "interface/state_descriptor.hpp" +#include "interface/swarm_default_names.hpp" #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" #include "mesh/meshblock.hpp" @@ -108,10 +109,33 @@ void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, // TODO(pgrete) ask OpenPMD group if this is the right approach of if our non-scalar // partices should be a multi-D `dataset` if is scalar if (vinfo.tensor_rank == 0) { - openPMD::RecordComponent rc = swm[vname][openPMD::MeshRecordComponent::SCALAR]; + // special sauce to align "positions" with standard + std::string mesh_record; + std::string mesh_record_component; + if (vname == swarm_position::x::name()) { + mesh_record = "position"; + mesh_record_component = "x"; + } else if (vname == swarm_position::y::name()) { + mesh_record = "position"; + mesh_record_component = "y"; + } else if (vname == swarm_position::z::name()) { + mesh_record = "position"; + mesh_record_component = "z"; + } else { + mesh_record = vname; + mesh_record_component = openPMD::MeshRecordComponent::SCALAR; + } + openPMD::RecordComponent rc = swm[mesh_record][mesh_record_component]; rc.resetDataset(dataset); rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); + // if positional, add offsets + if (mesh_record_component != openPMD::MeshRecordComponent::SCALAR) { + auto rc_offset = swm["positionOffset"][mesh_record_component]; + rc_offset.resetDataset(dataset); + rc_offset.makeConstant(0.0); + } + // else flatten components } else { for (auto n = 0; n < vinfo.nvar; n++) { From bf74c7e5f0cc2519a3d98e313f273cb8484389da Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 25 Jul 2024 23:26:22 +0200 Subject: [PATCH 50/70] Allow for particles restarts (serial works) --- src/outputs/parthenon_opmd.cpp | 26 ++++++++-------- src/outputs/restart_opmd.cpp | 21 +++++++++++-- src/outputs/restart_opmd.hpp | 57 ++++++++++++++++++++++++++++++++-- 3 files changed, 87 insertions(+), 17 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 8ea920b55604..0e4e391cb7df 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -110,28 +110,28 @@ void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, // partices should be a multi-D `dataset` if is scalar if (vinfo.tensor_rank == 0) { // special sauce to align "positions" with standard - std::string mesh_record; - std::string mesh_record_component; + std::string particle_record; + std::string particle_record_component; if (vname == swarm_position::x::name()) { - mesh_record = "position"; - mesh_record_component = "x"; + particle_record = "position"; + particle_record_component = "x"; } else if (vname == swarm_position::y::name()) { - mesh_record = "position"; - mesh_record_component = "y"; + particle_record = "position"; + particle_record_component = "y"; } else if (vname == swarm_position::z::name()) { - mesh_record = "position"; - mesh_record_component = "z"; + particle_record = "position"; + particle_record_component = "z"; } else { - mesh_record = vname; - mesh_record_component = openPMD::MeshRecordComponent::SCALAR; + particle_record = vname; + particle_record_component = openPMD::MeshRecordComponent::SCALAR; } - openPMD::RecordComponent rc = swm[mesh_record][mesh_record_component]; + openPMD::RecordComponent rc = swm[particle_record][particle_record_component]; rc.resetDataset(dataset); rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); // if positional, add offsets - if (mesh_record_component != openPMD::MeshRecordComponent::SCALAR) { - auto rc_offset = swm["positionOffset"][mesh_record_component]; + if (particle_record_component != openPMD::MeshRecordComponent::SCALAR) { + auto rc_offset = swm["positionOffset"][particle_record_component]; rc_offset.resetDataset(dataset); rc_offset.makeConstant(0.0); } diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 0bd3ba5d3768..862dfa001b9e 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -6,6 +6,7 @@ //! \file restart_opmd.cpp // \brief Restarts a simulation from an OpenPMD output with ADIOS2 backend +#include #include #include #include @@ -93,8 +94,24 @@ std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, const IndexRange &range, std::vector &counts, std::vector &offsets) { - // TODO(pgrete) needs impl - return 0; + // datasets + auto counts_dset = + it->particles[swarm].getAttribute("counts").get>(); + auto offsets_dset = + it->particles[swarm].getAttribute("offsets").get>(); + + // Read data for requested blocks in range + counts.resize(range.e - range.s + 1); + offsets.resize(range.e - range.s + 1); + + std::copy(counts_dset.begin() + range.s, counts_dset.begin() + range.e + 1, + counts.begin()); + std::copy(offsets_dset.begin() + range.s, offsets_dset.begin() + range.e + 1, + offsets.begin()); + + // Compute total count rank + std::size_t total_count_on_rank = std::accumulate(counts.begin(), counts.end(), 0); + return total_count_on_rank; } template diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index e8fad77b6efc..497de7766eef 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -13,6 +13,7 @@ #include #include "basic_types.hpp" +#include "interface/swarm_default_names.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" #include "outputs/restart.hpp" @@ -52,12 +53,64 @@ class RestartReaderOPMD : public RestartReader { const OutputUtils::VarInfo &info, std::vector &dataVec, int file_output_format_version, Mesh *pmesh) const override; + // Gets the data from a swarm var on current rank. Assumes all + // blocks are contiguous. Fills dataVec based on shape from swarmvar + // metadata. + template + void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, const Metadata &m, + std::vector &data_vec) { + openPMD::ParticleSpecies swm = it->particles[swarmname]; + + const auto &shape = m.Shape(); + const int rank = shape.size(); + std::size_t nvar = 1; + for (int i = 0; i < rank; ++i) { + nvar *= shape[rank - 1 - i]; + } + std::size_t total_count = nvar * count; + if (data_vec.size() < total_count) { // greedy re-alloc + data_vec.resize(total_count); + } + + std::string particle_record; + std::string particle_record_component; + for (auto n = 0; n < nvar; n++) { + if (varname == swarm_position::x::name()) { + particle_record = "position"; + particle_record_component = "x"; + } else if (varname == swarm_position::y::name()) { + particle_record = "position"; + particle_record_component = "y"; + } else if (varname == swarm_position::z::name()) { + particle_record = "position"; + particle_record_component = "z"; + } else { + particle_record = varname; + particle_record_component = + rank == 0 ? openPMD::MeshRecordComponent::SCALAR : std::to_string(n); + } + + openPMD::RecordComponent rc = swm[particle_record][particle_record_component]; + rc.loadChunkRaw(&data_vec[n * count], {offset}, {count}); + } + + // Now actually read the registered chunks form disk + it->seriesFlush(); + } + void ReadSwarmVar(const std::string &swarmname, const std::string &varname, const std::size_t count, const std::size_t offset, const Metadata &m, - std::vector &dataVec) override{}; + std::vector &dataVec) override { + + ReadSwarmVar<>(swarmname, varname, count, offset, m, dataVec); + }; void ReadSwarmVar(const std::string &swarmname, const std::string &varname, const std::size_t count, const std::size_t offset, const Metadata &m, - std::vector &dataVec) override{}; + std::vector &dataVec) override { + + ReadSwarmVar<>(swarmname, varname, count, offset, m, dataVec); + }; // Gets the counts and offsets for MPI ranks for the meshblocks set // by the indexrange. Returns the total count on this rank. From 3b37c26f11a6a2ab5b17a04428a782612a0ec129 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 26 Jul 2024 14:37:49 +0200 Subject: [PATCH 51/70] Support particle restarts in parallel --- src/outputs/output_utils.cpp | 2 ++ src/outputs/output_utils.hpp | 4 ++-- src/outputs/parthenon_opmd.cpp | 18 +++++++++++++----- src/utils/mpi_types.hpp | 6 ++++++ 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/outputs/output_utils.cpp b/src/outputs/output_utils.cpp index effa915df82c..70776dcea93c 100644 --- a/src/outputs/output_utils.cpp +++ b/src/outputs/output_utils.cpp @@ -285,6 +285,8 @@ std::vector FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_loca } // explicit template instantiation +template std::vector +FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_local); template std::vector FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_local); template std::vector FlattendedLocalToGlobal(Mesh *pm, diff --git a/src/outputs/output_utils.hpp b/src/outputs/output_utils.hpp index 8041ec96df02..ecd331cddd9e 100644 --- a/src/outputs/output_utils.hpp +++ b/src/outputs/output_utils.hpp @@ -210,8 +210,8 @@ struct SwarmInfo { std::size_t count_on_rank = 0; // per-meshblock std::size_t global_offset; // global std::size_t global_count; // global - std::vector counts; // per-meshblock - std::vector offsets; // global + std::vector counts; // on local meshblocks + std::vector offsets; // global offset for local meshblocks // std::vector> masks; // used for reading swarms without defrag std::vector max_indices; // JMM: If we defrag, unneeded? void AddOffsets(const SP_Swarm &swarm); // sets above metadata diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 0e4e391cb7df..1cbd31cdd5da 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -127,7 +127,10 @@ void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, } openPMD::RecordComponent rc = swm[particle_record][particle_record_component]; rc.resetDataset(dataset); - rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); + // only write if there's sth to write (otherwise the host_data nullptr is caught) + if (swinfo.count_on_rank != 0) { + rc.storeChunk(host_data, {swinfo.global_offset}, {host_data.size()}); + } // if positional, add offsets if (particle_record_component != openPMD::MeshRecordComponent::SCALAR) { @@ -141,8 +144,11 @@ void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, for (auto n = 0; n < vinfo.nvar; n++) { openPMD::RecordComponent rc = swm[vname][std::to_string(n)]; rc.resetDataset(dataset); - rc.storeChunkRaw(&host_data[n * swinfo.count_on_rank], - {swinfo.offsets[Globals::my_rank]}, {swinfo.count_on_rank}); + // only write if there's sth to write (otherwise the host_data nullptr is caught) + if (swinfo.count_on_rank != 0) { + rc.storeChunkRaw(&host_data[n * swinfo.count_on_rank], {swinfo.global_offset}, + {swinfo.count_on_rank}); + } } } // Flush because the host buffer is temporary @@ -581,8 +587,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, openPMD::ParticleSpecies swm = it.particles[swname]; // These indicate particles/meshblock and location in global index // space where each meshblock starts - swm.setAttribute("counts", swinfo.counts); - swm.setAttribute("offsets", swinfo.offsets); + auto counts_global = FlattendedLocalToGlobal(pm, swinfo.counts); + swm.setAttribute("counts", counts_global); + auto offsets_global = FlattendedLocalToGlobal(pm, swinfo.offsets); + swm.setAttribute("offsets", offsets_global); if (swinfo.global_count == 0) { continue; diff --git a/src/utils/mpi_types.hpp b/src/utils/mpi_types.hpp index 164ab66f1115..6f5e117504ad 100644 --- a/src/utils/mpi_types.hpp +++ b/src/utils/mpi_types.hpp @@ -53,6 +53,12 @@ inline MPI_Datatype MPITypeMap::type() { return MPI_CXX_BOOL; } +template <> +inline MPI_Datatype MPITypeMap::type() { + // TODO(pgrete) do we need special checks here wrt to conflicts on MacOS? + return MPI_UINT64_T; +} + } // namespace parthenon #endif From 5f95466b8e8d75b33267250439f4d84bac50a7c4 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 6 Aug 2024 12:35:36 +0200 Subject: [PATCH 52/70] Add now prefix to pmd outputs --- src/outputs/output_utils.hpp | 2 +- src/outputs/outputs.hpp | 3 +-- src/outputs/parthenon_opmd.cpp | 19 +++++++------------ src/outputs/parthenon_opmd.hpp | 2 ++ src/outputs/restart_opmd.cpp | 4 ++-- src/outputs/restart_opmd.hpp | 2 -- 6 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/outputs/output_utils.hpp b/src/outputs/output_utils.hpp index ecd331cddd9e..8d8b88d897bb 100644 --- a/src/outputs/output_utils.hpp +++ b/src/outputs/output_utils.hpp @@ -211,7 +211,7 @@ struct SwarmInfo { std::size_t global_offset; // global std::size_t global_count; // global std::vector counts; // on local meshblocks - std::vector offsets; // global offset for local meshblocks + std::vector offsets; // global offset for local meshblocks // std::vector> masks; // used for reading swarms without defrag std::vector max_indices; // JMM: If we defrag, unneeded? void AddOffsets(const SP_Swarm &swarm); // sets above metadata diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index e4d6738579e4..58f5646e4e67 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -178,8 +178,7 @@ class AscentOutput : public OutputType { class OpenPMDOutput : public OutputType { public: - explicit OpenPMDOutput(const OutputParameters &oparams, - std::string backend_config) + explicit OpenPMDOutput(const OutputParameters &oparams, std::string backend_config) : OutputType(oparams), backend_config_(std::move(backend_config)) {} void WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const SignalHandler::OutputSignal signal) override; diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 1cbd31cdd5da..646b4124b10d 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -17,23 +17,17 @@ //! \file parthenon_openpmd.cpp // \brief Output for OpenPMD https://www.openpmd.org/ (supporting various backends) -#include #include #include #include #include -#include -#include #include #include -#include #include -#include #include #include // Parthenon headers -#include "Kokkos_Core_fwd.hpp" #include "basic_types.hpp" #include "coordinates/coordinates.hpp" #include "defs.hpp" @@ -51,10 +45,9 @@ #include "openPMD/Mesh.hpp" #include "openPMD/ParticleSpecies.hpp" #include "openPMD/Series.hpp" -#include "openPMD/backend/MeshRecordComponent.hpp" #include "outputs/output_utils.hpp" #include "outputs/outputs.hpp" -#include "outputs/parthenon_hdf5.hpp" // needd for VALId_VEC_TYPES -> move +#include "outputs/parthenon_opmd.hpp" #include "parthenon_array_generic.hpp" #include "utils/error_checking.hpp" #include "utils/instrument.hpp" @@ -230,9 +223,12 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, std::string backend_config = backend_config_ == "default" ? "{}" : "@" + backend_config_; - Series series = - Series(output_params.file_basename + "." + output_params.file_id + ".%05T.bp", - Access::CREATE, MPI_COMM_WORLD, backend_config); + auto filename = output_params.file_basename + "." + output_params.file_id; + if (signal == SignalHandler::OutputSignal::now) { + filename.append(".now"); + } + filename.append(".%05T.bp"); + Series series = Series(filename, Access::CREATE, MPI_COMM_WORLD, backend_config); // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per @@ -278,7 +274,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, it.setDt(-1.0); } { // FIXME move this to dump params - // TODO(pgrete) Make this test piece work on devices if constexpr (false) { PARTHENON_INSTRUMENT_REGION("Dump Params"); diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index 31c606ef4e61..94a9161ec424 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -9,6 +9,8 @@ // \brief Provides support for restarting from OpenPMD output #include +#include +#include #include "mesh/meshblock.hpp" #include "openPMD/Dataset.hpp" diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 862dfa001b9e..b6767eaa3714 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -6,8 +6,8 @@ //! \file restart_opmd.cpp // \brief Restarts a simulation from an OpenPMD output with ADIOS2 backend +#include #include -#include #include #include #include @@ -32,7 +32,7 @@ RestartReaderOPMD::RestartReaderOPMD(const char *filename) PARTHENON_REQUIRE_THROWS( series.iterations.size() == 1, "Parthenon restarts should only contain one iteration/timestep."); - unsigned long idx; + std::uint64_t idx; for (const auto &i : series.iterations) { idx = i.first; } diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index 497de7766eef..4353bf59e233 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -102,13 +102,11 @@ class RestartReaderOPMD : public RestartReader { void ReadSwarmVar(const std::string &swarmname, const std::string &varname, const std::size_t count, const std::size_t offset, const Metadata &m, std::vector &dataVec) override { - ReadSwarmVar<>(swarmname, varname, count, offset, m, dataVec); }; void ReadSwarmVar(const std::string &swarmname, const std::string &varname, const std::size_t count, const std::size_t offset, const Metadata &m, std::vector &dataVec) override { - ReadSwarmVar<>(swarmname, varname, count, offset, m, dataVec); }; From 681159446354213cfd2713ba52b89edaa112ffad Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 7 Aug 2024 10:42:16 +0200 Subject: [PATCH 53/70] Make linter happy --- src/outputs/parthenon_opmd.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 646b4124b10d..8e1268ef4413 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -17,6 +17,7 @@ //! \file parthenon_openpmd.cpp // \brief Output for OpenPMD https://www.openpmd.org/ (supporting various backends) +#include #include #include #include @@ -25,6 +26,7 @@ #include #include #include +#include #include // Parthenon headers @@ -524,7 +526,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, if (out_var->IsAllocated()) { // TODO(pgrete) check if we can work with a direct copy from a subview to not // duplicate the memory footprint here -#if 0 +#if 0 // Pick a subview of the active cells of this component auto const data = Kokkos::subview( var->data, 0, 0, icomp, std::make_pair(kb.s, kb.e + 1), From b2d75253575774ace1eefe480eb46d8fa5b61210 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 4 Sep 2024 11:12:10 +0200 Subject: [PATCH 54/70] Bump OPMD version and add delim --- CMakeLists.txt | 2 +- src/outputs/parthenon_opmd.cpp | 3 ++- src/outputs/parthenon_opmd.hpp | 7 +++++++ src/outputs/restart_opmd.cpp | 11 ++++++++++- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eda4bee63126..5c4435b04c5a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,7 +205,7 @@ if (PARTHENON_ENABLE_OPENPMD) FetchContent_Declare(openPMD GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git" # we need newer than the latest 0.15.2 release to support writing attriutes from a subset of ranks - GIT_TAG "bda3544") # develop as of 2024-07-12 + GIT_TAG "1c7d7ff") # develop as of 2024-09-02 FetchContent_MakeAvailable(openPMD) install(TARGETS openPMD EXPORT parthenonTargets) endif() diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 8e1268ef4413..2526728b9c8d 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -65,7 +65,8 @@ using namespace OutputUtils; template void WriteAllParamsOfType(std::shared_ptr pkg, openPMD::Iteration *it) { - const std::string prefix = "Params/" + pkg->label() + "/"; + using OpenPMDUtils::delim; + const std::string prefix = "Params" + delim + pkg->label() + delim; const auto ¶ms = pkg->AllParams(); for (const auto &key : params.GetKeys()) { const auto type = params.GetType(key); diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index 94a9161ec424..2c7035dd13e9 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -20,6 +20,13 @@ namespace parthenon { namespace OpenPMDUtils { +// Deliminter to separate packages and parameters in attributes. +// More or less a workaround as the OpenPMD API does currently not expose +// access to non-standard groups (such as "Params" versus the standard "meshes"). +// TODO(pgrete & reviewer) (agree on delim and add check for package name and keys) OR +// better use of opmd-api +inline static const std::string delim = "+"; + // Construct OpenPMD Mesh "record" name and comonnent identifier. // - comp_idx is a flattended index over all components of the vectors and tensors, i.e., // the typical v,u,t indices. diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index b6767eaa3714..8fd939f2ae84 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -116,11 +116,20 @@ std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, template void RestartReaderOPMD::ReadAllParamsOfType(const std::string &pkg_name, Params ¶ms) { + using OpenPMDUtils::delim; for (const auto &key : params.GetKeys()) { const auto type = params.GetType(key); auto mutability = params.GetMutability(key); if (type == std::type_index(typeid(T)) && mutability == Params::Mutability::Restart) { - auto val = it->getAttribute("Params/" + pkg_name + "/" + key).get(); + auto attrs = it->attributes(); + for (const auto & attr : attrs) { + std::cout << "Contains attribute: " << attr << std::endl; + } + std::cout << "Reading '" + << "Params" + delim + pkg_name + delim + key << "' with type: " << typeid(T).name() + << std::endl; + + auto val = it->getAttribute("Params" + delim + pkg_name + delim + key).get(); params.Update(key, val); } } From 3283635ad1d509d78723e3412beede61eb466cb0 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 26 Sep 2024 17:40:33 +0200 Subject: [PATCH 55/70] Change delim. Is this stupid? --- src/outputs/parthenon_opmd.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index 2c7035dd13e9..c67c7366a1a8 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -25,7 +25,7 @@ namespace OpenPMDUtils { // access to non-standard groups (such as "Params" versus the standard "meshes"). // TODO(pgrete & reviewer) (agree on delim and add check for package name and keys) OR // better use of opmd-api -inline static const std::string delim = "+"; +inline static const std::string delim = "🤝"; // Construct OpenPMD Mesh "record" name and comonnent identifier. // - comp_idx is a flattended index over all components of the vectors and tensors, i.e., From 3f95fc4686b98ba4b23e30565356949f6ab907d0 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 7 Oct 2024 15:55:05 +0200 Subject: [PATCH 56/70] Make params IO test more flexible --- tst/unit/test_unit_params.cpp | 67 ++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 633d7f856163..8d69b62ab2b8 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -25,6 +25,7 @@ #include "interface/params.hpp" #include "kokkos_abstraction.hpp" #include "outputs/parthenon_hdf5.hpp" +#include "outputs/restart_hdf5.hpp" using parthenon::Params; using parthenon::Real; @@ -136,9 +137,19 @@ TEST_CASE("when hasKey is called", "[hasKey]") { } } -#ifdef ENABLE_HDF5 - -TEST_CASE("A set of params can be dumped to file", "[params][output]") { +#if defined(ENABLE_HDF5) && defined(PARTHENON_ENABLE_OPENPMD) +using parthenon::RestartReaderHDF5; +using OutputTypes = std::tuple; +#elif defined(ENABLE_HDF5) +using parthenon::RestartReaderHDF5; +using OutputTypes = std::tuple; +#elif defined(PARTHENON_ENABLE_OPENPMD) +#else +using OutputTypes = std::tuple<>; +#endif + +TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][output]", + OutputTypes) { GIVEN("A params object with a few kinds of objects") { Params params; const auto restart = Params::Mutability::Restart; @@ -171,35 +182,42 @@ TEST_CASE("A set of params can be dumped to file", "[params][output]") { } params.Add("hostarr2d", hostarr, restart); - THEN("We can output to hdf5") { + THEN("We can output") { const std::string filename = "params_test.h5"; const std::string groupname = "params"; const std::string prefix = "test_pkg"; - using namespace parthenon::HDF5; - { + if constexpr (std::is_same_v) { + using namespace parthenon::HDF5; + H5F file = H5F::FromHIDCheck( H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)); auto group = MakeGroup(file, groupname); params.WriteAllToHDF5(prefix, group); + } else { + FAIL("This logic is flawed. I should not be here."); } - AND_THEN("We can directly read the relevant data from the hdf5 file") { - H5F file = - H5F::FromHIDCheck(H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT)); - const H5O obj = H5O::FromHIDCheck(H5Oopen(file, groupname.c_str(), H5P_DEFAULT)); - + AND_THEN("We can directly read the relevant data from the file") { Real in_scalar; - HDF5ReadAttribute(obj, prefix + "/scalar", in_scalar); - REQUIRE(std::abs(scalar - in_scalar) <= 1e-10); - std::vector in_vector; - HDF5ReadAttribute(obj, prefix + "/vector", in_vector); + parthenon::ParArray2D in_arr2d("myarr", 1, 1); + if constexpr (std::is_same_v) { + + H5F file = + H5F::FromHIDCheck(H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT)); + const H5O obj = + H5O::FromHIDCheck(H5Oopen(file, groupname.c_str(), H5P_DEFAULT)); + + HDF5ReadAttribute(obj, prefix + "/scalar", in_scalar); + HDF5ReadAttribute(obj, prefix + "/vector", in_vector); + HDF5ReadAttribute(obj, prefix + "/arr2d", in_arr2d); + } + REQUIRE(scalar == in_scalar); + for (int i = 0; i < vector.size(); ++i) { REQUIRE(in_vector[i] == vector[i]); } // deliberately the wrong size - parthenon::ParArray2D in_arr2d("myarr", 1, 1); - HDF5ReadAttribute(obj, prefix + "/arr2d", in_arr2d); REQUIRE(in_arr2d.extent_int(0) == arr2d.extent_int(0)); REQUIRE(in_arr2d.extent_int(1) == arr2d.extent_int(1)); int nwrong = 1; @@ -232,14 +250,17 @@ TEST_CASE("A set of params can be dumped to file", "[params][output]") { parthenon::HostArray2D test_hostarr("hostarr2d", 1, 1); rparams.Add("hostarr2d", test_hostarr, restart); - H5F file = - H5F::FromHIDCheck(H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT)); - const H5G obj = H5G::FromHIDCheck(H5Oopen(file, groupname.c_str(), H5P_DEFAULT)); - rparams.ReadFromRestart(prefix, obj); + if constexpr (std::is_same_v) { + H5F file = + H5F::FromHIDCheck(H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT)); + const H5G obj = + H5G::FromHIDCheck(H5Oopen(file, groupname.c_str(), H5P_DEFAULT)); + rparams.ReadFromRestart(prefix, obj); + } AND_THEN("The values for the restartable params are updated to match the file") { auto test_scalar = rparams.Get("scalar"); - REQUIRE(std::abs(test_scalar - scalar) <= 1e-10); + REQUIRE(test_scalar == scalar); auto test_bool = rparams.Get("boolscalar"); REQUIRE(test_bool == boolscalar); @@ -264,5 +285,3 @@ TEST_CASE("A set of params can be dumped to file", "[params][output]") { } } } - -#endif // ENABLE_HDF5 From 1141ff3efa50386830433c7af1d56c2dfa11930a Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 7 Oct 2024 17:15:15 +0200 Subject: [PATCH 57/70] Add test case for opmd params IO --- src/outputs/parthenon_opmd.cpp | 52 +++++++++++++++++++++------------- src/outputs/parthenon_opmd.hpp | 4 +++ src/outputs/restart_opmd.cpp | 13 ++++++--- tst/unit/test_unit_params.cpp | 26 +++++++++++++++-- 4 files changed, 69 insertions(+), 26 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 2526728b9c8d..c11ad5517838 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -63,11 +63,11 @@ namespace parthenon { using namespace OutputUtils; +namespace OpenPMDUtils { + template -void WriteAllParamsOfType(std::shared_ptr pkg, openPMD::Iteration *it) { - using OpenPMDUtils::delim; - const std::string prefix = "Params" + delim + pkg->label() + delim; - const auto ¶ms = pkg->AllParams(); +void WriteAllParamsOfType(const Params ¶ms, const std::string &prefix, + openPMD::Iteration *it) { for (const auto &key : params.GetKeys()) { const auto type = params.GetType(key); if (type == std::type_index(typeid(T))) { @@ -78,19 +78,33 @@ void WriteAllParamsOfType(std::shared_ptr pkg, openPMD::Iterati } template -void WriteAllParamsOfMultipleTypes(std::shared_ptr pkg, +void WriteAllParamsOfMultipleTypes(const Params ¶ms, const std::string &prefix, openPMD::Iteration *it) { - ([&] { WriteAllParamsOfType(pkg, it); }(), ...); + ([&] { WriteAllParamsOfType(params, prefix, it); }(), ...); } template -void WriteAllParams(std::shared_ptr pkg, openPMD::Iteration *it) { - WriteAllParamsOfMultipleTypes>(pkg, it); +void WriteAllParams(const Params ¶ms, const std::string &prefix, + openPMD::Iteration *it) { + WriteAllParamsOfMultipleTypes>(params, prefix, it); // TODO(pgrete) check why this doens't work, i.e., which type is causing problems // WriteAllParamsOfMultipleTypes(pkg, it); } -namespace OpenPMDUtils { +void WriteAllParams(const Params ¶ms, const std::string &prefix, + openPMD::Iteration *it) { + // WriteAllParams(params, prefix, it); // check why this (vector of bool) doesn't + // work + WriteAllParams(params, prefix, it); + WriteAllParams(params, prefix, it); + WriteAllParams(params, prefix, it); + WriteAllParams(params, prefix, it); + WriteAllParams(params, prefix, it); + WriteAllParams(params, prefix, it); + WriteAllParams(params, prefix, it); + WriteAllParamsOfType(params, prefix, it); + // WriteAllParamsOfType>(params,prefix, it); +} template void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, @@ -231,7 +245,11 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, filename.append(".now"); } filename.append(".%05T.bp"); - Series series = Series(filename, Access::CREATE, MPI_COMM_WORLD, backend_config); + Series series = Series(filename, Access::CREATE, +#ifdef MPI_PARALLEL + MPI_COMM_WORLD, +#endif + backend_config); // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per @@ -291,16 +309,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } for (const auto &[key, pkg] : pm->packages.AllPackages()) { - // WriteAllParams(pkg, &it); // check why this (vector of bool) doesn't work - WriteAllParams(pkg, &it); - WriteAllParams(pkg, &it); - WriteAllParams(pkg, &it); - WriteAllParams(pkg, &it); - WriteAllParams(pkg, &it); - WriteAllParams(pkg, &it); - WriteAllParams(pkg, &it); - WriteAllParamsOfType(pkg, &it); - // WriteAllParamsOfType>(pkg, &it); + using OpenPMDUtils::delim; + const std::string prefix = "Params" + delim + pkg->label() + delim; + const auto ¶ms = pkg->AllParams(); + OpenPMDUtils::WriteAllParams(params, prefix, &it); } } // Then our own diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index c67c7366a1a8..078934f90088 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -14,12 +14,16 @@ #include "mesh/meshblock.hpp" #include "openPMD/Dataset.hpp" +#include "openPMD/Iteration.hpp" #include "outputs/output_utils.hpp" namespace parthenon { namespace OpenPMDUtils { +void WriteAllParams(const Params ¶ms, const std::string &prefix, + openPMD::Iteration *it); + // Deliminter to separate packages and parameters in attributes. // More or less a workaround as the OpenPMD API does currently not expose // access to non-standard groups (such as "Params" versus the standard "meshes"). diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 8fd939f2ae84..1696d5535fc5 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -28,7 +28,12 @@ namespace parthenon { //! \fn void RestartReader::RestartReader(const std::string filename) // \brief Opens the restart file and stores appropriate file handle in fh_ RestartReaderOPMD::RestartReaderOPMD(const char *filename) - : filename_(filename), series(filename, openPMD::Access::READ_ONLY, MPI_COMM_WORLD) { + : filename_(filename), series(filename, openPMD::Access::READ_ONLY +#ifdef MPI_PARALLEL + , + MPI_COMM_WORLD +#endif + ) { PARTHENON_REQUIRE_THROWS( series.iterations.size() == 1, "Parthenon restarts should only contain one iteration/timestep."); @@ -122,12 +127,12 @@ void RestartReaderOPMD::ReadAllParamsOfType(const std::string &pkg_name, Params auto mutability = params.GetMutability(key); if (type == std::type_index(typeid(T)) && mutability == Params::Mutability::Restart) { auto attrs = it->attributes(); - for (const auto & attr : attrs) { + for (const auto &attr : attrs) { std::cout << "Contains attribute: " << attr << std::endl; } std::cout << "Reading '" - << "Params" + delim + pkg_name + delim + key << "' with type: " << typeid(T).name() - << std::endl; + << "Params" + delim + pkg_name + delim + key + << "' with type: " << typeid(T).name() << std::endl; auto val = it->getAttribute("Params" + delim + pkg_name + delim + key).get(); params.Update(key, val); diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 8d69b62ab2b8..7def5ba46aa3 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -24,8 +24,11 @@ #include "config.hpp" #include "interface/params.hpp" #include "kokkos_abstraction.hpp" +#include "openPMD/Series.hpp" #include "outputs/parthenon_hdf5.hpp" +#include "outputs/parthenon_opmd.hpp" #include "outputs/restart_hdf5.hpp" +#include "outputs/restart_opmd.hpp" using parthenon::Params; using parthenon::Real; @@ -139,11 +142,14 @@ TEST_CASE("when hasKey is called", "[hasKey]") { #if defined(ENABLE_HDF5) && defined(PARTHENON_ENABLE_OPENPMD) using parthenon::RestartReaderHDF5; -using OutputTypes = std::tuple; +using parthenon::RestartReaderOPMD; +using OutputTypes = std::tuple; #elif defined(ENABLE_HDF5) using parthenon::RestartReaderHDF5; using OutputTypes = std::tuple; #elif defined(PARTHENON_ENABLE_OPENPMD) +using parthenon::RestartReaderOPMD; +using OutputTypes = std::tuple; #else using OutputTypes = std::tuple<>; #endif @@ -183,16 +189,23 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu params.Add("hostarr2d", hostarr, restart); THEN("We can output") { - const std::string filename = "params_test.h5"; + std::string filename; const std::string groupname = "params"; const std::string prefix = "test_pkg"; if constexpr (std::is_same_v) { using namespace parthenon::HDF5; + filename = "params_test.h5"; H5F file = H5F::FromHIDCheck( H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)); auto group = MakeGroup(file, groupname); params.WriteAllToHDF5(prefix, group); + } else if constexpr (std::is_same_v) { + filename = ("params_test.%05T.bp"); + auto series = openPMD::Series(filename, openPMD::Access::CREATE); + series.setIterationEncoding(openPMD::IterationEncoding::fileBased); + auto it = series.iterations[0]; + parthenon::OpenPMDUtils::WriteAllParams(params, prefix, &it); } else { FAIL("This logic is flawed. I should not be here."); } @@ -210,6 +223,15 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu HDF5ReadAttribute(obj, prefix + "/scalar", in_scalar); HDF5ReadAttribute(obj, prefix + "/vector", in_vector); HDF5ReadAttribute(obj, prefix + "/arr2d", in_arr2d); + } else if constexpr (std::is_same_v) { + auto series = openPMD::Series(filename, openPMD::Access::READ_ONLY); + auto it = std::make_unique(series.iterations[0]); + // Explicitly open (important for parallel execution) + it->open(); + in_scalar = it->getAttribute(prefix + "/scalar").get(); + in_vector = it->getAttribute(prefix + "/vector").get>(); + in_arr2d = + it->getAttribute(prefix + "/arr2d").get>(); } REQUIRE(scalar == in_scalar); From e6e4d0bcb4b5274eb3a416d79c38df6a0115ff0e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 7 Oct 2024 18:31:55 +0200 Subject: [PATCH 58/70] Reading/writing non-ParArray Params works --- src/outputs/parthenon_opmd.cpp | 8 +++++++- src/outputs/parthenon_opmd.hpp | 4 +--- tst/unit/test_unit_params.cpp | 16 +++++++++------- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index c11ad5517838..bb0986d9b549 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -72,7 +72,13 @@ void WriteAllParamsOfType(const Params ¶ms, const std::string &prefix, const auto type = params.GetType(key); if (type == std::type_index(typeid(T))) { // auto typed_ptr = dynamic_cast *>((p.second).get()); - it->setAttribute(prefix + key, params.Get(key)); + auto full_path = prefix + delim + key; + // The '/' is kind of a reserved character in the OpenPMD standard, which results + // in attribute keys with said character not being exposed. + // Thus we replace it. + std::replace(full_path.begin(), full_path.end(), '/', delim[0]); + + it->setAttribute(full_path, params.Get(key)); } } } diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index 078934f90088..23c3390c47f7 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -27,9 +27,7 @@ void WriteAllParams(const Params ¶ms, const std::string &prefix, // Deliminter to separate packages and parameters in attributes. // More or less a workaround as the OpenPMD API does currently not expose // access to non-standard groups (such as "Params" versus the standard "meshes"). -// TODO(pgrete & reviewer) (agree on delim and add check for package name and keys) OR -// better use of opmd-api -inline static const std::string delim = "🤝"; +inline static const std::string delim = "~"; // Construct OpenPMD Mesh "record" name and comonnent identifier. // - comp_idx is a flattended index over all components of the vectors and tensors, i.e., diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 7def5ba46aa3..e220e3d98e5c 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -212,7 +212,9 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu AND_THEN("We can directly read the relevant data from the file") { Real in_scalar; std::vector in_vector; + // deliberately the wrong size parthenon::ParArray2D in_arr2d("myarr", 1, 1); + if constexpr (std::is_same_v) { H5F file = @@ -226,12 +228,13 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu } else if constexpr (std::is_same_v) { auto series = openPMD::Series(filename, openPMD::Access::READ_ONLY); auto it = std::make_unique(series.iterations[0]); - // Explicitly open (important for parallel execution) - it->open(); - in_scalar = it->getAttribute(prefix + "/scalar").get(); - in_vector = it->getAttribute(prefix + "/vector").get>(); - in_arr2d = - it->getAttribute(prefix + "/arr2d").get>(); + // Note that we're explicitly using `delim` here which tests the character + // replacement of '/' in the WriteAllParams function. + using parthenon::OpenPMDUtils::delim; + in_scalar = it->getAttribute(prefix + delim + "scalar").get(); + in_vector = it->getAttribute(prefix + delim + "vector").get>(); + in_arr2d = it->getAttribute(prefix + delim + "arr2d") + .get>(); } REQUIRE(scalar == in_scalar); @@ -239,7 +242,6 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu REQUIRE(in_vector[i] == vector[i]); } - // deliberately the wrong size REQUIRE(in_arr2d.extent_int(0) == arr2d.extent_int(0)); REQUIRE(in_arr2d.extent_int(1) == arr2d.extent_int(1)); int nwrong = 1; From 8230f945bdd1ea84004a78545259ac2f16fd693e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 8 Oct 2024 11:01:11 +0200 Subject: [PATCH 59/70] Allow writing ParArray and Views to Params --- src/outputs/parthenon_opmd.cpp | 49 ++++++++++++++++++++++++---------- tst/unit/test_unit_params.cpp | 6 +++-- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index bb0986d9b549..cb3caba32b33 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -65,6 +66,25 @@ using namespace OutputUtils; namespace OpenPMDUtils { +template +auto GetFlatHostVecFromView(T view) { + // Take a view and return a flattendned (1D) std::vector that can then + // easily be passed to OpenPMD. + // Note, this function is not optimial as multiple (unnecessary) copies may be done. + // PG didn't come up with a smarter way but thinks that it's not a + // performance issue as this is only called for outputs (thus not that often) + // and for mostly small amounts of data. + // With a C++20 span we could probably direct reuse the host mirror data pointer. + auto view_h = Kokkos::create_mirror_view_and_copy(HostMemSpace(), view); + + using base_t = typename std::remove_pointer::type; + auto host_vec = std::vector(view_h.size()); + for (auto i = 0; i < view_h.size(); i++) { + host_vec[i] = view_h.data()[i]; + } + return host_vec; +} + template void WriteAllParamsOfType(const Params ¶ms, const std::string &prefix, openPMD::Iteration *it) { @@ -78,7 +98,17 @@ void WriteAllParamsOfType(const Params ¶ms, const std::string &prefix, // Thus we replace it. std::replace(full_path.begin(), full_path.end(), '/', delim[0]); - it->setAttribute(full_path, params.Get(key)); + if constexpr (implements::value) { + const auto &view = params.Get(key); + auto host_vec = GetFlatHostVecFromView(view); + it->setAttribute(full_path, host_vec); + } else if constexpr (is_specialization_of::value) { + const auto &view = params.Get(key).KokkosView(); + auto host_vec = GetFlatHostVecFromView(view); + it->setAttribute(full_path, host_vec); + } else { + it->setAttribute(full_path, params.Get(key)); + } } } } @@ -110,6 +140,8 @@ void WriteAllParams(const Params ¶ms, const std::string &prefix, WriteAllParams(params, prefix, it); WriteAllParamsOfType(params, prefix, it); // WriteAllParamsOfType>(params,prefix, it); + WriteAllParamsOfType>(params, prefix, it); + WriteAllParamsOfType>(params, prefix, it); } template @@ -300,19 +332,8 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, it.setTime(-1.0); it.setDt(-1.0); } - { // FIXME move this to dump params - // TODO(pgrete) Make this test piece work on devices - if constexpr (false) { - PARTHENON_INSTRUMENT_REGION("Dump Params"); - const auto view_d = - Kokkos::View("blub", 5, 3); - // Map a view onto a host allocation (so that we can call deep_copy) - auto host_vec = std::vector(view_d.size()); - Kokkos::View> - view_h(host_vec.data(), view_d.extent_int(0), view_d.extent_int(1)); - Kokkos::deep_copy(view_h, view_d); - it.setAttribute("blub", host_vec); - } + { + PARTHENON_INSTRUMENT_REGION("Dump Params"); for (const auto &[key, pkg] : pm->packages.AllPackages()) { using OpenPMDUtils::delim; diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index e220e3d98e5c..d0171c8bf629 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -233,8 +233,10 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu using parthenon::OpenPMDUtils::delim; in_scalar = it->getAttribute(prefix + delim + "scalar").get(); in_vector = it->getAttribute(prefix + delim + "vector").get>(); - in_arr2d = it->getAttribute(prefix + delim + "arr2d") - .get>(); + // Note that we also change the type here as ParArrays (or View in general) are + // downcasted to flattened vector. + // in_arr2d = it->getAttribute(prefix + delim + + // "arr2d").get>(); } REQUIRE(scalar == in_scalar); From e64ae7e7f72b24d79fa715062ed2106e785b2a92 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 8 Oct 2024 11:50:55 +0200 Subject: [PATCH 60/70] Read ParArray/View from opmd raw --- src/outputs/parthenon_opmd.cpp | 41 ++++++++++++++++++++++++++-------- tst/unit/test_unit_params.cpp | 24 ++++++++++++++++++-- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index cb3caba32b33..cbec6f3f7338 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -68,15 +68,36 @@ namespace OpenPMDUtils { template auto GetFlatHostVecFromView(T view) { - // Take a view and return a flattendned (1D) std::vector that can then - // easily be passed to OpenPMD. - // Note, this function is not optimial as multiple (unnecessary) copies may be done. - // PG didn't come up with a smarter way but thinks that it's not a - // performance issue as this is only called for outputs (thus not that often) - // and for mostly small amounts of data. - // With a C++20 span we could probably direct reuse the host mirror data pointer. + // Take a view and return a vector containing rank and dims and a flattened (1D) + // std::vector that can then easily be passed to OpenPMD. + // Note, this function is not + // optimial as multiple (unnecessary) copies may be done. PG didn't come up with a + // smarter way but thinks that it's not a performance issue as this is only called for + // outputs (thus not that often) and for mostly small amounts of data. With a C++20 span + // we could probably direct reuse the host mirror data pointer. auto view_h = Kokkos::create_mirror_view_and_copy(HostMemSpace(), view); + using base_t = typename std::remove_pointer::type; + auto host_vec = std::vector(view_h.size()); + for (auto i = 0; i < view_h.size(); i++) { + host_vec[i] = view_h.data()[i]; + } + // cpplint demands compile constants be all caps + constexpr auto RANK = static_cast(T::rank); + std::vector rank_and_dims(RANK + 1); + rank_and_dims[0] = RANK; + for (size_t d = 0; d < RANK; ++d) { + rank_and_dims[1 + d] = view.extent_int(d); + } + return std::make_tuple(rank_and_dims, host_vec); +} + +template +auto CopyFlatHostVecToView(const Vec &vec, T view) { + // Copy flat std::vector to view (doens't do any checks at the moment). + // As above, potentially extra copies are being made. + auto view_h = Kokkos::create_mirror_view(HostMemSpace(), view); + using base_t = typename std::remove_pointer::type; auto host_vec = std::vector(view_h.size()); for (auto i = 0; i < view_h.size(); i++) { @@ -100,11 +121,13 @@ void WriteAllParamsOfType(const Params ¶ms, const std::string &prefix, if constexpr (implements::value) { const auto &view = params.Get(key); - auto host_vec = GetFlatHostVecFromView(view); + auto [rank_and_dims, host_vec] = GetFlatHostVecFromView(view); + it->setAttribute(full_path + ".rankdims", rank_and_dims); it->setAttribute(full_path, host_vec); } else if constexpr (is_specialization_of::value) { const auto &view = params.Get(key).KokkosView(); - auto host_vec = GetFlatHostVecFromView(view); + auto [rank_and_dims, host_vec] = GetFlatHostVecFromView(view); + it->setAttribute(full_path + ".rankdims", rank_and_dims); it->setAttribute(full_path, host_vec); } else { it->setAttribute(full_path, params.Get(key)); diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index d0171c8bf629..3582d5513699 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -16,6 +16,7 @@ //======================================================================================== #include +#include #include #include @@ -29,6 +30,7 @@ #include "outputs/parthenon_opmd.hpp" #include "outputs/restart_hdf5.hpp" #include "outputs/restart_opmd.hpp" +#include "parthenon_array_generic.hpp" using parthenon::Params; using parthenon::Real; @@ -235,8 +237,26 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu in_vector = it->getAttribute(prefix + delim + "vector").get>(); // Note that we also change the type here as ParArrays (or View in general) are // downcasted to flattened vector. - // in_arr2d = it->getAttribute(prefix + delim + - // "arr2d").get>(); + auto rank_and_dims = it->getAttribute(prefix + delim + "arr2d.rankdims") + .get>(); + // Resize view. + using ViewType = + typename std::remove_reference::type; + ViewType::array_layout layout; + for (int d = 0; d < rank_and_dims[0]; ++d) { + layout.dimension[d] = rank_and_dims[1 + d]; + } + auto &in_arr2d_view = in_arr2d.KokkosView(); + in_arr2d_view = ViewType(in_arr2d_view.label(), layout); + auto view_h = + Kokkos::create_mirror_view(parthenon::HostMemSpace(), in_arr2d_view); + + auto in_arr2d_data = + it->getAttribute(prefix + delim + "arr2d").get>(); + for (auto i = 0; i < view_h.size(); i++) { + view_h.data()[i] = in_arr2d_data[i]; + } + Kokkos::deep_copy(in_arr2d_view, view_h); } REQUIRE(scalar == in_scalar); From b7897113d6981dacba5e02b1af5cb29c5d42a153 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 8 Oct 2024 12:39:02 +0200 Subject: [PATCH 61/70] Make basic parsing of Params work --- src/outputs/parthenon_opmd.cpp | 10 ++++---- src/outputs/restart_opmd.cpp | 45 ++++++++++++++++++++-------------- tst/unit/test_unit_params.cpp | 18 +++++++++----- 3 files changed, 44 insertions(+), 29 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index cbec6f3f7338..decb303a262b 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -150,8 +150,10 @@ void WriteAllParams(const Params ¶ms, const std::string &prefix, // WriteAllParamsOfMultipleTypes(pkg, it); } -void WriteAllParams(const Params ¶ms, const std::string &prefix, +void WriteAllParams(const Params ¶ms, const std::string &pkg_name, openPMD::Iteration *it) { + using OpenPMDUtils::delim; + const std::string prefix = "Params" + delim + pkg_name; // WriteAllParams(params, prefix, it); // check why this (vector of bool) doesn't // work WriteAllParams(params, prefix, it); @@ -358,11 +360,9 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, { PARTHENON_INSTRUMENT_REGION("Dump Params"); - for (const auto &[key, pkg] : pm->packages.AllPackages()) { - using OpenPMDUtils::delim; - const std::string prefix = "Params" + delim + pkg->label() + delim; + for (const auto &[pkg_name, pkg] : pm->packages.AllPackages()) { const auto ¶ms = pkg->AllParams(); - OpenPMDUtils::WriteAllParams(params, prefix, &it); + OpenPMDUtils::WriteAllParams(params, pkg_name, &it); } } // Then our own diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 1696d5535fc5..494b1c4fae16 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -27,13 +27,21 @@ namespace parthenon { //---------------------------------------------------------------------------------------- //! \fn void RestartReader::RestartReader(const std::string filename) // \brief Opens the restart file and stores appropriate file handle in fh_ -RestartReaderOPMD::RestartReaderOPMD(const char *filename) - : filename_(filename), series(filename, openPMD::Access::READ_ONLY +RestartReaderOPMD::RestartReaderOPMD(const char *filename) : filename_(filename) { + // This silly logic is required as the unit tests may or may not define MPI_PARALLEL but + // are always run in serial. #ifdef MPI_PARALLEL - , - MPI_COMM_WORLD + int mpi_initialized; + PARTHENON_MPI_CHECK(MPI_Initialized(&mpi_initialized)); + if (mpi_initialized) { + series = openPMD::Series(filename, openPMD::Access::READ_ONLY, MPI_COMM_WORLD); + } else { + series = openPMD::Series(filename, openPMD::Access::READ_ONLY); + } +#else + series = openPMD::Series(filename, openPMD::Access::READ_ONLY); + #endif - ) { PARTHENON_REQUIRE_THROWS( series.iterations.size() == 1, "Parthenon restarts should only contain one iteration/timestep."); @@ -120,9 +128,9 @@ std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, } template -void RestartReaderOPMD::ReadAllParamsOfType(const std::string &pkg_name, Params ¶ms) { - using OpenPMDUtils::delim; +void RestartReaderOPMD::ReadAllParamsOfType(const std::string &prefix, Params ¶ms) { for (const auto &key : params.GetKeys()) { + using OpenPMDUtils::delim; const auto type = params.GetType(key); auto mutability = params.GetMutability(key); if (type == std::type_index(typeid(T)) && mutability == Params::Mutability::Restart) { @@ -130,11 +138,10 @@ void RestartReaderOPMD::ReadAllParamsOfType(const std::string &pkg_name, Params for (const auto &attr : attrs) { std::cout << "Contains attribute: " << attr << std::endl; } - std::cout << "Reading '" - << "Params" + delim + pkg_name + delim + key + std::cout << "Reading '" << prefix + delim + key << "' with type: " << typeid(T).name() << std::endl; - auto val = it->getAttribute("Params" + delim + pkg_name + delim + key).get(); + auto val = it->getAttribute(prefix + delim + key).get(); params.Update(key, val); } } @@ -153,14 +160,16 @@ void RestartReaderOPMD::ReadAllParams(const std::string &pkg_name, Params &p) { // ReadAllParamsOfMultipleTypes(pkg, it); } void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { - ReadAllParams(pkg_name, p); - ReadAllParams(pkg_name, p); - ReadAllParams(pkg_name, p); - ReadAllParams(pkg_name, p); - ReadAllParams(pkg_name, p); - ReadAllParams(pkg_name, p); - ReadAllParams(pkg_name, p); - ReadAllParamsOfType(pkg_name, p); + using OpenPMDUtils::delim; + const auto prefix = "Params" + delim + pkg_name; + ReadAllParams(prefix, p); + ReadAllParams(prefix, p); + ReadAllParams(prefix, p); + ReadAllParams(prefix, p); + ReadAllParams(prefix, p); + ReadAllParams(prefix, p); + ReadAllParams(prefix, p); + ReadAllParamsOfType(prefix, p); } void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 3582d5513699..052231aca29a 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -192,7 +192,7 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu THEN("We can output") { std::string filename; - const std::string groupname = "params"; + const std::string groupname = "Params"; const std::string prefix = "test_pkg"; if constexpr (std::is_same_v) { using namespace parthenon::HDF5; @@ -233,12 +233,15 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu // Note that we're explicitly using `delim` here which tests the character // replacement of '/' in the WriteAllParams function. using parthenon::OpenPMDUtils::delim; - in_scalar = it->getAttribute(prefix + delim + "scalar").get(); - in_vector = it->getAttribute(prefix + delim + "vector").get>(); + in_scalar = + it->getAttribute(groupname + delim + prefix + delim + "scalar").get(); + in_vector = it->getAttribute(groupname + delim + prefix + delim + "vector") + .get>(); // Note that we also change the type here as ParArrays (or View in general) are // downcasted to flattened vector. - auto rank_and_dims = it->getAttribute(prefix + delim + "arr2d.rankdims") - .get>(); + auto rank_and_dims = + it->getAttribute(groupname + delim + prefix + delim + "arr2d.rankdims") + .get>(); // Resize view. using ViewType = typename std::remove_reference::type; @@ -277,7 +280,7 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu REQUIRE(nwrong == 0); } - AND_THEN("We can restart a params object from the HDF5 file") { + AND_THEN("We can restart a params object from the file") { Params rparams; // init the params object to restart into @@ -302,6 +305,9 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu const H5G obj = H5G::FromHIDCheck(H5Oopen(file, groupname.c_str(), H5P_DEFAULT)); rparams.ReadFromRestart(prefix, obj); + } else if constexpr (std::is_same_v) { + auto resfile = RestartReaderOPMD(filename.c_str()); + resfile.ReadParams(prefix, rparams); } AND_THEN("The values for the restartable params are updated to match the file") { From 466ecd2fdb7a0108623854e202d0a3137842763c Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 8 Oct 2024 14:37:10 +0200 Subject: [PATCH 62/70] Restore view from opmd params --- src/outputs/parthenon_opmd.cpp | 21 +++++++++++++-------- src/outputs/restart_opmd.cpp | 24 ++++++++++++++++++++---- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index decb303a262b..03b6fffbeb95 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -92,18 +92,24 @@ auto GetFlatHostVecFromView(T view) { return std::make_tuple(rank_and_dims, host_vec); } -template -auto CopyFlatHostVecToView(const Vec &vec, T view) { - // Copy flat std::vector to view (doens't do any checks at the moment). - // As above, potentially extra copies are being made. +template +auto RestoreViewAttribute(const std::string &full_path, T &view, openPMD::Iteration *it) { + auto rank_and_dims = + it->getAttribute(full_path + ".rankdims").get>(); + // Resize view. + typename T::array_layout layout; + for (int d = 0; d < rank_and_dims[0]; ++d) { + layout.dimension[d] = rank_and_dims[1 + d]; + } + Kokkos::resize(Kokkos::WithoutInitializing, view, layout); auto view_h = Kokkos::create_mirror_view(HostMemSpace(), view); using base_t = typename std::remove_pointer::type; - auto host_vec = std::vector(view_h.size()); + auto flat_data = it->getAttribute(full_path).get>(); for (auto i = 0; i < view_h.size(); i++) { - host_vec[i] = view_h.data()[i]; + view_h.data()[i] = flat_data[i]; } - return host_vec; + Kokkos::deep_copy(view, view_h); } template @@ -112,7 +118,6 @@ void WriteAllParamsOfType(const Params ¶ms, const std::string &prefix, for (const auto &key : params.GetKeys()) { const auto type = params.GetType(key); if (type == std::type_index(typeid(T))) { - // auto typed_ptr = dynamic_cast *>((p.second).get()); auto full_path = prefix + delim + key; // The '/' is kind of a reserved character in the OpenPMD standard, which results // in attribute keys with said character not being exposed. diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 494b1c4fae16..532ad28ecc2c 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -134,14 +134,30 @@ void RestartReaderOPMD::ReadAllParamsOfType(const std::string &prefix, Params &p const auto type = params.GetType(key); auto mutability = params.GetMutability(key); if (type == std::type_index(typeid(T)) && mutability == Params::Mutability::Restart) { + auto full_path = prefix + delim + key; + // The '/' is kind of a reserved character in the OpenPMD standard, which results + // in attribute keys with said character not being exposed. + // Thus we replace it. + std::replace(full_path.begin(), full_path.end(), '/', delim[0]); + auto attrs = it->attributes(); for (const auto &attr : attrs) { std::cout << "Contains attribute: " << attr << std::endl; } - std::cout << "Reading '" << prefix + delim + key - << "' with type: " << typeid(T).name() << std::endl; - - auto val = it->getAttribute(prefix + delim + key).get(); + std::cout << "Reading '" << full_path << "' with type: " << typeid(T).name() + << std::endl; + + T val; + if constexpr (implements::value) { + val = params.Get(key); + RestoreViewAttribute(full_path, val, it); + } else if constexpr (is_specialization_of::value) { + val = params.Get(key); + auto &view = val.KokkosView(); + RestoreViewAttribute(full_path, view, it); + } else { + val = it->getAttribute(full_path).get(); + } params.Update(key, val); } } From 3aa4c78e85376ec160eda6c9ab78650f084c0bb5 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 8 Oct 2024 16:30:34 +0200 Subject: [PATCH 63/70] Fix manual pararray reading --- src/outputs/parthenon_opmd.cpp | 3 ++- src/outputs/parthenon_opmd.hpp | 3 +++ src/outputs/restart_opmd.cpp | 2 ++ tst/unit/test_unit_params.cpp | 48 +++++++++++++++++++++++----------- 4 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 03b6fffbeb95..9b062b52cf38 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -93,7 +93,7 @@ auto GetFlatHostVecFromView(T view) { } template -auto RestoreViewAttribute(const std::string &full_path, T &view, openPMD::Iteration *it) { +void RestoreViewAttribute(const std::string &full_path, T &view, openPMD::Iteration *it) { auto rank_and_dims = it->getAttribute(full_path + ".rankdims").get>(); // Resize view. @@ -172,6 +172,7 @@ void WriteAllParams(const Params ¶ms, const std::string &pkg_name, // WriteAllParamsOfType>(params,prefix, it); WriteAllParamsOfType>(params, prefix, it); WriteAllParamsOfType>(params, prefix, it); + WriteAllParamsOfType>(params, prefix, it); } template diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index 23c3390c47f7..9e0c04e99a4c 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -21,6 +21,9 @@ namespace parthenon { namespace OpenPMDUtils { +template +void RestoreViewAttribute(const std::string &full_path, T &view, openPMD::Iteration *it); + void WriteAllParams(const Params ¶ms, const std::string &prefix, openPMD::Iteration *it); diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 532ad28ecc2c..f34aaaebb640 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -186,6 +186,8 @@ void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { ReadAllParams(prefix, p); ReadAllParams(prefix, p); ReadAllParamsOfType(prefix, p); + // ReadAllParamsOfType>(prefix, p); + // ReadAllParamsOfType>(prefix, p); } void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 052231aca29a..b164a7516565 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -182,13 +182,13 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu Kokkos::deep_copy(arr2d, arr2d_h); params.Add("arr2d", arr2d); - parthenon::HostArray2D hostarr("hostarr2d", 2, 3); + parthenon::HostArray2D hostarr2d("hostarr2d", 2, 3); for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { - hostarr(i, j) = 2 * i + j + 1; + hostarr2d(i, j) = 2 * i + j + 1; } } - params.Add("hostarr2d", hostarr, restart); + params.Add("hostarr2d", hostarr2d, restart); THEN("We can output") { std::string filename; @@ -216,6 +216,7 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu std::vector in_vector; // deliberately the wrong size parthenon::ParArray2D in_arr2d("myarr", 1, 1); + parthenon::HostArray2D in_hostarr2d("hostarr2d", 2, 3); if constexpr (std::is_same_v) { @@ -227,20 +228,28 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu HDF5ReadAttribute(obj, prefix + "/scalar", in_scalar); HDF5ReadAttribute(obj, prefix + "/vector", in_vector); HDF5ReadAttribute(obj, prefix + "/arr2d", in_arr2d); + HDF5ReadAttribute(obj, prefix + "/hostarr2d", in_hostarr2d); } else if constexpr (std::is_same_v) { auto series = openPMD::Series(filename, openPMD::Access::READ_ONLY); - auto it = std::make_unique(series.iterations[0]); + auto it = series.iterations[0]; // Note that we're explicitly using `delim` here which tests the character // replacement of '/' in the WriteAllParams function. using parthenon::OpenPMDUtils::delim; in_scalar = - it->getAttribute(groupname + delim + prefix + delim + "scalar").get(); - in_vector = it->getAttribute(groupname + delim + prefix + delim + "vector") + it.getAttribute(groupname + delim + prefix + delim + "scalar").get(); + in_vector = it.getAttribute(groupname + delim + prefix + delim + "vector") .get>(); - // Note that we also change the type here as ParArrays (or View in general) are - // downcasted to flattened vector. + + // auto &tmp_view = in_arr2d.KokkosView(); + // using ViewType = + // typename std::remove_reference::type; + // parthenon::OpenPMDUtils::RestoreViewAttribute( + // groupname + delim + prefix + delim + "arr2d", tmp_view, &it); + + // Note that we also change the type here as ParArrays (or View in general) + // are downcasted to flattened vector. auto rank_and_dims = - it->getAttribute(groupname + delim + prefix + delim + "arr2d.rankdims") + it.getAttribute(groupname + delim + prefix + delim + "arr2d.rankdims") .get>(); // Resize view. using ViewType = @@ -255,7 +264,8 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu Kokkos::create_mirror_view(parthenon::HostMemSpace(), in_arr2d_view); auto in_arr2d_data = - it->getAttribute(prefix + delim + "arr2d").get>(); + it.getAttribute(groupname + delim + prefix + delim + "arr2d") + .get>(); for (auto i = 0; i < view_h.size(); i++) { view_h.data()[i] = in_arr2d_data[i]; } @@ -278,6 +288,14 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu }, nwrong); REQUIRE(nwrong == 0); + + REQUIRE(in_hostarr2d.extent_int(0) == hostarr2d.extent_int(0)); + REQUIRE(in_hostarr2d.extent_int(1) == hostarr2d.extent_int(1)); + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 3; ++j) { + REQUIRE(hostarr2d(i, j) == in_hostarr2d(i, j)); + } + } } AND_THEN("We can restart a params object from the file") { @@ -318,11 +336,11 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu REQUIRE(test_bool == boolscalar); auto test_hostarr = params.Get>("hostarr2d"); - REQUIRE(test_hostarr.extent_int(0) == hostarr.extent_int(0)); - REQUIRE(test_hostarr.extent_int(1) == hostarr.extent_int(1)); - for (int i = 0; i < hostarr.extent_int(0); ++i) { - for (int j = 0; j < hostarr.extent_int(1); ++j) { - REQUIRE(test_hostarr(i, j) == hostarr(i, j)); + REQUIRE(test_hostarr.extent_int(0) == hostarr2d.extent_int(0)); + REQUIRE(test_hostarr.extent_int(1) == hostarr2d.extent_int(1)); + for (int i = 0; i < hostarr2d.extent_int(0); ++i) { + for (int j = 0; j < hostarr2d.extent_int(1); ++j) { + REQUIRE(test_hostarr(i, j) == hostarr2d(i, j)); } } } From 4c1c70fb830f9efc19962edf3ca5218763e39604 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 8 Oct 2024 22:38:59 +0200 Subject: [PATCH 64/70] Make linter happy? --- src/outputs/output_utils.cpp | 1 + src/outputs/restart_opmd.cpp | 3 ++- tst/unit/test_unit_params.cpp | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/outputs/output_utils.cpp b/src/outputs/output_utils.cpp index 927853d2f7f7..3b7de06d89a0 100644 --- a/src/outputs/output_utils.cpp +++ b/src/outputs/output_utils.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index f34aaaebb640..e7f51e437fb6 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -233,7 +234,7 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block } } } // loop over components - } // loop over blocks + } // loop over blocks // Now actually read the registered chunks form disk it->seriesFlush(); diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index b164a7516565..8fad599eb81b 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -1,6 +1,6 @@ //======================================================================================== -// Athena++ astrophysical MHD code -// Copyright(C) 2014 James M. Stone and other code contributors +// Parthenon performance portable AMR framework +// Copyright(C) 2020-2024 The Parthenon collaboration // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== // (C) (or copyright) 2020-2021. Triad National Security, LLC. All rights reserved. @@ -16,6 +16,7 @@ //======================================================================================== #include +#include #include #include @@ -219,7 +220,6 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu parthenon::HostArray2D in_hostarr2d("hostarr2d", 2, 3); if constexpr (std::is_same_v) { - H5F file = H5F::FromHIDCheck(H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT)); const H5O obj = From 8a1850b2de73d55624a96e4598a04bd5d85ed95b Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 9 Oct 2024 22:48:17 +0200 Subject: [PATCH 65/70] Make restoring HostViews possible --- src/outputs/parthenon_opmd.cpp | 26 +++----------------------- src/outputs/restart_opmd.cpp | 4 ++-- src/outputs/restart_opmd.hpp | 21 +++++++++++++++++++++ tst/unit/test_unit_params.cpp | 4 ++-- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 9b062b52cf38..ddd59c59c2fb 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -92,26 +92,6 @@ auto GetFlatHostVecFromView(T view) { return std::make_tuple(rank_and_dims, host_vec); } -template -void RestoreViewAttribute(const std::string &full_path, T &view, openPMD::Iteration *it) { - auto rank_and_dims = - it->getAttribute(full_path + ".rankdims").get>(); - // Resize view. - typename T::array_layout layout; - for (int d = 0; d < rank_and_dims[0]; ++d) { - layout.dimension[d] = rank_and_dims[1 + d]; - } - Kokkos::resize(Kokkos::WithoutInitializing, view, layout); - auto view_h = Kokkos::create_mirror_view(HostMemSpace(), view); - - using base_t = typename std::remove_pointer::type; - auto flat_data = it->getAttribute(full_path).get>(); - for (auto i = 0; i < view_h.size(); i++) { - view_h.data()[i] = flat_data[i]; - } - Kokkos::deep_copy(view, view_h); -} - template void WriteAllParamsOfType(const Params ¶ms, const std::string &prefix, openPMD::Iteration *it) { @@ -636,10 +616,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } } } // loop over components - } // out_var->IsAllocated() - } // loop over blocks + } // out_var->IsAllocated() + } // loop over blocks it.seriesFlush(); - } // loop over vars + } // loop over vars Kokkos::Profiling::popRegion(); // write all variable data // -------------------------------------------------------------------------------- // diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index e7f51e437fb6..532137cc2019 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -155,7 +155,7 @@ void RestartReaderOPMD::ReadAllParamsOfType(const std::string &prefix, Params &p } else if constexpr (is_specialization_of::value) { val = params.Get(key); auto &view = val.KokkosView(); - RestoreViewAttribute(full_path, view, it); + RestoreViewAttribute(full_path, view); } else { val = it->getAttribute(full_path).get(); } @@ -188,7 +188,7 @@ void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { ReadAllParams(prefix, p); ReadAllParamsOfType(prefix, p); // ReadAllParamsOfType>(prefix, p); - // ReadAllParamsOfType>(prefix, p); + ReadAllParamsOfType>(prefix, p); } void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index 4353bf59e233..f986e84b8b97 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -119,6 +119,27 @@ class RestartReaderOPMD : public RestartReader { void ReadParams(const std::string &name, Params &p) override; + template + void RestoreViewAttribute(const std::string &full_path, T &view) { + auto rank_and_dims = + it->getAttribute(full_path + ".rankdims").get>(); + // Resize view. + typename T::array_layout layout; + for (int d = 0; d < rank_and_dims[0]; ++d) { + layout.dimension[d] = rank_and_dims[1 + d]; + } + // Cannot use Kokkos::resize here as it's ambiguous at this point. + // Also, resize() interally also just create a new view. + view = T(Kokkos::view_alloc(Kokkos::WithoutInitializing, view.label()), layout); + auto view_h = Kokkos::create_mirror_view(HostMemSpace(), view); + + using base_t = typename std::remove_pointer::type; + auto flat_data = it->getAttribute(full_path).get>(); + for (auto i = 0; i < view_h.size(); i++) { + view_h.data()[i] = flat_data[i]; + } + Kokkos::deep_copy(view, view_h); + } // closes out the restart file // perhaps belongs in a destructor? void Close(); diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 8fad599eb81b..33b1fdef122e 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -293,7 +293,7 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu REQUIRE(in_hostarr2d.extent_int(1) == hostarr2d.extent_int(1)); for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { - REQUIRE(hostarr2d(i, j) == in_hostarr2d(i, j)); + REQUIRE((hostarr2d(i, j) == in_hostarr2d(i, j) || true)); } } } @@ -335,7 +335,7 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu auto test_bool = rparams.Get("boolscalar"); REQUIRE(test_bool == boolscalar); - auto test_hostarr = params.Get>("hostarr2d"); + auto test_hostarr = rparams.Get>("hostarr2d"); REQUIRE(test_hostarr.extent_int(0) == hostarr2d.extent_int(0)); REQUIRE(test_hostarr.extent_int(1) == hostarr2d.extent_int(1)); for (int i = 0; i < hostarr2d.extent_int(0); ++i) { From 1f02ffaf102632dfb68559c406dc88422c7000ba Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 9 Oct 2024 23:00:41 +0200 Subject: [PATCH 66/70] Make reading host arrays work using the direct interface in the unit test --- tst/unit/test_unit_params.cpp | 44 +++++++++++------------------------ 1 file changed, 13 insertions(+), 31 deletions(-) diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 33b1fdef122e..9cd9a08542d9 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -235,41 +235,23 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu // Note that we're explicitly using `delim` here which tests the character // replacement of '/' in the WriteAllParams function. using parthenon::OpenPMDUtils::delim; + in_scalar = it.getAttribute(groupname + delim + prefix + delim + "scalar").get(); + in_vector = it.getAttribute(groupname + delim + prefix + delim + "vector") .get>(); - // auto &tmp_view = in_arr2d.KokkosView(); - // using ViewType = - // typename std::remove_reference::type; - // parthenon::OpenPMDUtils::RestoreViewAttribute( - // groupname + delim + prefix + delim + "arr2d", tmp_view, &it); - - // Note that we also change the type here as ParArrays (or View in general) - // are downcasted to flattened vector. - auto rank_and_dims = - it.getAttribute(groupname + delim + prefix + delim + "arr2d.rankdims") - .get>(); - // Resize view. - using ViewType = - typename std::remove_reference::type; - ViewType::array_layout layout; - for (int d = 0; d < rank_and_dims[0]; ++d) { - layout.dimension[d] = rank_and_dims[1 + d]; - } - auto &in_arr2d_view = in_arr2d.KokkosView(); - in_arr2d_view = ViewType(in_arr2d_view.label(), layout); - auto view_h = - Kokkos::create_mirror_view(parthenon::HostMemSpace(), in_arr2d_view); - - auto in_arr2d_data = - it.getAttribute(groupname + delim + prefix + delim + "arr2d") - .get>(); - for (auto i = 0; i < view_h.size(); i++) { - view_h.data()[i] = in_arr2d_data[i]; - } - Kokkos::deep_copy(in_arr2d_view, view_h); + // Technically, we're not reading "directly" here but the restart reader ctor + // literally just opens the file. + auto resfile = RestartReaderOPMD(filename.c_str()); + auto &in_arr2d_v = in_arr2d.KokkosView(); + resfile.RestoreViewAttribute(groupname + delim + prefix + delim + "arr2d", + in_arr2d_v); + + auto &in_hostarr2d_v = in_hostarr2d.KokkosView(); + resfile.RestoreViewAttribute(groupname + delim + prefix + delim + "hostarr2d", + in_hostarr2d_v); } REQUIRE(scalar == in_scalar); @@ -293,7 +275,7 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu REQUIRE(in_hostarr2d.extent_int(1) == hostarr2d.extent_int(1)); for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { - REQUIRE((hostarr2d(i, j) == in_hostarr2d(i, j) || true)); + REQUIRE(hostarr2d(i, j) == in_hostarr2d(i, j)); } } } From d6565c5b97f85574ad599fe9362bc559cdb8040e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 9 Oct 2024 23:44:50 +0200 Subject: [PATCH 67/70] Expands types for read/write to all vec --- src/CMakeLists.txt | 1 + src/outputs/output_attr.hpp | 45 ++++++++++++++++++++++++++++++++++ src/outputs/parthenon_hdf5.hpp | 22 +---------------- src/outputs/parthenon_opmd.cpp | 15 ++++-------- src/outputs/restart_opmd.cpp | 10 +++----- tst/unit/test_unit_params.cpp | 15 ++++++++++++ 6 files changed, 71 insertions(+), 37 deletions(-) create mode 100644 src/outputs/output_attr.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6b4c5fc54108..c2a530664674 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -188,6 +188,7 @@ add_library(parthenon outputs/history.cpp outputs/io_wrapper.cpp outputs/io_wrapper.hpp + outputs/output_attr.hpp outputs/output_utils.cpp outputs/output_utils.hpp outputs/outputs.cpp diff --git a/src/outputs/output_attr.hpp b/src/outputs/output_attr.hpp new file mode 100644 index 000000000000..20e471c4c415 --- /dev/null +++ b/src/outputs/output_attr.hpp @@ -0,0 +1,45 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2023-2024 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== + +#ifndef OUTPUTS_OUTPUT_ATTR_HPP_ +#define OUTPUTS_OUTPUT_ATTR_HPP_ + +// JMM: This could probably be done with template magic but I think +// using a macro is honestly the simplest and cleanest solution here. +// Template solution would be to define a variatic class to conain the +// list of types and then a hierarchy of structs/functions to turn +// that into function calls. Preprocessor seems easier, given we're +// not manipulating this list in any way. +// The following types are the ones we allow to be stored as attributes in outputs +// (specifically within Params). +#define PARTHENON_ATTR_VALID_VEC_TYPES(T) \ + T, std::vector, ParArray1D, ParArray2D, ParArray3D, HostArray1D, \ + HostArray2D, HostArray3D, Kokkos::View, Kokkos::View, \ + ParArrayND, ParArrayHost +// JMM: This is the list of template specializations we +// "pre-instantiate" We only pre-instantiate device memory, not host +// memory. The reason is that when building with the Kokkos serial +// backend, DevMemSpace and HostMemSpace are the same and so this +// resolves to the same type in the macro, which causes problems. +#define PARTHENON_ATTR_FOREACH_VECTOR_TYPE(T) \ + PARTHENON_ATTR_APPLY(T); \ + PARTHENON_ATTR_APPLY(Kokkos::View); \ + PARTHENON_ATTR_APPLY(Kokkos::View); \ + PARTHENON_ATTR_APPLY(Kokkos::View); \ + PARTHENON_ATTR_APPLY(device_view_t) + +#endif // OUTPUTS_OUTPUT_ATTR_HPP_ \ No newline at end of file diff --git a/src/outputs/parthenon_hdf5.hpp b/src/outputs/parthenon_hdf5.hpp index 72deaab90681..b2c809c2fb0d 100644 --- a/src/outputs/parthenon_hdf5.hpp +++ b/src/outputs/parthenon_hdf5.hpp @@ -20,29 +20,9 @@ #include "config.hpp" #include "kokkos_abstraction.hpp" +#include "output_attr.hpp" #include "parthenon_arrays.hpp" -// JMM: This could probably be done with template magic but I think -// using a macro is honestly the simplest and cleanest solution here. -// Template solution would be to define a variatic class to conain the -// list of types and then a hierarchy of structs/functions to turn -// that into function calls. Preprocessor seems easier, given we're -// not manipulating this list in any way. -#define PARTHENON_ATTR_VALID_VEC_TYPES(T) \ - T, std::vector, ParArray1D, ParArray2D, ParArray3D, HostArray1D, \ - HostArray2D, HostArray3D, Kokkos::View, Kokkos::View, \ - ParArrayND, ParArrayHost -// JMM: This is the list of template specializations we -// "pre-instantiate" We only pre-instantiate device memory, not host -// memory. The reason is that when building with the Kokkos serial -// backend, DevMemSpace and HostMemSpace are the same and so this -// resolves to the same type in the macro, which causes problems. -#define PARTHENON_ATTR_FOREACH_VECTOR_TYPE(T) \ - PARTHENON_ATTR_APPLY(T); \ - PARTHENON_ATTR_APPLY(Kokkos::View); \ - PARTHENON_ATTR_APPLY(Kokkos::View); \ - PARTHENON_ATTR_APPLY(Kokkos::View); \ - PARTHENON_ATTR_APPLY(device_view_t) // Only proceed if HDF5 output enabled #ifdef ENABLE_HDF5 diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index ddd59c59c2fb..f2b588e84e20 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -48,6 +48,7 @@ #include "openPMD/Mesh.hpp" #include "openPMD/ParticleSpecies.hpp" #include "openPMD/Series.hpp" +#include "outputs/output_attr.hpp" #include "outputs/output_utils.hpp" #include "outputs/outputs.hpp" #include "outputs/parthenon_opmd.hpp" @@ -130,17 +131,16 @@ void WriteAllParamsOfMultipleTypes(const Params ¶ms, const std::string &pref template void WriteAllParams(const Params ¶ms, const std::string &prefix, openPMD::Iteration *it) { - WriteAllParamsOfMultipleTypes>(params, prefix, it); - // TODO(pgrete) check why this doens't work, i.e., which type is causing problems - // WriteAllParamsOfMultipleTypes(pkg, it); + WriteAllParamsOfMultipleTypes(params, prefix, it); } void WriteAllParams(const Params ¶ms, const std::string &pkg_name, openPMD::Iteration *it) { using OpenPMDUtils::delim; const std::string prefix = "Params" + delim + pkg_name; - // WriteAllParams(params, prefix, it); // check why this (vector of bool) doesn't - // work + // check why this (vector of bool) doesn't work + // WriteAllParams(params, prefix, it); + WriteAllParamsOfType(params, prefix, it); WriteAllParams(params, prefix, it); WriteAllParams(params, prefix, it); WriteAllParams(params, prefix, it); @@ -148,11 +148,6 @@ void WriteAllParams(const Params ¶ms, const std::string &pkg_name, WriteAllParams(params, prefix, it); WriteAllParams(params, prefix, it); WriteAllParams(params, prefix, it); - WriteAllParamsOfType(params, prefix, it); - // WriteAllParamsOfType>(params,prefix, it); - WriteAllParamsOfType>(params, prefix, it); - WriteAllParamsOfType>(params, prefix, it); - WriteAllParamsOfType>(params, prefix, it); } template diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 532137cc2019..249f1d7266b8 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -18,6 +18,7 @@ #include "interface/params.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" +#include "outputs/output_attr.hpp" #include "outputs/parthenon_opmd.hpp" #include "outputs/restart.hpp" #include "outputs/restart_opmd.hpp" @@ -151,7 +152,7 @@ void RestartReaderOPMD::ReadAllParamsOfType(const std::string &prefix, Params &p T val; if constexpr (implements::value) { val = params.Get(key); - RestoreViewAttribute(full_path, val, it); + RestoreViewAttribute(full_path, val); } else if constexpr (is_specialization_of::value) { val = params.Get(key); auto &view = val.KokkosView(); @@ -172,9 +173,7 @@ void RestartReaderOPMD::ReadAllParamsOfMultipleTypes(const std::string &pkg_name template void RestartReaderOPMD::ReadAllParams(const std::string &pkg_name, Params &p) { - ReadAllParamsOfMultipleTypes>(pkg_name, p); - // TODO(pgrete) check why this doens't work, i.e., which type is causing problems - // ReadAllParamsOfMultipleTypes(pkg, it); + ReadAllParamsOfMultipleTypes(pkg_name, p); } void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { using OpenPMDUtils::delim; @@ -186,9 +185,8 @@ void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { ReadAllParams(prefix, p); ReadAllParams(prefix, p); ReadAllParams(prefix, p); + // TODO(pgrete) same as for the writing. fix vec of bool ReadAllParamsOfType(prefix, p); - // ReadAllParamsOfType>(prefix, p); - ReadAllParamsOfType>(prefix, p); } void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 9cd9a08542d9..994f17727387 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -183,6 +183,16 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu Kokkos::deep_copy(arr2d, arr2d_h); params.Add("arr2d", arr2d); + // "Vectors" of bools have some special sauce under the hood so let's try the logic + // with a plain view + Kokkos::View bool1d("boolview", 10); + auto bool1d_h = Kokkos::create_mirror_view(bool1d); + for (int i = 0; i < 10; ++i) { + bool1d_h(i) = i % 2; + } + Kokkos::deep_copy(bool1d, bool1d_h); + params.Add("bool1d", bool1d); + parthenon::HostArray2D hostarr2d("hostarr2d", 2, 3); for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { @@ -218,6 +228,7 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu // deliberately the wrong size parthenon::ParArray2D in_arr2d("myarr", 1, 1); parthenon::HostArray2D in_hostarr2d("hostarr2d", 2, 3); + Kokkos::View in_bool1d("in_bool1d", 5); if constexpr (std::is_same_v) { H5F file = @@ -229,6 +240,7 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu HDF5ReadAttribute(obj, prefix + "/vector", in_vector); HDF5ReadAttribute(obj, prefix + "/arr2d", in_arr2d); HDF5ReadAttribute(obj, prefix + "/hostarr2d", in_hostarr2d); + HDF5ReadAttribute(obj, prefix + "/bool1d", in_bool1d); } else if constexpr (std::is_same_v) { auto series = openPMD::Series(filename, openPMD::Access::READ_ONLY); auto it = series.iterations[0]; @@ -252,6 +264,9 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu auto &in_hostarr2d_v = in_hostarr2d.KokkosView(); resfile.RestoreViewAttribute(groupname + delim + prefix + delim + "hostarr2d", in_hostarr2d_v); + // TODO(pgrete) make this work and also add checks for correctness below + // resfile.RestoreViewAttribute(groupname + delim + prefix + delim + "bool1d", + // in_bool1d); } REQUIRE(scalar == in_scalar); From 6065213f8042b61b0154225fd2715ef5b4ff206c Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 9 Oct 2024 23:50:45 +0200 Subject: [PATCH 68/70] Remove debug info --- src/outputs/restart_opmd.cpp | 11 ++--------- src/outputs/restart_opmd.hpp | 4 ++-- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 249f1d7266b8..c83eb1f6201c 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -142,13 +142,6 @@ void RestartReaderOPMD::ReadAllParamsOfType(const std::string &prefix, Params &p // Thus we replace it. std::replace(full_path.begin(), full_path.end(), '/', delim[0]); - auto attrs = it->attributes(); - for (const auto &attr : attrs) { - std::cout << "Contains attribute: " << attr << std::endl; - } - std::cout << "Reading '" << full_path << "' with type: " << typeid(T).name() - << std::endl; - T val; if constexpr (implements::value) { val = params.Get(key); @@ -166,9 +159,9 @@ void RestartReaderOPMD::ReadAllParamsOfType(const std::string &prefix, Params &p } template -void RestartReaderOPMD::ReadAllParamsOfMultipleTypes(const std::string &pkg_name, +void RestartReaderOPMD::ReadAllParamsOfMultipleTypes(const std::string &prefix, Params &p) { - ([&] { ReadAllParamsOfType(pkg_name, p); }(), ...); + ([&] { ReadAllParamsOfType(prefix, p); }(), ...); } template diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index f986e84b8b97..5f79a7662012 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -153,9 +153,9 @@ class RestartReaderOPMD : public RestartReader { std::unique_ptr it; template - void ReadAllParamsOfType(const std::string &pkg_name, Params ¶ms); + void ReadAllParamsOfType(const std::string &prefix, Params ¶ms); template - void ReadAllParamsOfMultipleTypes(const std::string &pkg_name, Params &p); + void ReadAllParamsOfMultipleTypes(const std::string &prefix, Params &p); template void ReadAllParams(const std::string &pkg_name, Params &p); }; From 503a5b61c2ff7c35b65f3369b6d7c4236eccc6c9 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Sat, 9 Nov 2024 16:32:59 +0100 Subject: [PATCH 69/70] Fix bc interface from PR 1177 --- src/outputs/parthenon_opmd.cpp | 15 +++++++++------ src/outputs/restart_opmd.cpp | 3 --- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index f2b588e84e20..201ae74f9fec 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -398,12 +398,15 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, pm->mesh_size.nx(X3DIR)}); // Boundary conditions - std::vector boundary_condition_str(BOUNDARY_NFACES); - for (size_t i = 0; i < boundary_condition_str.size(); i++) { - boundary_condition_str[i] = GetBoundaryString(pm->mesh_bcs[i]); - } - - it.setAttribute("BoundaryConditions", boundary_condition_str); + auto arr_to_vec = [](const auto &arr) { + std::vector vec(BOUNDARY_NFACES); + for (int i = 0; i < BOUNDARY_NFACES; i++) { + vec[i] = arr.at(i); + } + return vec; + }; + it.setAttribute("BoundaryConditions", arr_to_vec(pm->mesh_bc_names)); + it.setAttribute("SwarmBoundaryConditions", arr_to_vec(pm->mesh_swarm_bc_names)); } // Info section Kokkos::Profiling::popRegion(); // write Attributes diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index c83eb1f6201c..11f9943124a1 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -77,9 +77,6 @@ RestartReaderOPMD::MeshInfo RestartReaderOPMD::GetMeshInfo() const { mesh_info.nbtotal = it->getAttribute("NumMeshBlocks").get(); mesh_info.root_level = it->getAttribute("RootLevel").get(); - mesh_info.bound_cond = - it->getAttribute("BoundaryConditions").get>(); - mesh_info.block_size = it->getAttribute("MeshBlockSize").get>(); mesh_info.includes_ghost = it->getAttribute("IncludesGhost").get(); mesh_info.n_ghost = it->getAttribute("NGhost").get(); From 4769a632b40f9344db7d8bb44ecfa83de8c74f5d Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 29 Nov 2024 12:12:13 +0100 Subject: [PATCH 70/70] Fix output numbering for triggered opmd outputs --- src/outputs/output_attr.hpp | 4 +++- src/outputs/parthenon_opmd.cpp | 18 ++++++++++-------- tst/unit/test_unit_params.cpp | 4 ++-- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/outputs/output_attr.hpp b/src/outputs/output_attr.hpp index 20e471c4c415..8607157e09a2 100644 --- a/src/outputs/output_attr.hpp +++ b/src/outputs/output_attr.hpp @@ -18,6 +18,8 @@ #ifndef OUTPUTS_OUTPUT_ATTR_HPP_ #define OUTPUTS_OUTPUT_ATTR_HPP_ +#include + // JMM: This could probably be done with template magic but I think // using a macro is honestly the simplest and cleanest solution here. // Template solution would be to define a variatic class to conain the @@ -42,4 +44,4 @@ PARTHENON_ATTR_APPLY(Kokkos::View); \ PARTHENON_ATTR_APPLY(device_view_t) -#endif // OUTPUTS_OUTPUT_ATTR_HPP_ \ No newline at end of file +#endif // OUTPUTS_OUTPUT_ATTR_HPP_ diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 201ae74f9fec..c0ee2aa434b2 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -614,10 +614,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } } } // loop over components - } // out_var->IsAllocated() - } // loop over blocks + } // out_var->IsAllocated() + } // loop over blocks it.seriesFlush(); - } // loop over vars + } // loop over vars Kokkos::Profiling::popRegion(); // write all variable data // -------------------------------------------------------------------------------- // @@ -660,11 +660,13 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, series.close(); #endif // ifndef PARTHENON_ENABLE_OPENPMD - // advance output parameters - output_params.file_number++; - output_params.next_time += output_params.dt; - pin->SetInteger(output_params.block_name, "file_number", output_params.file_number); - pin->SetReal(output_params.block_name, "next_time", output_params.next_time); + // advance output parameters if this is not a triggered (now or final) output + if (signal == SignalHandler::OutputSignal::none) { + output_params.file_number++; + output_params.next_time += output_params.dt; + pin->SetInteger(output_params.block_name, "file_number", output_params.file_number); + pin->SetReal(output_params.block_name, "next_time", output_params.next_time); + } } } // namespace parthenon diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 994f17727387..aae19bd1916c 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -264,9 +264,9 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu auto &in_hostarr2d_v = in_hostarr2d.KokkosView(); resfile.RestoreViewAttribute(groupname + delim + prefix + delim + "hostarr2d", in_hostarr2d_v); - // TODO(pgrete) make this work and also add checks for correctness below + // TODO(pgrete) make this work and also add checks for correctness below // resfile.RestoreViewAttribute(groupname + delim + prefix + delim + "bool1d", - // in_bool1d); + // in_bool1d); } REQUIRE(scalar == in_scalar);