From ee90bee86718538900b219150717312b50e0883d Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 7 Nov 2024 16:18:54 +0000 Subject: [PATCH 01/22] CI: Enable HYPRE in builds --- .github/workflows/tests.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f1fc19aeac..1cd11c936f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,6 +47,7 @@ jobs: -DBOUT_USE_PETSC=ON -DBOUT_USE_SLEPC=ON -DBOUT_USE_SUNDIALS=ON + -DBOUT_USE_HYPRE=ON -DBOUT_USE_ADIOS2=ON -DBOUT_ENABLE_PYTHON=ON -DADIOS2_ROOT=/home/runner/local/adios2 @@ -74,6 +75,7 @@ jobs: -DBOUT_USE_PETSC=ON -DBOUT_USE_SLEPC=ON -DBOUT_USE_SUNDIALS=ON + -DBOUT_USE_HYPRE=ON -DSUNDIALS_ROOT=/home/runner/local" on_cron: false @@ -86,6 +88,7 @@ jobs: -DBOUT_USE_PETSC=ON -DBOUT_USE_SLEPC=ON -DBOUT_USE_SUNDIALS=ON + -DBOUT_USE_HYPRE=ON -DSUNDIALS_ROOT=/home/runner/local" on_cron: false @@ -97,6 +100,7 @@ jobs: -DBOUT_USE_PETSC=ON -DBOUT_USE_SLEPC=ON -DBOUT_USE_SUNDIALS=ON + -DBOUT_USE_HYPRE=ON -DBOUT_BUILD_DOCS=OFF -DSUNDIALS_ROOT=/home/runner/local" omp_num_threads: 2 @@ -110,6 +114,7 @@ jobs: -DBOUT_USE_PETSC=ON -DBOUT_USE_SLEPC=ON -DBOUT_USE_SUNDIALS=ON + -DBOUT_USE_HYPRE=ON -DBOUT_ENABLE_PYTHON=ON -DSUNDIALS_ROOT=/home/runner/local" omp_num_threads: 2 @@ -123,6 +128,7 @@ jobs: -DBOUT_USE_PETSC=ON -DBOUT_USE_SLEPC=ON -DBOUT_USE_SUNDIALS=ON + -DBOUT_USE_HYPRE=ON -DBOUT_ENABLE_PYTHON=ON -DSUNDIALS_ROOT=/home/runner/local -DPETSC_DIR=/home/runner/local/petsc @@ -178,6 +184,7 @@ jobs: slepc-dev liblapack-dev libparpack2-dev + libhypre-dev - uses: actions/checkout@v4 with: From e8adc3fe8d44a2b61a480d36639218aeda1fa053 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 7 Nov 2024 16:19:07 +0000 Subject: [PATCH 02/22] Fix use of private variable in HYPRE laplace solver --- src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx index 8de1a54099..d9dd9ecf2f 100644 --- a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx +++ b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx @@ -71,15 +71,15 @@ LaplaceHypre3d::LaplaceHypre3d(Options* opt, const CELL_LOC loc, Mesh* mesh_in, // Checking flags are set to something which is not implemented // This is done binary (which is possible as each flag is a power of 2) - if (global_flags & ~implemented_flags) { + if (isGlobalFlagSet(~implemented_flags)) { throw BoutException("Attempted to set Laplacian inversion flag that is not " "implemented in LaplaceHypre3d"); } - if (inner_boundary_flags & ~implemented_boundary_flags) { + if (isInnerBoundaryFlagSet(~implemented_boundary_flags)) { throw BoutException("Attempted to set Laplacian inversion boundary flag that is not " "implemented in LaplaceHypre3d"); } - if (outer_boundary_flags & ~implemented_boundary_flags) { + if (isOuterBoundaryFlagSet(~implemented_boundary_flags)) { throw BoutException("Attempted to set Laplacian inversion boundary flag that is not " "implemented in LaplaceHypre3d"); } From 403fd1915f6f6725e8bf05589d6ff4487ddf29fa Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 7 Nov 2024 16:19:33 +0000 Subject: [PATCH 03/22] Delete (unused) duplicate private function Caused linking issues. Also put used function in an anonymous namespace to prevent future linking issues --- src/invert/laplacexy2/laplacexy2.cxx | 2 ++ src/invert/laplacexy2/laplacexy2_hypre.cxx | 5 ----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/invert/laplacexy2/laplacexy2.cxx b/src/invert/laplacexy2/laplacexy2.cxx index 8b5e98d747..0f3ef00e2d 100644 --- a/src/invert/laplacexy2/laplacexy2.cxx +++ b/src/invert/laplacexy2/laplacexy2.cxx @@ -14,10 +14,12 @@ #include +namespace { Ind2D index2d(Mesh* mesh, int x, int y) { int ny = mesh->LocalNy; return Ind2D(x * ny + y, ny, 1); } +} LaplaceXY2::LaplaceXY2(Mesh* m, Options* opt, const CELL_LOC loc) : localmesh(m == nullptr ? bout::globals::mesh : m), diff --git a/src/invert/laplacexy2/laplacexy2_hypre.cxx b/src/invert/laplacexy2/laplacexy2_hypre.cxx index b18903be3a..a97654a2da 100644 --- a/src/invert/laplacexy2/laplacexy2_hypre.cxx +++ b/src/invert/laplacexy2/laplacexy2_hypre.cxx @@ -25,11 +25,6 @@ inline void gpuAssert(cudaError_t code, const char* file, int line, bool abort = } #endif -Ind2D index2d(Mesh* mesh, int x, int y) { - int ny = mesh->LocalNy; - return Ind2D(x * ny + y, ny, 1); -} - LaplaceXY2Hypre::LaplaceXY2Hypre(Mesh* m, Options* opt, const CELL_LOC loc) : localmesh(m == nullptr ? bout::globals::mesh : m), indexConverter(std::make_shared>( From ff63ff811345f12afcd6dfcf3dc7ee711ba93f7f Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 7 Nov 2024 16:28:50 +0000 Subject: [PATCH 04/22] CMake: Error if ADIOS2 requested and not found Also default to off --- cmake/SetupBOUTThirdParty.cmake | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/cmake/SetupBOUTThirdParty.cmake b/cmake/SetupBOUTThirdParty.cmake index 9c49fe6fdc..656835f30a 100644 --- a/cmake/SetupBOUTThirdParty.cmake +++ b/cmake/SetupBOUTThirdParty.cmake @@ -190,7 +190,7 @@ endif() message(STATUS "NetCDF support: ${BOUT_USE_NETCDF}") set(BOUT_HAS_NETCDF ${BOUT_USE_NETCDF}) -option(BOUT_USE_ADIOS2 "Enable support for ADIOS output" ON) +option(BOUT_USE_ADIOS2 "Enable support for ADIOS output" OFF) option(BOUT_DOWNLOAD_ADIOS2 "Download and build ADIOS2" OFF) if (BOUT_USE_ADIOS2) if (BOUT_DOWNLOAD_ADIOS2) @@ -214,14 +214,10 @@ if (BOUT_USE_ADIOS2) target_link_libraries(bout++ PUBLIC adios2::cxx11_mpi) message(STATUS "ADIOS2 done configuring") else() - find_package(ADIOS2) - if (ADIOS2_FOUND) - ENABLE_LANGUAGE(C) - find_package(MPI REQUIRED COMPONENTS C) - target_link_libraries(bout++ PUBLIC adios2::cxx11_mpi MPI::MPI_C) - else() - set(BOUT_USE_ADIOS2 OFF) - endif() + find_package(ADIOS2 REQUIRED) + enable_language(C) + find_package(MPI REQUIRED COMPONENTS C) + target_link_libraries(bout++ PUBLIC adios2::cxx11_mpi MPI::MPI_C) endif() endif() message(STATUS "ADIOS2 support: ${BOUT_USE_ADIOS2}") From 171b7ae3745c172a22a7452e8a2c7bf53798a19e Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 7 Nov 2024 16:40:04 +0000 Subject: [PATCH 05/22] Fix missing parameter from overridden virtual function --- src/sys/options/options_adios.cxx | 2 +- src/sys/options/options_adios.hxx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sys/options/options_adios.cxx b/src/sys/options/options_adios.cxx index 529ad5b7f5..b3acbaada6 100644 --- a/src/sys/options/options_adios.cxx +++ b/src/sys/options/options_adios.cxx @@ -245,7 +245,7 @@ bool readAttribute(adios2::IO& io, const std::string& name, const std::string& t return false; } -Options OptionsADIOS::read() { +Options OptionsADIOS::read([[maybe_unused]] bool lazy) { Timer timer("io"); // Open file diff --git a/src/sys/options/options_adios.hxx b/src/sys/options/options_adios.hxx index 73170c9501..6e81508ca3 100644 --- a/src/sys/options/options_adios.hxx +++ b/src/sys/options/options_adios.hxx @@ -50,7 +50,7 @@ public: OptionsADIOS& operator=(OptionsADIOS&&) noexcept = default; /// Read options from file - Options read() override; + Options read(bool lazy = true) override; /// Write options to file void write(const Options& options, const std::string& time_dim) override; From 237207b120b6aa3b922ef7d86c639ef3caf332f8 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 7 Nov 2024 16:40:31 +0000 Subject: [PATCH 06/22] CI: Fix ADIOS2 build not finding ADIOS2 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1cd11c936f..f366a99d55 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -50,7 +50,7 @@ jobs: -DBOUT_USE_HYPRE=ON -DBOUT_USE_ADIOS2=ON -DBOUT_ENABLE_PYTHON=ON - -DADIOS2_ROOT=/home/runner/local/adios2 + -DADIOS2_ROOT=/home/runner/local -DSUNDIALS_ROOT=/home/runner/local -DPETSC_DIR=/home/runner/local/petsc -DSLEPC_DIR=/home/runner/local/slepc" From c3c7f460260d2ded0b38a56daeb6628ab7b47d5f Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 7 Nov 2024 17:17:08 +0000 Subject: [PATCH 07/22] CMake: Fix FindHYPRE for non-CMake builds Usually installed under `usr/include/hypre` (or similar) for system packages --- cmake/FindHYPRE.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/FindHYPRE.cmake b/cmake/FindHYPRE.cmake index 2412981c80..1b9a5ca6f9 100644 --- a/cmake/FindHYPRE.cmake +++ b/cmake/FindHYPRE.cmake @@ -7,6 +7,7 @@ include(FindPackageHandleStandardArgs) find_package(HYPRE CONFIG QUIET) if (HYPRE_FOUND) + message(STATUS "Found HYPRE: ${HYPRE_VERSION}") return() endif() @@ -14,7 +15,7 @@ find_path(HYPRE_INCLUDE_DIR NAMES HYPRE.h DOC "HYPRE include directories" REQUIRED - PATH_SUFFIXES include + PATH_SUFFIXES include include/hypre ) find_library(HYPRE_LIBRARY From ae8840a415953ddfaff96ec59ba91f687147a23f Mon Sep 17 00:00:00 2001 From: ZedThree Date: Thu, 7 Nov 2024 17:24:21 +0000 Subject: [PATCH 08/22] Apply clang-format changes --- include/bout/field3d.hxx | 2 +- include/bout/utils.hxx | 4 ++-- src/invert/laplacexy2/laplacexy2.cxx | 2 +- src/physics/physicsmodel.cxx | 2 +- tests/unit/include/bout/test_hypre_interface.cxx | 2 +- tests/unit/include/bout/test_petsc_setters.cxx | 2 +- tests/unit/mesh/test_coordinates.cxx | 2 +- tests/unit/mesh/test_mesh.cxx | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/include/bout/field3d.hxx b/include/bout/field3d.hxx index 644c14030a..3989b5f62a 100644 --- a/include/bout/field3d.hxx +++ b/include/bout/field3d.hxx @@ -569,7 +569,7 @@ void checkData(const Field3D& f, const std::string& region = "RGN_NOBNDRY"); /// Ignored with disabled CHECK; Throw an exception if \p f is not /// allocated or if any elements are non-finite (for CHECK > 2) inline void checkData(const Field3D& UNUSED(f), - const std::string& UNUSED(region) = "RGN_NOBNDRY") {}; + const std::string& UNUSED(region) = "RGN_NOBNDRY"){}; #endif /// Fourier filtering, removes all except one mode diff --git a/include/bout/utils.hxx b/include/bout/utils.hxx index 5845dd40e0..f4a41c1a20 100644 --- a/include/bout/utils.hxx +++ b/include/bout/utils.hxx @@ -45,8 +45,8 @@ #include #include #include -#include #include +#include #ifdef _MSC_VER // finite is not actually standard C++, it's a BSD extention for C @@ -544,7 +544,7 @@ inline void checkData(BoutReal f) { } #else /// Ignored with disabled CHECK; Throw an exception if \p f is not finite -inline void checkData(BoutReal UNUSED(f)) {}; +inline void checkData(BoutReal UNUSED(f)){}; #endif /*! diff --git a/src/invert/laplacexy2/laplacexy2.cxx b/src/invert/laplacexy2/laplacexy2.cxx index 0f3ef00e2d..713fd8e68f 100644 --- a/src/invert/laplacexy2/laplacexy2.cxx +++ b/src/invert/laplacexy2/laplacexy2.cxx @@ -19,7 +19,7 @@ Ind2D index2d(Mesh* mesh, int x, int y) { int ny = mesh->LocalNy; return Ind2D(x * ny + y, ny, 1); } -} +} // namespace LaplaceXY2::LaplaceXY2(Mesh* m, Options* opt, const CELL_LOC loc) : localmesh(m == nullptr ? bout::globals::mesh : m), diff --git a/src/physics/physicsmodel.cxx b/src/physics/physicsmodel.cxx index f23c87cf8e..f4be3bde2a 100644 --- a/src/physics/physicsmodel.cxx +++ b/src/physics/physicsmodel.cxx @@ -32,11 +32,11 @@ #include #undef BOUT_NO_USING_NAMESPACE_BOUTGLOBALS +#include "bout/version.hxx" #include #include #include #include -#include "bout/version.hxx" #include diff --git a/tests/unit/include/bout/test_hypre_interface.cxx b/tests/unit/include/bout/test_hypre_interface.cxx index b299cbce70..ec22140647 100644 --- a/tests/unit/include/bout/test_hypre_interface.cxx +++ b/tests/unit/include/bout/test_hypre_interface.cxx @@ -6,8 +6,8 @@ #include -#include "test_extras.hxx" #include "fake_mesh_fixture.hxx" +#include "test_extras.hxx" #include "gmock/gmock.h" #include "gtest/gtest.h" diff --git a/tests/unit/include/bout/test_petsc_setters.cxx b/tests/unit/include/bout/test_petsc_setters.cxx index e46056c5da..5e04cc97e8 100644 --- a/tests/unit/include/bout/test_petsc_setters.cxx +++ b/tests/unit/include/bout/test_petsc_setters.cxx @@ -5,8 +5,8 @@ #if BOUT_HAS_PETSC #include "gtest/gtest.h" -#include #include +#include ///////////////// Test PetscMatrixElement ///////////////// diff --git a/tests/unit/mesh/test_coordinates.cxx b/tests/unit/mesh/test_coordinates.cxx index 5b1172e990..c5091b6599 100644 --- a/tests/unit/mesh/test_coordinates.cxx +++ b/tests/unit/mesh/test_coordinates.cxx @@ -6,8 +6,8 @@ #include "bout/mesh.hxx" #include "bout/output.hxx" -#include "test_extras.hxx" #include "fake_mesh_fixture.hxx" +#include "test_extras.hxx" using bout::globals::mesh; diff --git a/tests/unit/mesh/test_mesh.cxx b/tests/unit/mesh/test_mesh.cxx index 43e62e0104..4d0c9111dd 100644 --- a/tests/unit/mesh/test_mesh.cxx +++ b/tests/unit/mesh/test_mesh.cxx @@ -6,8 +6,8 @@ #include "bout/output.hxx" #include "bout/region.hxx" -#include "test_extras.hxx" #include "fake_mesh.hxx" +#include "test_extras.hxx" /// Test fixture to make sure the global mesh is our fake one class MeshTest : public ::testing::Test { From 6398368d9da09b7cc974496b7def30936e4e8ab8 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 8 Nov 2024 09:32:42 +0000 Subject: [PATCH 09/22] Fix hypre3d laplace solver for 3D metrics --- src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx index d9dd9ecf2f..a1a781de79 100644 --- a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx +++ b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx @@ -277,7 +277,7 @@ void LaplaceHypre3d::updateMatrix3D() { const Field3D dc_dx = issetC ? DDX(C2) : Field3D(); const Field3D dc_dy = issetC ? DDY(C2) : Field3D(); const Field3D dc_dz = issetC ? DDZ(C2) : Field3D(); - const Field2D dJ_dy = DDY(coords->J / coords->g_22); + const auto dJ_dy = DDY(coords->J / coords->g_22); // Set up the matrix for the internal points on the grid. // Boundary conditions were set in the constructor. From d23cff84d1e1dccc67831a3a0d6bacf5c9661dbd Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 8 Nov 2024 09:35:39 +0000 Subject: [PATCH 10/22] Fix a bunch of clang-tidy issues --- .../laplace/impls/hypre3d/hypre3d_laplace.cxx | 69 ++++++++++--------- .../laplace/impls/hypre3d/hypre3d_laplace.hxx | 8 +-- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx index a1a781de79..2636b7a612 100644 --- a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx +++ b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx @@ -83,11 +83,11 @@ LaplaceHypre3d::LaplaceHypre3d(Options* opt, const CELL_LOC loc, Mesh* mesh_in, throw BoutException("Attempted to set Laplacian inversion boundary flag that is not " "implemented in LaplaceHypre3d"); } - if (lower_boundary_flags & ~implemented_boundary_flags) { + if ((lower_boundary_flags & ~implemented_boundary_flags) != 0) { throw BoutException("Attempted to set Laplacian inversion boundary flag that is not " "implemented in LaplaceHypre3d"); } - if (upper_boundary_flags & ~implemented_boundary_flags) { + if ((upper_boundary_flags & ~implemented_boundary_flags) != 0) { throw BoutException("Attempted to set Laplacian inversion boundary flag that is not " "implemented in LaplaceHypre3d"); } @@ -123,7 +123,7 @@ LaplaceHypre3d::LaplaceHypre3d(Options* opt, const CELL_LOC loc, Mesh* mesh_in, } BOUT_FOR_SERIAL(i, indexer->getRegionLowerY()) { - if (lower_boundary_flags & INVERT_AC_GRAD) { + if ((lower_boundary_flags & INVERT_AC_GRAD) != 0) { // Neumann on lower Y boundary operator3D(i, i) = -1. / coords->dy[i] / sqrt(coords->g_22[i]); operator3D(i, i.yp()) = 1. / coords->dy[i] / sqrt(coords->g_22[i]); @@ -135,7 +135,7 @@ LaplaceHypre3d::LaplaceHypre3d(Options* opt, const CELL_LOC loc, Mesh* mesh_in, } BOUT_FOR_SERIAL(i, indexer->getRegionUpperY()) { - if (upper_boundary_flags & INVERT_AC_GRAD) { + if ((upper_boundary_flags & INVERT_AC_GRAD) != 0) { // Neumann on upper Y boundary operator3D(i, i) = 1. / coords->dy[i] / sqrt(coords->g_22[i]); operator3D(i, i.ym()) = -1. / coords->dy[i] / sqrt(coords->g_22[i]); @@ -200,9 +200,9 @@ Field3D LaplaceHypre3d::solve(const Field3D& b_in, const Field3D& x0) { } BOUT_FOR_SERIAL(i, indexer->getRegionLowerY()) { - const BoutReal val = (lower_boundary_flags & INVERT_SET) ? x0[i] : 0.; + const BoutReal val = ((lower_boundary_flags & INVERT_SET) != 0) ? x0[i] : 0.; ASSERT1(std::isfinite(val)); - if (!(lower_boundary_flags & INVERT_RHS)) { + if ((lower_boundary_flags & INVERT_RHS) == 0) { b[i] = val; } else { ASSERT1(std::isfinite(b[i])); @@ -210,9 +210,9 @@ Field3D LaplaceHypre3d::solve(const Field3D& b_in, const Field3D& x0) { } BOUT_FOR_SERIAL(i, indexer->getRegionUpperY()) { - const BoutReal val = (upper_boundary_flags & INVERT_SET) ? x0[i] : 0.; + const BoutReal val = ((upper_boundary_flags & INVERT_SET) != 0) ? x0[i] : 0.; ASSERT1(std::isfinite(val)); - if (!(upper_boundary_flags & INVERT_RHS)) { + if ((upper_boundary_flags & INVERT_RHS) == 0) { b[i] = val; } else { ASSERT1(std::isfinite(b[i])); @@ -286,7 +286,8 @@ void LaplaceHypre3d::updateMatrix3D() { // avoid confusing it with the x-index. // Calculate coefficients for the terms in the differential operator - BoutReal C_df_dx = coords->G1[l], C_df_dz = coords->G3[l]; + BoutReal C_df_dx = coords->G1[l]; + BoutReal C_df_dz = coords->G3[l]; if (issetD) { C_df_dx *= D[l]; C_df_dz *= D[l]; @@ -304,9 +305,9 @@ void LaplaceHypre3d::updateMatrix3D() { C_df_dz += Ez[l]; } - BoutReal C_d2f_dx2 = coords->g11[l], - C_d2f_dy2 = (coords->g22[l] - 1.0 / coords->g_22[l]), - C_d2f_dz2 = coords->g33[l]; + BoutReal C_d2f_dx2 = coords->g11[l]; + BoutReal C_d2f_dy2 = (coords->g22[l] - 1.0 / coords->g_22[l]); + BoutReal C_d2f_dz2 = coords->g33[l]; if (issetD) { C_d2f_dx2 *= D[l]; C_d2f_dy2 *= D[l]; @@ -343,8 +344,8 @@ void LaplaceHypre3d::updateMatrix3D() { // The values stored in the y-boundary are already interpolated // up/down, so we don't want the matrix to do any such // interpolation there. - const int yup = (l.y() == localmesh->yend && upperY.intersects(l.x())) ? -1 : 0, - ydown = (l.y() == localmesh->ystart && lowerY.intersects(l.x())) ? -1 : 0; + const int yup = (l.y() == localmesh->yend && upperY.intersects(l.x())) ? -1 : 0; + const int ydown = (l.y() == localmesh->ystart && lowerY.intersects(l.x())) ? -1 : 0; operator3D.yup(yup)(l, l.yp()) = 0.0; operator3D.ydown(ydown)(l, l.ym()) = 0.0; operator3D.yup(yup)(l, l.xp().yp()) = 0.0; @@ -376,7 +377,8 @@ void LaplaceHypre3d::updateMatrix3D() { C_d2f_dy2 *= D[l]; } - BoutReal C_d2f_dxdy = 2 * coords->g12[l], C_d2f_dydz = 2 * coords->g23[l]; + BoutReal C_d2f_dxdy = 2 * coords->g12[l]; + BoutReal C_d2f_dydz = 2 * coords->g23[l]; if (issetD) { C_d2f_dxdy *= D[l]; C_d2f_dydz *= D[l]; @@ -396,8 +398,8 @@ void LaplaceHypre3d::updateMatrix3D() { // The values stored in the y-boundary are already interpolated // up/down, so we don't want the matrix to do any such // interpolation there. - const int yup = (l.y() == localmesh->yend && upperY.intersects(l.x())) ? -1 : 0, - ydown = (l.y() == localmesh->ystart && lowerY.intersects(l.x())) ? -1 : 0; + const int yup = (l.y() == localmesh->yend && upperY.intersects(l.x())) ? -1 : 0; + const int ydown = (l.y() == localmesh->ystart && lowerY.intersects(l.x())) ? -1 : 0; operator3D.yup(yup)(l, l.yp()) += C_df_dy + C_d2f_dy2; operator3D.ydown(ydown)(l, l.ym()) += -C_df_dy + C_d2f_dy2; @@ -437,19 +439,15 @@ OperatorStencil LaplaceHypre3d::getStencil(Mesh* localmesh, OffsetInd3D zero; // Add interior cells - const std::vector interpolatedUpElements = {zero.yp(), zero.xp().yp(), - zero.xm().yp(), zero.yp().zp(), - zero.yp().zm()}, - interpolatedDownElements = { - zero.ym(), zero.xp().ym(), zero.xm().ym(), - zero.ym().zp(), zero.ym().zm()}; - std::set interiorStencil = {zero, zero.xp(), - zero.xm(), zero.zp(), - zero.zm(), zero.xp().zp(), - zero.xp().zm(), zero.xm().zp(), - zero.xm().zm()}, - lowerEdgeStencil = interiorStencil, - upperEdgeStencil = interiorStencil; + const std::vector interpolatedUpElements = { + zero.yp(), zero.xp().yp(), zero.xm().yp(), zero.yp().zp(), zero.yp().zm()}; + const std::vector interpolatedDownElements = { + zero.ym(), zero.xp().ym(), zero.xm().ym(), zero.ym().zp(), zero.ym().zm()}; + std::set interiorStencil = { + zero, zero.xp(), zero.xm(), zero.zp(), zero.zm(), + zero.xp().zp(), zero.xp().zm(), zero.xm().zp(), zero.xm().zm()}; + std::set lowerEdgeStencil = interiorStencil; + std::set upperEdgeStencil = interiorStencil; for (const auto& i : interpolatedDownElements) { for (auto& j : interpPattern) { @@ -466,9 +464,11 @@ OperatorStencil LaplaceHypre3d::getStencil(Mesh* localmesh, upperEdgeStencil.insert(i); } const std::vector interiorStencilVector(interiorStencil.begin(), - interiorStencil.end()), - lowerEdgeStencilVector(lowerEdgeStencil.begin(), lowerEdgeStencil.end()), - upperEdgeStencilVector(upperEdgeStencil.begin(), upperEdgeStencil.end()); + interiorStencil.end()); + const std::vector lowerEdgeStencilVector(lowerEdgeStencil.begin(), + lowerEdgeStencil.end()); + const std::vector upperEdgeStencilVector(upperEdgeStencil.begin(), + upperEdgeStencil.end()); // If there is a lower y-boundary then create a part of the stencil // for cells immediately adjacent to it. @@ -536,7 +536,8 @@ OperatorStencil LaplaceHypre3d::getStencil(Mesh* localmesh, return stencil; } -int LaplaceHypre3d::Hypre3dMonitor::call(Solver*, BoutReal, int, int) { +int LaplaceHypre3d::Hypre3dMonitor::call(Solver* /*solver*/, BoutReal /*time*/, + int /*iter*/, int /*nout*/) { if (laplace.n_solves == 0) { laplace.average_iterations = 0.0; return 0; diff --git a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx index f3ca44df46..2a5761440f 100644 --- a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx +++ b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx @@ -209,7 +209,7 @@ public: public: Hypre3dMonitor(LaplaceHypre3d& laplace_in) : laplace(laplace_in) {} - int call(Solver*, BoutReal, int, int) override; + int call(Solver* /*solver*/, BoutReal /*time*/, int /*iter*/, int /*nout*/) override; private: LaplaceHypre3d& laplace; @@ -220,9 +220,9 @@ public: bool rightprec; // Right preconditioning // These are the implemented flags - static constexpr int implemented_flags = INVERT_START_NEW, - implemented_boundary_flags = - INVERT_AC_GRAD + INVERT_SET + INVERT_RHS; + static constexpr int implemented_flags = INVERT_START_NEW; + static constexpr int implemented_boundary_flags = + INVERT_AC_GRAD + INVERT_SET + INVERT_RHS; }; #endif // BOUT_HAS_HYPRE From 9c1cfc98eca401edcfaa97f1a8c25a752d357c2c Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 8 Nov 2024 09:36:02 +0000 Subject: [PATCH 11/22] Remove a couple of unused private members --- src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx index 2a5761440f..2d45fc742e 100644 --- a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx +++ b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx @@ -216,9 +216,6 @@ public: }; Hypre3dMonitor monitor; - bool use_precon; // Switch for preconditioning - bool rightprec; // Right preconditioning - // These are the implemented flags static constexpr int implemented_flags = INVERT_START_NEW; static constexpr int implemented_boundary_flags = From 31567f9ce925c0c99eb04ee2830f11aa4ccafe44 Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Thu, 9 Jan 2025 10:44:28 -0800 Subject: [PATCH 12/22] Hypre3D Laplacian: Save diagnostics using outputVars Saved to dmp files by the Laplacian savePerformance monitor. If the savePerformance() method is called to enable outputs, the mean solver and AMG iterations, and the final residual, are saved as time-varying scalars. --- include/bout/hypre_interface.hxx | 14 ++++- .../laplace/impls/hypre3d/hypre3d_laplace.cxx | 63 +++++++++++-------- .../laplace/impls/hypre3d/hypre3d_laplace.hxx | 36 ++++++----- 3 files changed, 71 insertions(+), 42 deletions(-) diff --git a/include/bout/hypre_interface.hxx b/include/bout/hypre_interface.hxx index 86791fbd3d..e8a0d462b0 100644 --- a/include/bout/hypre_interface.hxx +++ b/include/bout/hypre_interface.hxx @@ -894,20 +894,30 @@ public: void setMaxIter(int max_iter) { checkHypreError(solverSetMaxIter(solver, max_iter)); } - double getFinalRelResNorm() { + double getFinalRelResNorm() const { HYPRE_Real resnorm{}; checkHypreError(solverGetFinalRelativeResidualNorm(solver, &resnorm)); return resnorm; } - int getNumItersTaken() { + /// Return the number of solver iterations taken + int getNumItersTaken() const { HYPRE_Int iters{}; checkHypreError(solverGetNumIterations(solver, &iters)); return iters; } + /// Return the number of BoomerAMG preconditioner iterations taken + int getNumItersTakenAMG() const { + HYPRE_Int iters{}; + checkHypreError(HYPRE_BoomerAMGGetNumIterations(precon, &iters)); + return iters; + } + + /// Set the HypreMatrix to be used in the solver void setMatrix(HypreMatrix* A_) { A = A_; } + /// Enable BoomerAMG preconditioner with the given HypreMatrix int setupAMG(HypreMatrix* P_) { CALI_CXX_MARK_FUNCTION; diff --git a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx index 8de1a54099..b166e4979c 100644 --- a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx +++ b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx @@ -3,9 +3,9 @@ * Using Hypre Solvers * ************************************************************************** - * Copyright 2021 J.Omotani, C.MacMackin + * Copyright 2021 - 2025 BOUT++ contributors * - * Contact: Ben Dudson, bd512@york.ac.uk + * Contact: Ben Dudson, dudson2@llnl.gov * * This file is part of BOUT++. * @@ -43,14 +43,14 @@ #include LaplaceHypre3d::LaplaceHypre3d(Options* opt, const CELL_LOC loc, Mesh* mesh_in, - Solver* solver) + Solver*) : Laplacian(opt, loc, mesh_in), A(0.0), C1(1.0), C2(1.0), D(1.0), Ex(0.0), Ez(0.0), opts(opt == nullptr ? Options::getRoot()->getSection("laplace") : opt), lowerY(localmesh->iterateBndryLowerY()), upperY(localmesh->iterateBndryUpperY()), indexer(std::make_shared>( localmesh, getStencil(localmesh, lowerY, upperY))), operator3D(indexer), solution(indexer), rhs(indexer), - linearSystem(*localmesh, *opts), monitor(*this) { + linearSystem(*localmesh, *opts) { // Provide basic initialisation of field coefficients, etc. // Get relevent options from user input A.setLocation(location); @@ -61,6 +61,7 @@ LaplaceHypre3d::LaplaceHypre3d(Options* opt, const CELL_LOC loc, Mesh* mesh_in, Ez.setLocation(location); // Initialise Hypre objects + // Note: linearSystem is a bout::HypreSystem object linearSystem.setMatrix(&operator3D); linearSystem.setRHSVector(&rhs); linearSystem.setSolutionVector(&solution); @@ -71,15 +72,15 @@ LaplaceHypre3d::LaplaceHypre3d(Options* opt, const CELL_LOC loc, Mesh* mesh_in, // Checking flags are set to something which is not implemented // This is done binary (which is possible as each flag is a power of 2) - if (global_flags & ~implemented_flags) { + if (getGlobalFlags() & ~implemented_flags) { throw BoutException("Attempted to set Laplacian inversion flag that is not " "implemented in LaplaceHypre3d"); } - if (inner_boundary_flags & ~implemented_boundary_flags) { + if (getInnerBoundaryFlags() & ~implemented_boundary_flags) { throw BoutException("Attempted to set Laplacian inversion boundary flag that is not " "implemented in LaplaceHypre3d"); } - if (outer_boundary_flags & ~implemented_boundary_flags) { + if (getOuterBoundaryFlags() & ~implemented_boundary_flags) { throw BoutException("Attempted to set Laplacian inversion boundary flag that is not " "implemented in LaplaceHypre3d"); } @@ -145,15 +146,6 @@ LaplaceHypre3d::LaplaceHypre3d(Options* opt, const CELL_LOC loc, Mesh* mesh_in, operator3D(i, i.ym()) = 0.5; } } - - // FIXME: This needs to be converted to outputVars - if (solver == nullptr) { - output_warn << "Warning: Need to pass a Solver to " - "Laplacian::create() to get iteration counts in the output." - << endl; - } else { - solver->addMonitor(&monitor); - } } LaplaceHypre3d::~LaplaceHypre3d() {} @@ -239,6 +231,7 @@ Field3D LaplaceHypre3d::solve(const Field3D& b_in, const Field3D& x0) { // Increment counters n_solves++; cumulative_iterations += linearSystem.getNumItersTaken(); + cumulative_amg_iterations += linearSystem.getNumItersTakenAMG(); CALI_MARK_END("LaplaceHypre3d_solve:solve"); @@ -246,6 +239,8 @@ Field3D LaplaceHypre3d::solve(const Field3D& b_in, const Field3D& x0) { // Create field from solution Field3D result = solution.toField(); + + // Fill guard and boundary cells localmesh->communicate(result); if (result.hasParallelSlices()) { BOUT_FOR(i, indexer->getRegionLowerY()) { result.ydown()[i] = result[i]; } @@ -536,19 +531,35 @@ OperatorStencil LaplaceHypre3d::getStencil(Mesh* localmesh, return stencil; } -int LaplaceHypre3d::Hypre3dMonitor::call(Solver*, BoutReal, int, int) { - if (laplace.n_solves == 0) { - laplace.average_iterations = 0.0; - return 0; - } +void LaplaceHypre3d::outputVars(Options& output_options, + const std::string& time_dimension) const { + BoutReal mean_iterations = 0.0; + BoutReal mean_amg_iterations = 0.0; + BoutReal rel_res_norm = linearSystem.getFinalRelResNorm(); - // Calculate average and reset counters - laplace.average_iterations = static_cast(laplace.cumulative_iterations) - / static_cast(laplace.n_solves); + if (n_solves > 0) { + // Calculate average + mean_iterations = static_cast(cumulative_iterations) + / static_cast(n_solves); - output_info.write("\nHypre3d average iterations: {}\n", laplace.average_iterations); + mean_amg_iterations = static_cast(cumulative_amg_iterations) + / static_cast(n_solves); + } - return 0; + std::string name = getPerformanceName(); + + output_options[fmt::format("{}_mean_its", name)].assignRepeat(mean_iterations, time_dimension).setAttributes({ + {"source", "hypre3d_laplace"}, + {"description", "Mean number of solver iterations"} + }); + output_options[fmt::format("{}_mean_amg_its", name)].assignRepeat(mean_amg_iterations, time_dimension).setAttributes({ + {"source", "hypre3d_laplace"}, + {"description", "Mean number of BoomerAMG iterations"} + }); + output_options[fmt::format("{}_rel_res_norm", name)].assignRepeat(rel_res_norm, time_dimension).setAttributes({ + {"source", "hypre3d_laplace"}, + {"description", "Final relative residual norm"} + }); } #endif // BOUT_HAS_HYPRE diff --git a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx index f3ca44df46..612f410a9a 100644 --- a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx +++ b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx @@ -6,9 +6,9 @@ *ex\nabla_x x + ez\nabla_z x + a x = b\f$ * ************************************************************************** - * Copyright 2021 J. Omotani, C. MacMackin + * Copyright 2021 - 2025 BOUT++ contributors * - * Contact: Ben Dudson, bd512@york.ac.uk + * Contact: Ben Dudson, dudson2@llnl.gov * * This file is part of BOUT++. * @@ -201,20 +201,28 @@ public: bout::HypreVector rhs; bout::HypreSystem linearSystem; - // used to save some statistics + // used to save Hypre solver statistics int n_solves = 0; int cumulative_iterations = 0; - BoutReal average_iterations = 0.0; - class Hypre3dMonitor : public Monitor { - public: - Hypre3dMonitor(LaplaceHypre3d& laplace_in) : laplace(laplace_in) {} - - int call(Solver*, BoutReal, int, int) override; - - private: - LaplaceHypre3d& laplace; - }; - Hypre3dMonitor monitor; + int cumulative_amg_iterations = 0; + + /// Save performance metrics + /// This is called by LaplacianMonitor, that is enabled + /// if Laplacian::savePerformance(Solver&, string&) is called. + /// + /// # Example + /// + /// // Create a Laplacian solver using "phiSolver" options + /// phiSolver = Laplacian::create(&Options::root()["phiSolver"]); + /// // Enable output diagnostics with "phiSolver" prefix + /// phiSolver->savePerformance(solver, "phiSolver"); + /// + /// Saves the following variables to the output: + /// - phiSolver_mean_its Mean number of solver iterations taken + /// - phiSolver_mean_amg_its Mean number of BoomerAMG preconditioner iterations + /// - phiSolver_rel_res_norm The final solver relative residual norm + void outputVars(Options& output_options, + const std::string& time_dimension) const override; bool use_precon; // Switch for preconditioning bool rightprec; // Right preconditioning From 9e765b6a6cca17bcdb94f3fcbf07c12307828171 Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Thu, 9 Jan 2025 10:46:53 -0800 Subject: [PATCH 13/22] ParallelTransform: Add missing space in error message --- include/bout/paralleltransform.hxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/bout/paralleltransform.hxx b/include/bout/paralleltransform.hxx index 231fc5220c..49dea67743 100644 --- a/include/bout/paralleltransform.hxx +++ b/include/bout/paralleltransform.hxx @@ -216,7 +216,7 @@ public: std::vector getWeightsForYApproximation(int UNUSED(i), int UNUSED(j), int UNUSED(k), int UNUSED(yoffset)) override { - throw BoutException("ParallelTransform::getWeightsForYApproximation not implemented" + throw BoutException("ParallelTransform::getWeightsForYApproximation not implemented " "for `type = shifted`. Try `type = shiftedinterp`"); } From 90332a8b8e35b25de1313cef8b7cd98379fb4522 Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Thu, 9 Jan 2025 10:47:30 -0800 Subject: [PATCH 14/22] BoundaryFactory: Fix default for parallel boundary regions Default was "parallel_dirichlet", but no boundary is registered with that name. Changed to "parallel_dirichlet_o2". --- src/mesh/boundary_factory.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mesh/boundary_factory.cxx b/src/mesh/boundary_factory.cxx index 00282566a9..d112a216ad 100644 --- a/src/mesh/boundary_factory.cxx +++ b/src/mesh/boundary_factory.cxx @@ -318,7 +318,7 @@ BoundaryOpBase* BoundaryFactory::createFromOptions(const string& varname, /// Then (all, all) if (region->isParallel) { // Different default for parallel boundary regions - varOpts->get(prefix + "par_all", set, "parallel_dirichlet"); + varOpts->get(prefix + "par_all", set, "parallel_dirichlet_o2"); } else { varOpts->get(prefix + "all", set, "dirichlet"); } From dc3b020e7bb9e491956cfdde416b815dd9be074f Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Thu, 9 Jan 2025 10:48:54 -0800 Subject: [PATCH 15/22] elm-pb example: Fix hypre input, save performance metrics Calling `phiSolver->savePerformance(...)` causes performance (e.g. iteration count) diagnostics to be saved to the output file. hypre input did have the wrong parallel transform, boundary conditions that have not been implemented. Now uses hypre3d as the phiSolver. --- examples/elm-pb/data-hypre/BOUT.inp | 59 ++++++++--------------------- examples/elm-pb/data/BOUT.inp | 10 ++--- examples/elm-pb/elm_pb.cxx | 3 ++ 3 files changed, 23 insertions(+), 49 deletions(-) diff --git a/examples/elm-pb/data-hypre/BOUT.inp b/examples/elm-pb/data-hypre/BOUT.inp index b6c816d38a..6b4673a34d 100644 --- a/examples/elm-pb/data-hypre/BOUT.inp +++ b/examples/elm-pb/data-hypre/BOUT.inp @@ -6,29 +6,23 @@ nout = 40 # number of time-steps timestep = 1 # time between outputs -wall_limit = 1.55 # wall time limit (in hours) zperiod = 15 # Fraction of a torus to simulate MZ = 16 # Number of points in Z grid = "cbm18_dens8.grid_nx68ny64.nc" # Grid file -dump_format = "nc" # Dump file format. "nc" = NetCDF, "pdb" = PDB -restart_format = "nc" # Restart file format - [mesh] staggergrids = false # Use staggered grids [mesh:paralleltransform] - -type = shifted # Use shifted metric method +type = shiftedinterp ################################################## # derivative methods [mesh:ddx] - first = C4 # order of first x derivatives second = C4 # order of second x derivatives upwind = W3 # order of upwinding method W3 = Weno3 @@ -44,20 +38,6 @@ first = C4 # Z derivatives can be done using FFT second = C4 upwind = W3 -[output] -shiftoutput = true # Put the output into field-aligned coordinates - -################################################## -# Laplacian inversion routines - -[laplace] -type=hypre3d - -flags = 0 # Flags for Laplacian inversion - -rtol = 1.e-9 -atol = 1.e-14 - ################################################## # FFTs @@ -71,15 +51,12 @@ fft_measurement_flag = measure # If using FFTW, perform tests to determine fast [solver] # mudq, mldq, mukeep, mlkeep preconditioner options -atol = 1e-08 # absolute tolerance -rtol = 1e-05 # relative tolerance +atol = 1.0e-8 # absolute tolerance +rtol = 1.0e-5 # relative tolerance use_precon = false # Use preconditioner: User-supplied or BBD -use_jacobian = false # Use user-supplied Jacobian mxstep = 5000 # Number of internal steps between outputs -adams_moulton = false # Use Adams-Moulton method (default is BDF) -func_iter = false # Functional iteration (default is Newton) ################################################## # settings for high-beta reduced MHD @@ -116,7 +93,7 @@ diamag_phi0 = true # Balance ExB against Vd for stationary equilibrium bm_exb_flag = 0 bm_mag_flag = 0 ################################################################## -withflow = false # With flow or not +withflow = false # With flow or not D_0 = 130000 # differential potential D_s = 20 # shear parameter K_H_term = false # Contain K-H term @@ -125,8 +102,8 @@ x0 = 0.855 # peak location D_min = 3000 # constant ################################################################## -eHall = false # Include electron pressue effects in Ohm's law? -AA = 2.0 # ion mass in units of proton mass +eHall = false # Include electron pressure effects in Ohm's law? +AA = 2.0 # ion mass in units of proton mass noshear = false # zero all shear @@ -174,14 +151,14 @@ damp_t_const = 1e-2 # Damping time constant ## Parallel pressure diffusion -diffusion_par = -1.0 # Parallel pressure diffusion (< 0 = none) +diffusion_par = -1.0 # Parallel pressure diffusion (< 0 = none) diffusion_p4 = -1e-05 # parallel hyper-viscous diffusion for pressure (< 0 = none) -diffusion_u4 = 1e-05 # parallel hyper-viscous diffusion for vorticity (< 0 = none) +diffusion_u4 = -1e-05 # parallel hyper-viscous diffusion for vorticity (< 0 = none) diffusion_a4 = -1e-05 # parallel hyper-viscous diffusion for vector potential (< 0 = none) ## heat source in pressure in watts -heating_P = -1 # heat power in watts (< 0 = none) +heating_P = -1 # heat power in watts (< 0 = none) hp_width = 0.1 # heat width, in percentage of nx (< 0 = none) hp_length = 0.3 # heat length in percentage of nx (< 0 = none) @@ -205,7 +182,7 @@ su_lengthr = 0.1 # right edge sink length in percentage of nx (< 0 = none) ## Viscosity and Hyper-viscosity viscos_par = -0.1 # Parallel viscosity (< 0 = none) -viscos_perp = -1.0 # Perpendicular +viscos_perp = -1.0 # Perpendicular viscosity (< 0 = none) hyperviscos = -1.0 # Radial hyper viscosity ## Compressional terms (only when compress = true) @@ -213,14 +190,11 @@ phi_curv = true # Include curvature*Grad(phi) in P equation # gamma = 1.6666 [phiSolver] -#inner_boundary_flags = 1 + 2 + 128 # INVERT_DC_GRAD + INVERT_AC_GRAD + INVERT_BNDRY_ONE -#outer_boundary_flags = 1 + 2 + 128 # INVERT_DC_GRAD + INVERT_AC_GRAD + INVERT_BNDRY_ONE -inner_boundary_flags = 1 + 4 # INVERT_DC_GRAD + INVERT_AC_LAP -outer_boundary_flags = 1 + 4 # INVERT_DC_GRAD + INVERT_AC_LAP +type = hypre3d +inner_boundary_flags = 0 # Dirichlet +outer_boundary_flags = 0 # Dirichlet [aparSolver] -#inner_boundary_flags = 1 + 2 + 128 # INVERT_DC_GRAD + INVERT_AC_GRAD + INVERT_BNDRY_ONE -#outer_boundary_flags = 1 + 2 + 128 # INVERT_DC_GRAD + INVERT_AC_GRAD + INVERT_BNDRY_ONE inner_boundary_flags = 1 + 4 # INVERT_DC_GRAD + INVERT_AC_LAP outer_boundary_flags = 1 + 4 # INVERT_DC_GRAD + INVERT_AC_LAP @@ -271,12 +245,9 @@ bndry_ydown = free_o3 # Zero gradient in the core bndry_core = neumann -[Vpar] - -bndry_core = neumann - [phi] bndry_xin = none bndry_xout = none -bndry_target = neumann +#bndry_target = neumann + diff --git a/examples/elm-pb/data/BOUT.inp b/examples/elm-pb/data/BOUT.inp index 55a4a30c06..a8b5b11e9f 100644 --- a/examples/elm-pb/data/BOUT.inp +++ b/examples/elm-pb/data/BOUT.inp @@ -94,7 +94,7 @@ diamag_phi0 = true # Balance ExB against Vd for stationary equilibrium bm_exb_flag = 0 bm_mag_flag = 0 ################################################################## -withflow = false # With flow or not +withflow = false # With flow or not D_0 = 130000 # differential potential D_s = 20 # shear parameter K_H_term = false # Contain K-H term @@ -104,7 +104,7 @@ D_min = 3000 # constant ################################################################## eHall = false # Include electron pressure effects in Ohm's law? -AA = 2.0 # ion mass in units of proton mass +AA = 2.0 # ion mass in units of proton mass noshear = false # zero all shear @@ -152,14 +152,14 @@ damp_t_const = 1e-2 # Damping time constant ## Parallel pressure diffusion -diffusion_par = -1.0 # Parallel pressure diffusion (< 0 = none) +diffusion_par = -1.0 # Parallel pressure diffusion (< 0 = none) diffusion_p4 = -1e-05 # parallel hyper-viscous diffusion for pressure (< 0 = none) -diffusion_u4 = -1e-05 # parallel hyper-viscous diffusion for vorticity (< 0 = none) +diffusion_u4 = -1e-05 # parallel hyper-viscous diffusion for vorticity (< 0 = none) diffusion_a4 = -1e-05 # parallel hyper-viscous diffusion for vector potential (< 0 = none) ## heat source in pressure in watts -heating_P = -1 # heat power in watts (< 0 = none) +heating_P = -1 # heat power in watts (< 0 = none) hp_width = 0.1 # heat width, in percentage of nx (< 0 = none) hp_length = 0.3 # heat length in percentage of nx (< 0 = none) diff --git a/examples/elm-pb/elm_pb.cxx b/examples/elm-pb/elm_pb.cxx index f108e58e2f..f830f3d98a 100644 --- a/examples/elm-pb/elm_pb.cxx +++ b/examples/elm-pb/elm_pb.cxx @@ -1142,6 +1142,9 @@ class ELMpb : public PhysicsModel { // Create a solver for the Laplacian phiSolver = Laplacian::create(&globalOptions["phiSolver"]); + // Save performance metrics to output, using the + // given name as the prefix. + phiSolver->savePerformance(*solver, "phiSolver"); aparSolver = Laplacian::create(&globalOptions["aparSolver"], loc); From f46d0a6006a051ecc22fafe6896e0bf4ee9dfb1f Mon Sep 17 00:00:00 2001 From: bendudson Date: Thu, 9 Jan 2025 18:56:13 +0000 Subject: [PATCH 16/22] Apply clang-format changes --- .../laplace/impls/hypre3d/hypre3d_laplace.cxx | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx index b166e4979c..fd705a9c83 100644 --- a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx +++ b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx @@ -42,8 +42,7 @@ #include -LaplaceHypre3d::LaplaceHypre3d(Options* opt, const CELL_LOC loc, Mesh* mesh_in, - Solver*) +LaplaceHypre3d::LaplaceHypre3d(Options* opt, const CELL_LOC loc, Mesh* mesh_in, Solver*) : Laplacian(opt, loc, mesh_in), A(0.0), C1(1.0), C2(1.0), D(1.0), Ex(0.0), Ez(0.0), opts(opt == nullptr ? Options::getRoot()->getSection("laplace") : opt), lowerY(localmesh->iterateBndryLowerY()), upperY(localmesh->iterateBndryUpperY()), @@ -539,27 +538,27 @@ void LaplaceHypre3d::outputVars(Options& output_options, if (n_solves > 0) { // Calculate average - mean_iterations = static_cast(cumulative_iterations) - / static_cast(n_solves); + mean_iterations = + static_cast(cumulative_iterations) / static_cast(n_solves); mean_amg_iterations = static_cast(cumulative_amg_iterations) - / static_cast(n_solves); + / static_cast(n_solves); } std::string name = getPerformanceName(); - output_options[fmt::format("{}_mean_its", name)].assignRepeat(mean_iterations, time_dimension).setAttributes({ - {"source", "hypre3d_laplace"}, - {"description", "Mean number of solver iterations"} - }); - output_options[fmt::format("{}_mean_amg_its", name)].assignRepeat(mean_amg_iterations, time_dimension).setAttributes({ - {"source", "hypre3d_laplace"}, - {"description", "Mean number of BoomerAMG iterations"} - }); - output_options[fmt::format("{}_rel_res_norm", name)].assignRepeat(rel_res_norm, time_dimension).setAttributes({ - {"source", "hypre3d_laplace"}, - {"description", "Final relative residual norm"} - }); + output_options[fmt::format("{}_mean_its", name)] + .assignRepeat(mean_iterations, time_dimension) + .setAttributes({{"source", "hypre3d_laplace"}, + {"description", "Mean number of solver iterations"}}); + output_options[fmt::format("{}_mean_amg_its", name)] + .assignRepeat(mean_amg_iterations, time_dimension) + .setAttributes({{"source", "hypre3d_laplace"}, + {"description", "Mean number of BoomerAMG iterations"}}); + output_options[fmt::format("{}_rel_res_norm", name)] + .assignRepeat(rel_res_norm, time_dimension) + .setAttributes({{"source", "hypre3d_laplace"}, + {"description", "Final relative residual norm"}}); } #endif // BOUT_HAS_HYPRE From 07603f0127ae3f424f3a39168fe87a8cf6df6caa Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Fri, 17 Jan 2025 14:16:09 -0800 Subject: [PATCH 17/22] Disable hypre with 3D metrics The combination doesn't yet work, so disable for now. --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a2f56c15d6..afee90531b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,7 +47,7 @@ jobs: -DBOUT_USE_PETSC=ON -DBOUT_USE_SLEPC=ON -DBOUT_USE_SUNDIALS=ON - -DBOUT_USE_HYPRE=ON + -DBOUT_USE_HYPRE=OFF -DBOUT_USE_ADIOS2=ON -DBOUT_ENABLE_PYTHON=ON -DADIOS2_ROOT=/home/runner/local @@ -114,7 +114,7 @@ jobs: -DBOUT_USE_PETSC=ON -DBOUT_USE_SLEPC=ON -DBOUT_USE_SUNDIALS=ON - -DBOUT_USE_HYPRE=ON + -DBOUT_USE_HYPRE=OFF -DBOUT_ENABLE_PYTHON=ON -DSUNDIALS_ROOT=/home/runner/local" omp_num_threads: 2 From 0012914f7da201b0443b16d751540f506f695e0c Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Fri, 17 Jan 2025 14:23:05 -0800 Subject: [PATCH 18/22] Testing: Missed 3D + hypre combination --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index afee90531b..bee6941c42 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -128,7 +128,7 @@ jobs: -DBOUT_USE_PETSC=ON -DBOUT_USE_SLEPC=ON -DBOUT_USE_SUNDIALS=ON - -DBOUT_USE_HYPRE=ON + -DBOUT_USE_HYPRE=OFF -DBOUT_ENABLE_PYTHON=ON -DSUNDIALS_ROOT=/home/runner/local -DPETSC_DIR=/home/runner/local/petsc From e3a4e1e24e833c53dfc17307949a27ca4b9e7c0a Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Fri, 17 Jan 2025 15:04:17 -0800 Subject: [PATCH 19/22] OptionsNetCDF: verifyTimesteps, write override base class Add `override` keyword to squash warning. --- src/sys/options/options_netcdf.hxx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sys/options/options_netcdf.hxx b/src/sys/options/options_netcdf.hxx index 657068f5a8..09a2da3220 100644 --- a/src/sys/options/options_netcdf.hxx +++ b/src/sys/options/options_netcdf.hxx @@ -51,12 +51,12 @@ public: /// Write options to file void write(const Options& options) { write(options, "t"); } - void write(const Options& options, const std::string& time_dim); + void write(const Options& options, const std::string& time_dim) override; /// Check that all variables with the same time dimension have the /// same size in that dimension. Throws BoutException if there are /// any differences, otherwise is silent - void verifyTimesteps() const; + void verifyTimesteps() const override; private: enum class FileMode { From a18868233f6da3ffbbc9fce2185cfdf39de9d1ed Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Fri, 17 Jan 2025 15:25:19 -0800 Subject: [PATCH 20/22] tests/integrated: Fix hypre tests - Mesh variable names are case sensitive - mpirun needs ./ to find executable on some systems --- .../test-laplace-hypre3d/data_circular_core-sol/BOUT.inp | 4 ++-- .../test-laplace-hypre3d/data_circular_core/BOUT.inp | 4 ++-- tests/integrated/test-laplacexy2-hypre/runtest | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integrated/test-laplace-hypre3d/data_circular_core-sol/BOUT.inp b/tests/integrated/test-laplace-hypre3d/data_circular_core-sol/BOUT.inp index 46d3cb55ba..613e01f441 100644 --- a/tests/integrated/test-laplace-hypre3d/data_circular_core-sol/BOUT.inp +++ b/tests/integrated/test-laplace-hypre3d/data_circular_core-sol/BOUT.inp @@ -61,8 +61,8 @@ g_23 = Bt*hthe*Rxy/Bp # zShift = \int_{theta0}^{theta} {nu dtheta} arctanarg = (r - 1)*tan(theta/2.)/sqrt(1. - r^2) -zshift = r^2*(r*sin(theta)/((r^2 - 1)*(r*cos(theta) - 1.)) - 2.*atan(arctanarg)/(1. - r^2)^1.5) -shiftangle = r^2 * 2.*pi/(1. - r^2)^1.5 +zShift = r^2*(r*sin(theta)/((r^2 - 1)*(r*cos(theta) - 1.)) - 2.*atan(arctanarg)/(1. - r^2)^1.5) +ShiftAngle = r^2 * 2.*pi/(1. - r^2)^1.5 [mesh:paralleltransform] type = shiftedinterp diff --git a/tests/integrated/test-laplace-hypre3d/data_circular_core/BOUT.inp b/tests/integrated/test-laplace-hypre3d/data_circular_core/BOUT.inp index be0c697d80..c62e032a95 100644 --- a/tests/integrated/test-laplace-hypre3d/data_circular_core/BOUT.inp +++ b/tests/integrated/test-laplace-hypre3d/data_circular_core/BOUT.inp @@ -58,8 +58,8 @@ g_23 = Bt*hthe*Rxy/Bp # zShift = \int_{theta0}^{theta} {nu dtheta} arctanarg = (r - 1)*tan(theta/2.)/sqrt(1. - r^2) -zshift = r^2*(r*sin(theta)/((r^2 - 1)*(r*cos(theta) - 1.)) - 2.*atan(arctanarg)/(1. - r^2)^1.5) -shiftangle = r^2 * 2.*pi/(1. - r^2)^1.5 +zShift = r^2*(r*sin(theta)/((r^2 - 1)*(r*cos(theta) - 1.)) - 2.*atan(arctanarg)/(1. - r^2)^1.5) +ShiftAngle = r^2 * 2.*pi/(1. - r^2)^1.5 [mesh:paralleltransform] type = shiftedinterp diff --git a/tests/integrated/test-laplacexy2-hypre/runtest b/tests/integrated/test-laplacexy2-hypre/runtest index c35c3609da..0a6a7ea53e 100755 --- a/tests/integrated/test-laplacexy2-hypre/runtest +++ b/tests/integrated/test-laplacexy2-hypre/runtest @@ -44,7 +44,7 @@ success = True for nproc in [8]: print(" %d processors...." % nproc) for args in argslist: - cmd = "test-laplacexy " + args + cmd = "./test-laplacexy " + args shell("rm data/BOUT.dmp.*.nc > /dev/null 2>&1") From 9d8cae1b28ffb4c6055f60b630e343882b62b953 Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Fri, 17 Jan 2025 15:29:24 -0800 Subject: [PATCH 21/22] hypre_interface: Add print() method to HypreMatrix Calls a Hypre function to print the matrix elements. --- include/bout/hypre_interface.hxx | 9 +++++++++ src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx | 8 ++++++++ src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx | 2 ++ 3 files changed, 19 insertions(+) diff --git a/include/bout/hypre_interface.hxx b/include/bout/hypre_interface.hxx index e8a0d462b0..ae392de4f3 100644 --- a/include/bout/hypre_interface.hxx +++ b/include/bout/hypre_interface.hxx @@ -710,6 +710,15 @@ public: bool isAssembled() const { return assembled; } + /// Call HYPRE_IJMatrixPrint + void print() const { + if (!assembled) { + output_warn.write(""); + return; + } + HYPRE_IJMatrixPrint(*hypre_matrix, "hypreIJ.matrix"); + } + HypreMatrix yup(int index = 0) { return ynext(index + 1); } HypreMatrix ydown(int index = 0) { return ynext(-index - 1); } HypreMatrix ynext(int dir) { diff --git a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx index fd705a9c83..a3abdd7b06 100644 --- a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx +++ b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx @@ -59,6 +59,9 @@ LaplaceHypre3d::LaplaceHypre3d(Options* opt, const CELL_LOC loc, Mesh* mesh_in, Ex.setLocation(location); Ez.setLocation(location); + // Print matrix coefficients? + print_matrix = (*opts)["print_matrix"].doc("Print matrix coefficients when the operator is updated?").withDefault(false); + // Initialise Hypre objects // Note: linearSystem is a bout::HypreSystem object linearSystem.setMatrix(&operator3D); @@ -405,6 +408,11 @@ void LaplaceHypre3d::updateMatrix3D() { operator3D.ydown(ydown)(l, l.ym().zm()) += C_d2f_dydz; } operator3D.assemble(); + + if (print_matrix) { + operator3D.print(); + } + linearSystem.setupAMG(&operator3D); updateRequired = false; diff --git a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx index 612f410a9a..eda9369815 100644 --- a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx +++ b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.hxx @@ -231,6 +231,8 @@ public: static constexpr int implemented_flags = INVERT_START_NEW, implemented_boundary_flags = INVERT_AC_GRAD + INVERT_SET + INVERT_RHS; + + bool print_matrix; ///< Print matrix coefficients }; #endif // BOUT_HAS_HYPRE From 99bce95c33e56c41ff05cae80d0eb074ec9b55b3 Mon Sep 17 00:00:00 2001 From: bendudson <219233+bendudson@users.noreply.github.com> Date: Fri, 17 Jan 2025 23:47:40 +0000 Subject: [PATCH 22/22] Apply clang-format changes --- src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx index 73558f679f..c50be1db85 100644 --- a/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx +++ b/src/invert/laplace/impls/hypre3d/hypre3d_laplace.cxx @@ -60,7 +60,9 @@ LaplaceHypre3d::LaplaceHypre3d(Options* opt, const CELL_LOC loc, Mesh* mesh_in, Ez.setLocation(location); // Print matrix coefficients? - print_matrix = (*opts)["print_matrix"].doc("Print matrix coefficients when the operator is updated?").withDefault(false); + print_matrix = (*opts)["print_matrix"] + .doc("Print matrix coefficients when the operator is updated?") + .withDefault(false); // Initialise Hypre objects // Note: linearSystem is a bout::HypreSystem object