Fix minor HYPRE and ADIOS2 compilation issues

.github/workflows/tests.yml

@@ -47,9 +47,10 @@ jobs:
                             -DBOUT_USE_PETSC=ON
                             -DBOUT_USE_SLEPC=ON
                             -DBOUT_USE_SUNDIALS=ON
+                            -DBOUT_USE_HYPRE=OFF
                             -DBOUT_USE_ADIOS2=ON
                             -DBOUT_ENABLE_PYTHON=ON
-                            -DADIOS2_ROOT=/home/runner/local/adios2
+                            -DADIOS2_ROOT=/home/runner/local
                             -DSUNDIALS_ROOT=/home/runner/local
                             -DPETSC_DIR=/home/runner/local/petsc
                             -DSLEPC_DIR=/home/runner/local/slepc"
@@ -74,6 +75,7 @@ jobs:
                             -DBOUT_USE_PETSC=ON
                             -DBOUT_USE_SLEPC=ON
                             -DBOUT_USE_SUNDIALS=ON
+                            -DBOUT_USE_HYPRE=ON
                             -DSUNDIALS_ROOT=/home/runner/local"
             on_cron: false
 
@@ -86,6 +88,7 @@ jobs:
                             -DBOUT_USE_PETSC=ON
                             -DBOUT_USE_SLEPC=ON
                             -DBOUT_USE_SUNDIALS=ON
+                            -DBOUT_USE_HYPRE=ON
                             -DSUNDIALS_ROOT=/home/runner/local"
             on_cron: false
 
@@ -97,6 +100,7 @@ jobs:
                             -DBOUT_USE_PETSC=ON
                             -DBOUT_USE_SLEPC=ON
                             -DBOUT_USE_SUNDIALS=ON
+                            -DBOUT_USE_HYPRE=ON
                             -DBOUT_BUILD_DOCS=OFF
                             -DSUNDIALS_ROOT=/home/runner/local"
             omp_num_threads: 2
@@ -110,6 +114,7 @@ jobs:
                             -DBOUT_USE_PETSC=ON
                             -DBOUT_USE_SLEPC=ON
                             -DBOUT_USE_SUNDIALS=ON
+                            -DBOUT_USE_HYPRE=OFF
                             -DBOUT_ENABLE_PYTHON=ON
                             -DSUNDIALS_ROOT=/home/runner/local"
             omp_num_threads: 2
@@ -123,6 +128,7 @@ jobs:
                             -DBOUT_USE_PETSC=ON
                             -DBOUT_USE_SLEPC=ON
                             -DBOUT_USE_SUNDIALS=ON
+                            -DBOUT_USE_HYPRE=OFF
                             -DBOUT_ENABLE_PYTHON=ON
                             -DSUNDIALS_ROOT=/home/runner/local
                             -DPETSC_DIR=/home/runner/local/petsc
@@ -178,6 +184,7 @@ jobs:
                  slepc-dev
                  liblapack-dev
                  libparpack2-dev
+                 libhypre-dev
 
       - uses: actions/checkout@v4
         with:

cmake/FindHYPRE.cmake

@@ -7,14 +7,15 @@ include(FindPackageHandleStandardArgs)
 
 find_package(HYPRE CONFIG QUIET)
 if (HYPRE_FOUND)
+  message(STATUS "Found HYPRE: ${HYPRE_VERSION}")
   return()
 endif()
 
 find_path(HYPRE_INCLUDE_DIR
   NAMES HYPRE.h
   DOC "HYPRE include directories"
   REQUIRED
-  PATH_SUFFIXES include
+  PATH_SUFFIXES include include/hypre
 )
 
 find_library(HYPRE_LIBRARY

cmake/SetupBOUTThirdParty.cmake

@@ -190,7 +190,7 @@ endif()
 message(STATUS "NetCDF support: ${BOUT_USE_NETCDF}")
 set(BOUT_HAS_NETCDF ${BOUT_USE_NETCDF})
 
-option(BOUT_USE_ADIOS2 "Enable support for ADIOS output" ON)
+option(BOUT_USE_ADIOS2 "Enable support for ADIOS output" OFF)
 option(BOUT_DOWNLOAD_ADIOS2 "Download and build ADIOS2" OFF)
 if (BOUT_USE_ADIOS2)
   if (BOUT_DOWNLOAD_ADIOS2)
@@ -214,14 +214,10 @@ if (BOUT_USE_ADIOS2)
     target_link_libraries(bout++ PUBLIC adios2::cxx11_mpi)
     message(STATUS "ADIOS2 done configuring")
   else()
-    find_package(ADIOS2)
-    if (ADIOS2_FOUND)
-      ENABLE_LANGUAGE(C)
-      find_package(MPI REQUIRED COMPONENTS C)
-      target_link_libraries(bout++ PUBLIC adios2::cxx11_mpi MPI::MPI_C)
-    else()
-      set(BOUT_USE_ADIOS2 OFF)
-    endif()
+    find_package(ADIOS2 REQUIRED)
+    enable_language(C)
+    find_package(MPI REQUIRED COMPONENTS C)
+    target_link_libraries(bout++ PUBLIC adios2::cxx11_mpi MPI::MPI_C)
   endif()
 endif()
 message(STATUS "ADIOS2 support: ${BOUT_USE_ADIOS2}")

examples/elm-pb/data-hypre/BOUT.inp

@@ -6,29 +6,23 @@
 
 nout = 40          # number of time-steps
 timestep = 1       # time between outputs
-wall_limit = 1.55  # wall time limit (in hours)
 
 zperiod = 15        # Fraction of a torus to simulate
 MZ = 16             # Number of points in Z
 
 grid = "cbm18_dens8.grid_nx68ny64.nc"  # Grid file
 
-dump_format = "nc"      # Dump file format. "nc" = NetCDF, "pdb" = PDB
-restart_format = "nc"   # Restart file format
-
 [mesh]
 
 staggergrids = false    # Use staggered grids
 
 [mesh:paralleltransform]
-
-type = shifted # Use shifted metric method
+type = shiftedinterp
 
 ##################################################
 # derivative methods
 
 [mesh:ddx]
-
 first = C4  # order of first x derivatives
 second = C4 # order of second x derivatives
 upwind = W3 # order of upwinding method W3 = Weno3
@@ -44,20 +38,6 @@ first = C4  # Z derivatives can be done using FFT
 second = C4
 upwind = W3
 
-[output]
-shiftoutput = true  # Put the output into field-aligned coordinates
-
-##################################################
-# Laplacian inversion routines
-
-[laplace]
-type=hypre3d
-
-flags = 0   # Flags for Laplacian inversion
-
-rtol = 1.e-9
-atol = 1.e-14
-
 ##################################################
 # FFTs
 
@@ -71,15 +51,12 @@ fft_measurement_flag = measure  # If using FFTW, perform tests to determine fast
 [solver]
 
 # mudq, mldq, mukeep, mlkeep preconditioner options
-atol = 1e-08 # absolute tolerance
-rtol = 1e-05  # relative tolerance
+atol = 1.0e-8 # absolute tolerance
+rtol = 1.0e-5  # relative tolerance
 
 use_precon = false    # Use preconditioner: User-supplied or BBD
-use_jacobian = false  # Use user-supplied Jacobian
 
 mxstep = 5000   # Number of internal steps between outputs
-adams_moulton = false # Use Adams-Moulton method (default is BDF)
-func_iter = false     # Functional iteration (default is Newton)
 
 ##################################################
 # settings for high-beta reduced MHD
@@ -116,7 +93,7 @@ diamag_phi0 = true    # Balance ExB against Vd for stationary equilibrium
 bm_exb_flag = 0
 bm_mag_flag = 0
 ##################################################################
-withflow = false     # With flow or not
+withflow = false    # With flow or not
 D_0 = 130000        # differential potential
 D_s = 20            # shear parameter
 K_H_term = false    # Contain K-H term
@@ -125,8 +102,8 @@ x0 = 0.855          # peak location
 D_min = 3000        # constant
 ##################################################################
 
-eHall = false         # Include electron pressue effects in Ohm's law?
-AA = 2.0          # ion mass in units of proton mass
+eHall = false         # Include electron pressure effects in Ohm's law?
+AA = 2.0              # ion mass in units of proton mass
 
 noshear = false        # zero all shear
 
@@ -174,14 +151,14 @@ damp_t_const = 1e-2  # Damping time constant
 
 ## Parallel pressure diffusion
 
-diffusion_par = -1.0   # Parallel pressure diffusion (< 0 = none)
+diffusion_par = -1.0    # Parallel pressure diffusion (< 0 = none)
 diffusion_p4 = -1e-05   # parallel hyper-viscous diffusion for pressure (< 0 = none)
-diffusion_u4 = 1e-05    # parallel hyper-viscous diffusion for vorticity (< 0 = none)
+diffusion_u4 = -1e-05   # parallel hyper-viscous diffusion for vorticity (< 0 = none)
 diffusion_a4 = -1e-05   # parallel hyper-viscous diffusion for vector potential (< 0 = none)
 
 ## heat source in pressure in watts
 
-heating_P = -1   #   heat power in watts (< 0 = none)
+heating_P = -1     #   heat power in watts (< 0 = none)
 hp_width = 0.1     #   heat width, in percentage of nx (< 0 = none)
 hp_length = 0.3    #   heat length in percentage of nx (< 0 = none)
 
@@ -205,22 +182,19 @@ su_lengthr = 0.1     #   right edge sink length in percentage of nx (< 0 = none)
 ## Viscosity and Hyper-viscosity
 
 viscos_par = -0.1   # Parallel viscosity (< 0 = none)
-viscos_perp = -1.0  # Perpendicular
+viscos_perp = -1.0  # Perpendicular viscosity (< 0 = none)
 hyperviscos = -1.0  # Radial hyper viscosity
 
 ## Compressional terms (only when compress = true)
 phi_curv = true    # Include curvature*Grad(phi) in P equation
 # gamma = 1.6666
 
 [phiSolver]
-#inner_boundary_flags = 1 + 2 + 128 # INVERT_DC_GRAD + INVERT_AC_GRAD + INVERT_BNDRY_ONE
-#outer_boundary_flags = 1 + 2 + 128 # INVERT_DC_GRAD + INVERT_AC_GRAD + INVERT_BNDRY_ONE
-inner_boundary_flags = 1 + 4 # INVERT_DC_GRAD + INVERT_AC_LAP
-outer_boundary_flags = 1 + 4 # INVERT_DC_GRAD + INVERT_AC_LAP
+type = hypre3d
+inner_boundary_flags = 0 # Dirichlet
+outer_boundary_flags = 0 # Dirichlet
 
 [aparSolver]
-#inner_boundary_flags = 1 + 2 + 128 # INVERT_DC_GRAD + INVERT_AC_GRAD + INVERT_BNDRY_ONE
-#outer_boundary_flags = 1 + 2 + 128 # INVERT_DC_GRAD + INVERT_AC_GRAD + INVERT_BNDRY_ONE
 inner_boundary_flags = 1 + 4 # INVERT_DC_GRAD + INVERT_AC_LAP
 outer_boundary_flags = 1 + 4 # INVERT_DC_GRAD + INVERT_AC_LAP
 
@@ -271,12 +245,9 @@ bndry_ydown = free_o3
 # Zero gradient in the core
 bndry_core = neumann
 
-[Vpar]
-
-bndry_core = neumann
-
 [phi]
 
 bndry_xin = none
 bndry_xout = none
-bndry_target = neumann
+#bndry_target = neumann
+

examples/elm-pb/data/BOUT.inp

@@ -94,7 +94,7 @@ diamag_phi0 = true    # Balance ExB against Vd for stationary equilibrium
 bm_exb_flag = 0
 bm_mag_flag = 0
 ##################################################################
-withflow = false     # With flow or not
+withflow = false    # With flow or not
 D_0 = 130000        # differential potential
 D_s = 20            # shear parameter
 K_H_term = false    # Contain K-H term
@@ -104,7 +104,7 @@ D_min = 3000        # constant
 ##################################################################
 
 eHall = false         # Include electron pressure effects in Ohm's law?
-AA = 2.0          # ion mass in units of proton mass
+AA = 2.0              # ion mass in units of proton mass
 
 noshear = false        # zero all shear
 
@@ -152,14 +152,14 @@ damp_t_const = 1e-2  # Damping time constant
 
 ## Parallel pressure diffusion
 
-diffusion_par = -1.0   # Parallel pressure diffusion (< 0 = none)
+diffusion_par = -1.0    # Parallel pressure diffusion (< 0 = none)
 diffusion_p4 = -1e-05   # parallel hyper-viscous diffusion for pressure (< 0 = none)
-diffusion_u4 = -1e-05    # parallel hyper-viscous diffusion for vorticity (< 0 = none)
+diffusion_u4 = -1e-05   # parallel hyper-viscous diffusion for vorticity (< 0 = none)
 diffusion_a4 = -1e-05   # parallel hyper-viscous diffusion for vector potential (< 0 = none)
 
 ## heat source in pressure in watts
 
-heating_P = -1   #   heat power in watts (< 0 = none)
+heating_P = -1     #   heat power in watts (< 0 = none)
 hp_width = 0.1     #   heat width, in percentage of nx (< 0 = none)
 hp_length = 0.3    #   heat length in percentage of nx (< 0 = none)
 

examples/elm-pb/elm_pb.cxx

@@ -1142,6 +1142,9 @@ class ELMpb : public PhysicsModel {
 
     // Create a solver for the Laplacian
     phiSolver = Laplacian::create(&globalOptions["phiSolver"]);
+    // Save performance metrics to output, using the
+    // given name as the prefix.
+    phiSolver->savePerformance(*solver, "phiSolver");
 
     aparSolver = Laplacian::create(&globalOptions["aparSolver"], loc);
 

include/bout/hypre_interface.hxx

@@ -710,6 +710,15 @@ public:
 
   bool isAssembled() const { return assembled; }
 
+  /// Call HYPRE_IJMatrixPrint
+  void print() const {
+    if (!assembled) {
+      output_warn.write("<HypreMatrix not assembled>");
+      return;
+    }
+    HYPRE_IJMatrixPrint(*hypre_matrix, "hypreIJ.matrix");
+  }
+
   HypreMatrix<T> yup(int index = 0) { return ynext(index + 1); }
   HypreMatrix<T> ydown(int index = 0) { return ynext(-index - 1); }
   HypreMatrix<T> ynext(int dir) {
@@ -894,20 +903,30 @@ public:
 
   void setMaxIter(int max_iter) { checkHypreError(solverSetMaxIter(solver, max_iter)); }
 
-  double getFinalRelResNorm() {
+  double getFinalRelResNorm() const {
     HYPRE_Real resnorm{};
     checkHypreError(solverGetFinalRelativeResidualNorm(solver, &resnorm));
     return resnorm;
   }
 
-  int getNumItersTaken() {
+  /// Return the number of solver iterations taken
+  int getNumItersTaken() const {
     HYPRE_Int iters{};
     checkHypreError(solverGetNumIterations(solver, &iters));
     return iters;
   }
 
+  /// Return the number of BoomerAMG preconditioner iterations taken
+  int getNumItersTakenAMG() const {
+    HYPRE_Int iters{};
+    checkHypreError(HYPRE_BoomerAMGGetNumIterations(precon, &iters));
+    return iters;
+  }
+
+  /// Set the HypreMatrix to be used in the solver
   void setMatrix(HypreMatrix<T>* A_) { A = A_; }
 
+  /// Enable BoomerAMG preconditioner with the given HypreMatrix
   int setupAMG(HypreMatrix<T>* P_) {
     CALI_CXX_MARK_FUNCTION;
 

include/bout/paralleltransform.hxx

@@ -216,7 +216,7 @@ public:
   std::vector<PositionsAndWeights>
   getWeightsForYApproximation(int UNUSED(i), int UNUSED(j), int UNUSED(k),
                               int UNUSED(yoffset)) override {
-    throw BoutException("ParallelTransform::getWeightsForYApproximation not implemented"
+    throw BoutException("ParallelTransform::getWeightsForYApproximation not implemented "
                         "for `type = shifted`. Try `type = shiftedinterp`");
   }