diff --git a/.github/workflows/check-compilers.yml b/.github/workflows/check-compilers.yml index af7cc22f23aa..4a445e79347a 100644 --- a/.github/workflows/check-compilers.yml +++ b/.github/workflows/check-compilers.yml @@ -13,7 +13,7 @@ jobs: continue-on-error: true strategy: matrix: - cxx: ['g++', 'clang++-13'] + cxx: ['g++', 'clang++-15'] cmake_build_type: ['Release', 'Debug'] device: ['cuda', 'host'] parallel: ['serial', 'mpi'] @@ -21,7 +21,7 @@ jobs: # Debug cuda clang build fail for the unit test. # Exclude for now until we figure out what's going on. # https://github.com/lanl/parthenon/issues/630 - - cxx: clang++-13 + - cxx: clang++-15 device: cuda cmake_build_type: Debug runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 1918b5d78fd7..3bab48bc8d37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - [[PR 868]](https://github.com/parthenon-hpc-lab/parthenon/pull/868) Add block-local face, edge, and nodal fields and allow for packing ### Changed (changing behavior/API/variables/...) +- [[PR 896]](https://github.com/parthenon-hpc-lab/parthenon/pull/896) Update Kokkos integration to support installed version. Use `serial` (flat MPI) host parallelization by default (instead of OpenMP) - [[PR 894]](https://github.com/parthenon-hpc-lab/parthenon/pull/894) Demand that sparse pool order sparse ids - [[PR 888]](https://github.com/parthenon-hpc-lab/parthenon/pull/888) Bump Kokkos submodule to 4.0.1 - [[PR 885]](https://github.com/parthenon-hpc-lab/parthenon/pull/885) Expose PackDescriptor and use uids in SparsePacks diff --git a/CMakeLists.txt b/CMakeLists.txt index 1206de05fd90..455d63e4eb07 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,7 +34,6 @@ include(CTest) option(PARTHENON_SINGLE_PRECISION "Run in single precision" OFF) option(PARTHENON_DISABLE_MPI "MPI is enabled by default if found, set this to True to disable MPI" OFF) option(PARTHENON_ENABLE_HOST_COMM_BUFFERS "CUDA/HIP Only: Allocate communication buffers on host (may be slower)" OFF) -option(PARTHENON_DISABLE_OPENMP "OpenMP is enabled by default if found, set this to True to disable OpenMP" OFF) option(PARTHENON_DISABLE_HDF5 "HDF5 is enabled by default if found, set this to True to disable HDF5" OFF) option(PARTHENON_DISABLE_HDF5_COMPRESSION "HDF5 compression is enabled by default, set this to True to disable compression in HDF5 output/restart files" OFF) option(PARTHENON_DISABLE_SPARSE "Sparse capability is enabled by default, set this to True to compile-time disable all sparse capability" OFF) @@ -59,13 +58,6 @@ option(TEST_INTEL_OPTIMIZATION "Test intel optimization and vectorization" OFF) option(TEST_ERROR_CHECKING "Enables the error checking unit test. This test will FAIL" OFF) option(CODE_COVERAGE "Enable code coverage reporting" OFF) -# Default to an external Kokkos package if the submodule is not populated -if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/external/Kokkos/CMakeLists.txt" AND NOT EXISTS "${Kokkos_ROOT}/CMakeLists.txt") - option(PARTHENON_IMPORT_KOKKOS "If ON, attempt to link to an external Kokkos library. If OFF, build Kokkos from source and package with Parthenon" ON) -else() - option(PARTHENON_IMPORT_KOKKOS "If ON, attempt to link to an external Kokkos library. If OFF, build Kokkos from source and package with Parthenon" OFF) -endif() - include(cmake/Format.cmake) include(cmake/Lint.cmake) @@ -135,36 +127,11 @@ if (NOT PARTHENON_DISABLE_MPI) set(ENABLE_MPI ON) endif() -set(ENABLE_OPENMP OFF) -if (NOT PARTHENON_DISABLE_OPENMP) - # Using Host OpenMP and Cuda with nvcc currently does not compile with C++17 standard. - # Also there is no proper support for two separate execution spaces in Parthenon right now. - # We may need to revisit this logic when OpenMP target for devices will be used in the future. - if (Kokkos_ENABLE_CUDA OR Kokkos_ENABLE_HIP) - message(STATUS "Parthenon is not using Host OpenMP because Cuda or HIP is enabled") - else() - find_package(OpenMP COMPONENTS CXX) - if (NOT OpenMP_FOUND) - message(FATAL_ERROR "OpenMP is required but couldn't be found. " - "If you want to build Parthenon without OpenMP, please rerun CMake with -DPARTHENON_DISABLE_OPENMP=ON") - endif() - set(ENABLE_OPENMP ON) - set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "Allow Kokkos to use OpenMP as execution space.") - endif() -endif() - set(ENABLE_SPARSE ON) if (PARTHENON_DISABLE_SPARSE) set(ENABLE_SPARSE OFF) endif() -if (Kokkos_ENABLE_CUDA AND TEST_INTEL_OPTIMIZATION) - message(WARNING - "Intel optimizer flags may not be passed through NVCC wrapper correctly. " - "If you encounter problems, please delete your CMake cache " - "and rerun CMake with -DTEST_INTEL_OPTIMIZATION=OFF.") -endif() - set(ENABLE_HDF5 OFF) if (NOT PARTHENON_DISABLE_HDF5) set(HDF5_PREFER_PARALLEL ${ENABLE_MPI}) @@ -220,53 +187,55 @@ endif() # Kokkos recommendatation resulting in not using default GNU extensions set(CMAKE_CXX_EXTENSIONS OFF) -# Tell Kokkos to vectorize aggressively -# Kokkos prefers this capitalization for debugging reasons -SET (Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION ON CACHE BOOL - "Kokkos aggressive vectorization") - # Check that gpu devices are actually detected set(NUM_GPU_DEVICES_PER_NODE "1" CACHE STRING "Number of gpu devices to use when testing if built with Kokkos_ENABLE_CUDA") set(NUM_OMP_THREADS_PER_RANK "1" CACHE STRING "Number of threads to use when testing if built with Kokkos_ENABLE_OPENMP") -if (Kokkos_ENABLE_CUDA) - # Tell Kokkos we need lambdas in Cuda. - SET (Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL - "Enable lambda expressions in CUDA") - if ( "${PARTHENON_ENABLE_GPU_MPI_CHECKS}" ) - configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/CTestCustom.cmake.in ${CMAKE_BINARY_DIR}/CTestCustom.cmake @ONLY) - endif() -endif() - -# If this is a debug build, set kokkos debug on -if (${CMAKE_BUILD_TYPE} STREQUAL "Debug") - message(STATUS "Enabling Kokkos debug mode") - set(Kokkos_ENABLE_DEBUG ON CACHE BOOL "Most general debug settings") - set(Kokkos_ENABLE_DEBUG_BOUNDS_CHECK ON CACHE BOOL - "Bounds checking on Kokkos views") - set(Kokkos_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK ON CACHE BOOL - "Sanity checks on Kokkos DualView") -endif() if (ENABLE_COMPILER_WARNINGS) - message(STATUS "Enabling -Wall and setting Kokkos_ENABLE_COMPILER_WARNINGS=True") - set(Kokkos_ENABLE_COMPILER_WARNINGS True CACHE BOOL - "Make the compiler warn us about things") + message(STATUS "Enabling -Wall.") add_compile_options(-Wall) endif() -set(Kokkos_ENABLE_DEPRECATED_CODE_3 OFF CACHE BOOL "No need for old/unused code.") - set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_STANDARD 17) +option(PARTHENON_IMPORT_KOKKOS "If ON, attempt to link to an external Kokkos library. If OFF, build Kokkos from source and package with Parthenon" OFF) if (NOT TARGET Kokkos::kokkos) if (PARTHENON_IMPORT_KOKKOS) find_package(Kokkos 4) if (NOT Kokkos_FOUND) - unset(PARTHENON_IMPORT_KOKKOS CACHE) message(FATAL_ERROR "Could not find external Kokkos. Consider importing a Kokkos installation into your environment or disabling external Kokkos with e.g. -DPARTHENON_IMPORT_KOKKOS=OFF") + else() + message(STATUS "Using imported Kokkos from ${Kokkos_CONFIG}. " + "All manually defined Kokkos variables will be ignored as they have no affect on the imported " + "(installed) version. If you want to use a custom Kokkos version, please use the one shipped " + "as Parthenon submodule.") endif() else() + # First, set some Kokkos options + # Tell Kokkos to vectorize aggressively + # Kokkos prefers this capitalization for debugging reasons + SET (Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION ON CACHE BOOL + "Kokkos aggressive vectorization") + + # If this is a debug build, set kokkos debug on + if (${CMAKE_BUILD_TYPE} STREQUAL "Debug") + message(STATUS "Enabling Kokkos debug mode") + set(Kokkos_ENABLE_DEBUG ON CACHE BOOL "Most general debug settings") + set(Kokkos_ENABLE_DEBUG_BOUNDS_CHECK ON CACHE BOOL + "Bounds checking on Kokkos views") + set(Kokkos_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK ON CACHE BOOL + "Sanity checks on Kokkos DualView") + endif() + + # Also enable Compiler warnings for Kokkos + if (ENABLE_COMPILER_WARNINGS) + message(STATUS "Setting Kokkos_ENABLE_COMPILER_WARNINGS=True") + set(Kokkos_ENABLE_COMPILER_WARNINGS True CACHE BOOL + "Make the compiler warn us about things") + endif() + + # Second, add Kokkos if (EXISTS ${Kokkos_ROOT}/CMakeLists.txt) add_subdirectory(${Kokkos_ROOT} Kokkos) message(STATUS "Using Kokkos source from Kokkos_ROOT=${Kokkos_ROOT}") @@ -279,6 +248,10 @@ if (NOT TARGET Kokkos::kokkos) endif() endif() +message(STATUS "CUDA: ${Kokkos_ENABLE_CUDA}") + +# After we have imported Kokkos we can now report/check our config as Kokkos_ENABLE_XXX +# is also availalbe when imported. if (Kokkos_ENABLE_SYCL) message(WARNING "SYCL backend is currently NOT tested in Parthenon due to lack of access to hardware. " @@ -293,6 +266,32 @@ if (PARTHENON_ENABLE_HOST_COMM_BUFFERS) endif() endif() +if (Kokkos_ENABLE_CUDA AND TEST_INTEL_OPTIMIZATION) + message(WARNING + "Intel optimizer flags may not be passed through NVCC wrapper correctly. " + "If you encounter problems, please delete your CMake cache " + "and rerun CMake with -DTEST_INTEL_OPTIMIZATION=OFF.") +endif() + +# Globally turn on useful intel and/or nvcc compiler output +if (Kokkos_ENABLE_CUDA) + if(CHECK_REGISTRY_PRESSURE) + add_compile_options(-Xptxas=-v) + endif() +endif() +# Note that these options may not play nice with nvcc wrapper +if (TEST_INTEL_OPTIMIZATION) + add_compile_options(-fp-model fast=2 -qopt_report5 -vec-threshold0 -qopt_report_phase=vec) +endif() + +# GPU check on the build node are currently only supported for Nvidia GPUs +if (Kokkos_ENABLE_CUDA AND "${PARTHENON_ENABLE_GPU_MPI_CHECKS}" ) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/CTestCustom.cmake.in ${CMAKE_BINARY_DIR}/CTestCustom.cmake @ONLY) +endif() + + + + # Build Tests and download Catch2 if (PARTHENON_ENABLE_UNIT_TESTS OR PARTHENON_ENABLE_INTEGRATION_TESTS OR PARTHENON_ENABLE_REGRESSION_TESTS OR PARTHENON_ENABLE_PERFORMANCE_TESTS) @@ -352,17 +351,6 @@ if (PARTHENON_ENABLE_UNIT_TESTS OR PARTHENON_ENABLE_INTEGRATION_TESTS OR PARTHEN add_subdirectory(tst) endif() -# Globally turn on useful intel and/or nvcc compiler output -if (Kokkos_ENABLE_CUDA) - if(CHECK_REGISTRY_PRESSURE) - add_compile_options(-Xptxas=-v) - endif() -endif() -# Note that these options may not play nice with nvcc wrapper -if (TEST_INTEL_OPTIMIZATION) - add_compile_options(-fp-model fast=2 -qopt_report5 -vec-threshold0 -qopt_report_phase=vec) -endif() - if (PARTHENON_ENABLE_ASCENT) find_package(Ascent REQUIRED NO_DEFAULT_PATH) endif() diff --git a/benchmarks/burgers/README.md b/benchmarks/burgers/README.md index 9995ca7ca837..9b4acfb85243 100644 --- a/benchmarks/burgers/README.md +++ b/benchmarks/burgers/README.md @@ -58,7 +58,7 @@ To build Parthenon on CPU, including this benchmark, with minimal external depen ```bash parthenon$ mkdir build && cd build -parthenon$ cmake -DPARTHENON_DISABLE_HDF5=ON -DPARTHENON_DISABLE_OPENMP=ON -DPARTHENON_ENABLE_PYTHON_MODULE_CHECK=OFF ../ +parthenon$ cmake -DPARTHENON_DISABLE_HDF5=ON -DPARTHENON_ENABLE_PYTHON_MODULE_CHECK=OFF ../ parthenon$ make -j ``` The executable `burgers-benchmark` should be built in `parthenon/build/benchmarks/burgers/` and can be run as, e.g. diff --git a/cmake/machinecfg/FrontierAndCrusher.cmake b/cmake/machinecfg/FrontierAndCrusher.cmake index 3d0769dba965..ae318c93232c 100644 --- a/cmake/machinecfg/FrontierAndCrusher.cmake +++ b/cmake/machinecfg/FrontierAndCrusher.cmake @@ -21,16 +21,15 @@ message(STATUS "Loading machine configuration for OLCF's Frontier and Crusher.\n " $ module load PrgEnv-amd craype-accel-amd-gfx90a cmake hdf5 cray-python amd/5.4.0 cray-mpich/8.1.21\n" "and environment variables:\n" " $ export MPICH_GPU_SUPPORT_ENABLED=1\n\n" - "On Frontier, different modules are required (tested on 2023-03-14): \n" - " $ module load PrgEnv-cray craype-accel-amd-gfx90a cmake hdf5 cray-python rocm/5.3.0 cray-mpich/8.1.23\n" + "On Frontier, different modules are required (tested on 2023-06-21): \n" + " $ load PrgEnv-cray craype-accel-amd-gfx90a cmake cray-hdf5-parallel cray-python amd-mixed/5.3.0 cray-mpich/8.1.23 cce/15.0.1\n" " $ export MPICH_GPU_SUPPORT_ENABLED=1\n\n" "NOTE: In order to run the test suite, the build directory should be on GPFS (or on\n" "Frontier, Lustre) work filesystem and not in your NFS user or project home (because\n" "they are read-only on compute nodes).\n\n") # common options -set(Kokkos_ARCH_ZEN2 ON CACHE BOOL "CPU architecture") -set(PARTHENON_DISABLE_OPENMP ON CACHE BOOL "OpenMP support not yet tested in Parthenon.") +set(Kokkos_ARCH_ZEN3 ON CACHE BOOL "CPU architecture") set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Default release build") set(MACHINE_VARIANT "hip-mpi" CACHE STRING "Default build for CUDA and MPI") @@ -63,7 +62,7 @@ if (${MACHINE_VARIANT} MATCHES "hip") set(MACHINE_CXX_FLAGS "${MACHINE_CXX_FLAGS} -I$ENV{MPICH_DIR}/include") set(CMAKE_EXE_LINKER_FLAGS "-L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa" CACHE STRING "Default flags for this config") elseif (${MACHINE_VARIANT} MATCHES "cray") - set(MACHINE_CXX_FLAGS "${MACHINE_CXX_FLAGS} -I$ENV{ROCM_PATH_DIR}/include") + set(MACHINE_CXX_FLAGS "${MACHINE_CXX_FLAGS} -I$ENV{ROCM_PATH}/include") set(CMAKE_EXE_LINKER_FLAGS "-L$ENV{ROCM_PATH}/lib -lamdhip64" CACHE STRING "Default flags for this config") endif() diff --git a/cmake/machinecfg/Ookami.cmake b/cmake/machinecfg/Ookami.cmake index b6a5e0862c07..e00b5b8ff03a 100644 --- a/cmake/machinecfg/Ookami.cmake +++ b/cmake/machinecfg/Ookami.cmake @@ -23,7 +23,6 @@ message(STATUS "Loading machine configuration for Stony Brook's A64FX Ookami.\n" set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Default release build") set(Kokkos_ARCH_A64FX ON CACHE BOOL "CPU architecture") -set(PARTHENON_DISABLE_OPENMP ON CACHE BOOL "OpenMP support not yet tested in Parthenon.") set(CMAKE_CXX_COMPILER "mpiFCC" CACHE STRING "Default compiler") set(CMAKE_CXX_FLAGS "-Nclang -ffj-fast-matmul -ffast-math -ffp-contract=fast -ffj-fp-relaxed -ffj-ilfunc -fbuiltin -fomit-frame-pointer -finline-functions -ffj-preex -ffj-zfill -ffj-swp -fopenmp-simd" CACHE STRING "Default opt flags") diff --git a/cmake/machinecfg/Spock.cmake b/cmake/machinecfg/Spock.cmake index 42018f8ba1a2..89cdbeb6a5f1 100644 --- a/cmake/machinecfg/Spock.cmake +++ b/cmake/machinecfg/Spock.cmake @@ -26,7 +26,6 @@ message(STATUS "Loading machine configuration for OLCF's Spock.\n" # common options set(Kokkos_ARCH_ZEN2 ON CACHE BOOL "CPU architecture") -set(PARTHENON_DISABLE_OPENMP ON CACHE BOOL "OpenMP support not yet tested in Parthenon.") set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Default release build") set(MACHINE_VARIANT "hip-mpi" CACHE STRING "Default build for CUDA and MPI") diff --git a/cmake/machinecfg/Summit.cmake b/cmake/machinecfg/Summit.cmake index 18b5ec87dd0e..c30fdc21ed58 100644 --- a/cmake/machinecfg/Summit.cmake +++ b/cmake/machinecfg/Summit.cmake @@ -23,7 +23,6 @@ message(STATUS "Loading machine configuration for OLCF's Summit.\n" # common options set(Kokkos_ARCH_POWER9 ON CACHE BOOL "CPU architecture") -set(PARTHENON_DISABLE_OPENMP ON CACHE BOOL "OpenMP support not yet tested in Parthenon.") set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Default release build") set(MACHINE_VARIANT "cuda-mpi" CACHE STRING "Default build for CUDA and MPI") diff --git a/doc/sphinx/src/building.rst b/doc/sphinx/src/building.rst index fdf3ab4b210b..29559f1b1101 100644 --- a/doc/sphinx/src/building.rst +++ b/doc/sphinx/src/building.rst @@ -22,7 +22,6 @@ General list of cmake options: || PARTHENON\_DISABLE\_MPI || OFF || Option || MPI is enabled by default if found, set this to True to disable MPI | || PARTHENON\_ENABLE\_HOST\_COMM\_BUFFERS || OFF || Option || MPI communication buffers are by default allocated on the execution device. This options forces allocation in memory accessible directly by the host. | || PARTHENON\_DISABLE\_SPARSE || OFF || Option || Disable sparse allocation of sparse variables, i.e., sparse variable still work but are always allocated. See also :ref:`sparse doc `. | -|| PARTHENON\_DISABLE\_OPENMP || OFF || Option || OpenMP is enabled by default if found, set this to True to disable OpenMP | || ENABLE\_COMPILER\_WARNINGS || OFF || Option || Enable compiler warnings | || TEST\_ERROR\_CHECKING || OFF || Option || Enables the error checking unit test. This test will FAIL | || TEST\_INTEL\_OPTIMIZATION || OFF || Option || Test intel optimization and vectorization | @@ -173,7 +172,7 @@ The below example ``CMakeLists.txt`` can be used to compile the cmake_minimum_required(VERSION 3.11) project(parthenon_linking_example) - set(Kokkos_CXX_STANDARD "c++14") + set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_EXTENSIONS OFF) find_package(parthenon REQUIRED PATHS "/path/to/parthenon/install") add_executable( @@ -579,7 +578,7 @@ Cuda with MPI # configure and build. Make sure to build in an directory on the GPFS filesystem if you want to run the regression tests because the home directory is not writeable from the compute nodes (which will result in the regression tests failing) $ mkdir build-cuda-mpi && cd build-cuda-mpi # note that we do not specify the mpicxx wrapper in the following as cmake automatically extracts the required include and linker options - $ cmake -DPARTHENON_DISABLE_HDF5=On -DCMAKE_BUILD_TYPE=Release -DKokkos_ENABLE_OPENMP=True -DKokkos_ARCH_POWER9=True -DKokkos_ENABLE_CUDA=True -DKokkos_ARCH_VOLTA70=True -DCMAKE_CXX_COMPILER=${PWD}/../external/Kokkos/bin/nvcc_wrapper .. + $ cmake -DPARTHENON_DISABLE_HDF5=On -DCMAKE_BUILD_TYPE=Release -DKokkos_ENABLE_OPENMP=ON -DKokkos_ARCH_POWER9=True -DKokkos_ENABLE_CUDA=True -DKokkos_ARCH_VOLTA70=True -DCMAKE_CXX_COMPILER=${PWD}/../external/Kokkos/bin/nvcc_wrapper .. $ make -j # The following commands are exepected to be run within job (interactive or scheduled) diff --git a/example/particle_tracers/particle_tracers.cpp b/example/particle_tracers/particle_tracers.cpp index 5887584cdc75..005bb6651848 100644 --- a/example/particle_tracers/particle_tracers.cpp +++ b/example/particle_tracers/particle_tracers.cpp @@ -374,7 +374,6 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { int num_tracers_meshblock = std::round(num_tracers * number_meshblock / number_mesh); int gid = pmb->gid; - int nbtotal = pmb->pmy_mesh->nbtotal; ParArrayND new_indices; swarm->AddEmptyParticles(num_tracers_meshblock, new_indices); diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 1ad91493d91d..698db6f0c0f3 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -314,8 +314,8 @@ TaskStatus ApplyCoarseBoundaryConditions(std::shared_ptr> &md) { TaskID AddBoundaryExchangeTasks(TaskID dependency, TaskList &tl, std::shared_ptr> &md, bool multilevel) { const auto any = BoundaryType::any; - const auto local = BoundaryType::local; - const auto nonlocal = BoundaryType::nonlocal; + // const auto local = BoundaryType::local; + // const auto nonlocal = BoundaryType::nonlocal; // auto send = tl.AddTask(dependency, SendBoundBufs, md); // auto send_local = tl.AddTask(dependency, SendBoundBufs, md); diff --git a/tst/unit/test_unit_integrators.cpp b/tst/unit/test_unit_integrators.cpp index e6ab151e05c2..834526d45503 100644 --- a/tst/unit/test_unit_integrators.cpp +++ b/tst/unit/test_unit_integrators.cpp @@ -142,7 +142,7 @@ void Integrate(const Integrator &integrator, const Stepper &step, const Real tf, TEST_CASE("Low storage integrator", "[StagedIntegrator]") { GIVEN("A state with an initial condition") { - Real t0 = 0, tf = 1.15; // delibarately not a nice fraction of a period + Real tf = 1.15; // delibarately not a nice fraction of a period State_t ufinal; GetTrueSolution(tf, ufinal); WHEN("We integrate with LowStorage rk1") {