Merge branch 'develop' into brryan/fix_swarm_unit_mem

.github/workflows/ci-extended.yml

@@ -115,7 +115,7 @@ jobs:
             exit 1
           fi
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: log-and-convergence-${{ matrix.device }}-${{ matrix.parallel }}
           path: |
@@ -173,7 +173,7 @@ jobs:
           cd build
           ctest -L regression -L ${{ matrix.parallel }} -LE perf-reg --timeout 3600
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: log-and-convergence-${{ matrix.parallel }}
           path: |

.github/workflows/ci-macos.yml

@@ -43,7 +43,7 @@ jobs:
         cd build
         ctest -LE mpi
 
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         name: configure-log-unit-macos
         path: build/CMakeFiles/CMakeOutput.log

.github/workflows/ci-short.yml

@@ -35,7 +35,7 @@ jobs:
         run: |
           cmake -DCMAKE_CXX_FLAGS=-Werror -Bbuild-copyright-check
           cmake --build build-copyright-check -t check-copyright
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: configure-log-style
           path: build-copyright-check/CMakeFiles/CMakeOutput.log
@@ -67,7 +67,7 @@ jobs:
           # Pick GPU with most available memory
           export CUDA_VISIBLE_DEVICES=$(nvidia-smi --query-gpu=memory.free,index --format=csv,nounits,noheader | sort -nr | head -1 | awk '{ print $NF }')
           ctest -LE 'performance|regression'
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: configure-log-unit-${{ matrix.device }}
           path: build/CMakeFiles/CMakeOutput.log
@@ -126,7 +126,7 @@ jobs:
           if grep HtoD profile.txt; then exit 1; fi
           if grep DtoH profile.txt; then exit 1; fi
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: configure-log-integration-${{ matrix.device }}
           path: |
@@ -168,7 +168,7 @@ jobs:
           cd build
           ctest -R regression_mpi_test:particle_leapfrog
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: configure-log-integration-amdgpu
           path: |

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Current develop
 
 ### Added (new features/APIs/variables/...)
+- [[PR 1210]](https://github.com/parthenon-hpc-lab/parthenon/pull/1210) Add cycle based output
 - [[PR 1103]](https://github.com/parthenon-hpc-lab/parthenon/pull/1103) Add sparsity to vector wave equation test
 - [[PR 1185]](https://github.com/parthenon-hpc-lab/parthenon/pull/1185) Bugfix to particle defragmentation
 - [[PR 1184]](https://github.com/parthenon-hpc-lab/parthenon/pull/1184) Fix swarm block neighbor indexing in 1D, 2D
@@ -12,6 +13,8 @@
 - [[PR 1161]](https://github.com/parthenon-hpc-lab/parthenon/pull/1161) Make flux field Metadata accessible, add Metadata::CellMemAligned flag, small perfomance upgrades
 
 ### Changed (changing behavior/API/variables/...)
+- [[PR 1223]](https://github.com/parthenon-hpc-lab/parthenon/pull/1223) Update to v4 artifacts
+- [[PR 1216]](https://github.com/parthenon-hpc-lab/parthenon/pull/1216) Move to Kokkos 4.5
 - [[PR 1191]](https://github.com/parthenon-hpc-lab/parthenon/pull/1191) Update Kokkos version to 4.4.1
 - [[PR 1209]](https://github.com/parthenon-hpc-lab/parthenon/pull/1209) Ordered history output
 - [[PR 1206]](https://github.com/parthenon-hpc-lab/parthenon/pull/1206) Leapfrog fix
@@ -22,6 +25,8 @@
 - [[PR 1172]](https://github.com/parthenon-hpc-lab/parthenon/pull/1172) Make parthenon manager robust against external MPI init and finalize calls
 
 ### Fixed (not changing behavior/API/variables/...)
+- [[PR 1217]](https://github.com/parthenon-hpc-lab/parthenon/pull/1217) Swarm comm debug fix
+- [[PR 1215]](https://github.com/parthenon-hpc-lab/parthenon/pull/1215) Fix issue with uninitialized input parameter block data
 - [[PR 1211]](https://github.com/parthenon-hpc-lab/parthenon/pull/1211) remove inline from WriteTaskGraph
 - [[PR 1188]](https://github.com/parthenon-hpc-lab/parthenon/pull/1188) Fix hdf5 output issue for metadata none variables, update test.
 - [[PR 1170]](https://github.com/parthenon-hpc-lab/parthenon/pull/1170) Fixed incorrect initialization of array by a const not constexpr
@@ -30,6 +35,7 @@
 - [[PR 1173]](https://github.com/parthenon-hpc-lab/parthenon/pull/1173) Make debugging easier by making parthenon throw an error if ParameterInput is different on multiple MPI ranks.
 
 ### Infrastructure (changes irrelevant to downstream codes)
+- [[PR1218]](https://github.com/parthenon-hpc-lab/parthenon/pull/1219) Add ability to run the tasking without threading
 - [[PR 1176]](https://github.com/parthenon-hpc-lab/parthenon/pull/1176) Move some code from header to implementation files
 
 ### Removed (removing behavior/API/varaibles/...)

CMakeLists.txt

@@ -1,9 +1,9 @@
 #=========================================================================================
 # Parthenon performance portable AMR framework
-# Copyright(C) 2020-2024 The Parthenon collaboration
+# Copyright(C) 2020-2025 The Parthenon collaboration
 # Licensed under the 3-clause BSD License, see LICENSE file for details
 #=========================================================================================
-# (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved.
+# (C) (or copyright) 2020-2025. Triad National Security, LLC. All rights reserved.
 #
 # This program was produced under U.S. Government contract 89233218CNA000001 for Los
 # Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -59,6 +59,7 @@ option(TEST_ERROR_CHECKING "Enables the error checking unit test. This test will
 option(CODE_COVERAGE "Enable code coverage reporting" OFF)
 option(ENABLE_ASAN "Turn on ASAN" OFF)
 option(ENABLE_HWASAN "Turn on HWASAN (currently ARM-only)" OFF)
+option(PARTHENON_USE_SERIAL_POOL "Disable threaded tasking" ON)
 
 option(PARTHENON_USE_SYSTEM_PACKAGES "Enables search for system packages when available" OFF)
 if (PARTHENON_USE_SYSTEM_PACKAGES)

doc/sphinx/src/building.rst

@@ -21,6 +21,7 @@ General list of cmake options:
 || PARTHENON\_ENABLE\_ASCENT                || OFF                           || Option || Enable Ascent for in situ visualization and analysis                                                                                                        |
 || PARTHENON\_DISABLE\_MPI                  || OFF                           || Option || MPI is enabled by default if found, set this to True to disable MPI                                                                                         |
 || PARTHENON\_ENABLE\_HOST\_COMM\_BUFFERS   || OFF                           || Option || MPI communication buffers are by default allocated on the execution device. This options forces allocation in memory accessible directly by the host.       |
+|| PARTHENON\_USE\_SERIAL\_POOL             || ON                            || Option || The parthenon tasking infrastructure is multi-threaded by default. This option disables the multi-threading in the tasking, but not in individual tasks.    |
 || PARTHENON\_DISABLE\_SPARSE               || OFF                           || Option || Disable sparse allocation of sparse variables, i.e., sparse variable still work but are always allocated. See also :ref:`sparse doc <sparse compile-time>`. |
 || ENABLE\_COMPILER\_WARNINGS               || OFF                           || Option || Enable compiler warnings                                                                                                                                    |
 || TEST\_ERROR\_CHECKING                    || OFF                           || Option || Enables the error checking unit test. This test will FAIL                                                                                                   |

doc/sphinx/src/outputs.rst

@@ -6,10 +6,20 @@ Outputs
 Outputs from Parthenon are controlled via ``<parthenon/output*>`` blocks,
 where ``*`` should be replaced by a unique integer for each block.
 
+The frequency of outputs can be controlled for each block separately
+and can be triggered by either (simulation) time or cycle, i.e.,
+
+- ``dt = 0.1`` means that the output for the block is written every 0.1
+  in simulation time.
+- ``dn = 100`` means that the output for the block is written every 100
+  cycles.
+
+Note that only one option can be chosen for a given block.
 To disable an output block without removing it from the input file set
-the block's ``dt < 0.0``.
+the block's ``dt < 0.0`` and ``dn < 0`` (which is also happening by default
+if the paramter is not provided in the input file).
 
-In addition to time base outputs, two additional options to trigger
+In addition to time or cycle based outputs, two additional options to trigger
 outputs (applies to HDF5, restart and histogram outputs) exist.
 
 -  Signaling: If ``Parthenon`` catches a signal, e.g., ``SIGALRM`` which

src/CMakeLists.txt

@@ -207,7 +207,6 @@ add_library(parthenon
   outputs/restart.hpp
   outputs/restart_hdf5.cpp
   outputs/restart_hdf5.hpp
-  outputs/vtk.cpp
 
   parthenon/driver.hpp
   parthenon/package.hpp

src/config.hpp.in

@@ -54,6 +54,9 @@
 // define PARTHENON_ENABLE_ASCENT or not at all
 #cmakedefine PARTHENON_ENABLE_ASCENT
 
+// define PARTHENON_USE_SERIAL_POOL or not at all
+#cmakedefine PARTHENON_USE_SERIAL_POOL
+
 // Default loop patterns for MeshBlock par_for() wrappers,
 // see kokkos_abstraction.hpp for available tags.
 // Kokkos tight loop layout

src/interface/swarm_comms.cpp

@@ -250,8 +250,8 @@ void Swarm::LoadBuffers_() {
           auto p_index = buffer_sorted(n).swarm_idx_;
           if (swarm_d.IsActive(p_index)) {
             const int m = buffer_sorted(n).sort_idx_;
-            const int bufid = neighbor_buffer_index(m);
             if (m >= 0) {
+              const int bufid = neighbor_buffer_index(m);
               const int bid = n - buffer_start[m];
               int buffer_index = bid * particle_size;
               swarm_d.MarkParticleForRemoval(p_index);

src/outputs/ascent.cpp

@@ -210,9 +210,15 @@ void AscentOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm,
 
   // advance output parameters
   output_params.file_number++;
-  output_params.next_time += output_params.dt;
   pin->SetInteger(output_params.block_name, "file_number", output_params.file_number);
-  pin->SetReal(output_params.block_name, "next_time", output_params.next_time);
+  if (output_params.dt > 0.0) {
+    output_params.next_time += output_params.dt;
+    pin->SetReal(output_params.block_name, "next_time", output_params.next_time);
+  }
+  if (output_params.dn > 0) {
+    output_params.next_n += output_params.dn;
+    pin->SetInteger(output_params.block_name, "next_n", output_params.next_n);
+  }
 }
 
 } // namespace parthenon

src/outputs/histogram.cpp

@@ -559,9 +559,15 @@ void HistogramOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm
     // Only applies to default time-based data dumps, so that writing "now" and "final"
     // outputs does not change the desired output numbering.
     output_params.file_number++;
-    output_params.next_time += output_params.dt;
     pin->SetInteger(output_params.block_name, "file_number", output_params.file_number);
-    pin->SetReal(output_params.block_name, "next_time", output_params.next_time);
+    if (output_params.dt > 0.0) {
+      output_params.next_time += output_params.dt;
+      pin->SetReal(output_params.block_name, "next_time", output_params.next_time);
+    }
+    if (output_params.dn > 0) {
+      output_params.next_n += output_params.dn;
+      pin->SetInteger(output_params.block_name, "next_n", output_params.next_n);
+    }
   }
 }
 

src/outputs/history.cpp

@@ -211,9 +211,15 @@ void HistoryOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm,
 
   // advance output parameters
   output_params.file_number++;
-  output_params.next_time += output_params.dt;
   pin->SetInteger(output_params.block_name, "file_number", output_params.file_number);
-  pin->SetReal(output_params.block_name, "next_time", output_params.next_time);
+  if (output_params.dt > 0.0) {
+    output_params.next_time += output_params.dt;
+    pin->SetReal(output_params.block_name, "next_time", output_params.next_time);
+  }
+  if (output_params.dn > 0) {
+    output_params.next_n += output_params.dn;
+    pin->SetInteger(output_params.block_name, "next_n", output_params.next_n);
+  }
 }
 
 } // namespace parthenon

src/outputs/output_parameters.hpp

@@ -44,6 +44,7 @@ struct OutputParameters {
   std::string data_format;
   std::vector<std::string> packages;
   Real next_time, dt;
+  int next_n, dn;
   int file_number;
   bool include_ghost_zones, cartesian_vector;
   bool single_precision_output;
@@ -53,7 +54,7 @@ struct OutputParameters {
   bool write_swarm_xdmf;
   // TODO(felker): some of the parameters in this class are not initialized in constructor
   OutputParameters()
-      : block_number(0), next_time(0.0), dt(-1.0), file_number(0),
+      : block_number(0), next_time(0.0), dt(-1.0), next_n(0), dn(-1), file_number(0),
         include_ghost_zones(false), cartesian_vector(false),
         single_precision_output(false), sparse_seed_nans(false),
         hdf5_compression_level(5), write_xdmf(false), write_swarm_xdmf(false) {}

src/outputs/outputs.cpp

@@ -117,19 +117,31 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) {
       op.block_name.assign(pib->block_name);
 
       Real dt = 0.0; // default value == 0 means that initial data is written by default
-      // for temporal drivers, setting dt to tlim ensures a final output is also written
+      int dn = -1;
       if (tm != nullptr) {
-        dt = pin->GetOrAddReal(op.block_name, "dt", tm->tlim);
+        dn = pin->GetOrAddInteger(op.block_name, "dn", -1.0);
+
+        // If this is a dn controlled output (dn >= 0), soft disable dt based triggering
+        // (-> dt = -1), otherwise setting dt to tlim ensures a final output is also
+        // written for temporal drivers.
+        const auto tlim = dn >= 0 ? -1 : tm->tlim;
+        dt = pin->GetOrAddReal(op.block_name, "dt", tlim);
       }
       // if this output is "soft-disabled" (negative value) skip processing
-      if (dt < 0.0) {
+      if (dt < 0.0 && dn < 0) {
         pib = pib->pnext; // move to next input block name
         continue;
       }
+
+      PARTHENON_REQUIRE_THROWS(!(dt >= 0.0 && dn >= 0),
+                               "dt and dn are enabled for the same output block, which "
+                               "is not supported. Please set at most one value >= 0.");
       // set time of last output, time between outputs
       if (tm != nullptr) {
         op.next_time = pin->GetOrAddReal(op.block_name, "next_time", tm->time);
         op.dt = dt;
+        op.next_n = pin->GetOrAddInteger(op.block_name, "next_n", tm->ncycle);
+        op.dn = dn;
       }
 
       // set file number, basename, id, and format
@@ -259,8 +271,6 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) {
       if (op.file_type == "hst") {
         pnew_type = new HistoryOutput(op);
         num_hst_outputs++;
-      } else if (op.file_type == "vtk") {
-        pnew_type = new VTKOutput(op);
       } else if (op.file_type == "ascent") {
         pnew_type = new AscentOutput(op);
       } else if (op.file_type == "histogram") {
@@ -438,9 +448,19 @@ void Outputs::MakeOutputs(Mesh *pm, ParameterInput *pin, SimTime *tm,
   OutputType *ptype = pfirst_type_;
   while (ptype != nullptr) {
     if ((tm == nullptr) ||
-        ((ptype->output_params.dt >= 0.0) &&
-         ((tm->ncycle == 0) || (tm->time >= ptype->output_params.next_time) ||
-          (tm->time >= tm->tlim) || (signal == SignalHandler::OutputSignal::now) ||
+        // output is not soft disabled and
+        (((ptype->output_params.dt >= 0.0) || (ptype->output_params.dn >= 0)) &&
+         // either dump initial data
+         ((tm->ncycle == 0) ||
+          //  or by triggering time or cycle based conditions
+          ((ptype->output_params.dt >= 0.0) &&
+           ((tm->time >= ptype->output_params.next_time) ||
+            (tm->tlim > 0.0 && tm->time >= tm->tlim))) ||
+          ((ptype->output_params.dn >= 0) &&
+           ((tm->ncycle >= ptype->output_params.next_n) ||
+            (tm->nlim > 0 && tm->ncycle >= tm->nlim))) ||
+          // or by manual triggers
+          (signal == SignalHandler::OutputSignal::now) ||
           (signal == SignalHandler::OutputSignal::final) ||
           (signal == SignalHandler::OutputSignal::analysis &&
            ptype->output_params.analysis_flag)))) {

src/outputs/outputs.hpp

@@ -143,18 +143,6 @@ class HistoryOutput : public OutputType {
                        const SignalHandler::OutputSignal signal) override;
 };
 
-//----------------------------------------------------------------------------------------
-//! \class VTKOutput
-//  \brief derived OutputType class for vtk dumps
-
-class VTKOutput : public OutputType {
- public:
-  explicit VTKOutput(const OutputParameters &oparams) : OutputType(oparams) {}
-  void WriteContainer(SimTime &tm, Mesh *pm, ParameterInput *pin, bool flag) override;
-  void WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm,
-                       const SignalHandler::OutputSignal signal) override;
-};
-
 //----------------------------------------------------------------------------------------
 //! \class AscentOutput
 //  \brief derived OutputType class for Ascent in situ situ visualization and analysis

src/outputs/parthenon_hdf5.cpp

@@ -597,9 +597,15 @@ std::string PHDF5Output::GenerateFilename_(ParameterInput *pin, SimTime *tm,
     // Only applies to default time-based data dumps, so that writing "now" and "final"
     // outputs does not change the desired output numbering.
     output_params.file_number++;
-    output_params.next_time += output_params.dt;
     pin->SetInteger(output_params.block_name, "file_number", output_params.file_number);
-    pin->SetReal(output_params.block_name, "next_time", output_params.next_time);
+    if (output_params.dt > 0.0) {
+      output_params.next_time += output_params.dt;
+      pin->SetReal(output_params.block_name, "next_time", output_params.next_time);
+    }
+    if (output_params.dn > 0) {
+      output_params.next_n += output_params.dn;
+      pin->SetInteger(output_params.block_name, "next_n", output_params.next_n);
+    }
   }
   return filename;
 }