diff --git a/.github/workflows/formatting.yml b/.github/workflows/formatting.yml index a1c441815..6ef9375c9 100644 --- a/.github/workflows/formatting.yml +++ b/.github/workflows/formatting.yml @@ -18,6 +18,11 @@ jobs: run: export DEBIAN_FRONTEND=noninteractive - name: install dependencies run: | - sudo apt-get install -y --force-yes -qq git clang-format-12 + sudo apt-get install -y --force-yes -qq git + - name: Install pip, clang-format + uses: BSFishy/pip-action@v1 + with: + packages: | + clang-format==12.0.1 - name: check formatting - run: find . -regex '.*\.\(cpp\|hpp\)' | xargs clang-format-12 -style=file -i && git diff --exit-code --ignore-submodules + run: find . -regex '.*\.\(cpp\|hpp\)' | xargs clang-format -style=file -i && git diff --exit-code --ignore-submodules diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 67a558c2d..fa1df2f1f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,22 +32,22 @@ jobs: cd tst/regression mkdir build_minkowski_release cd build_minkowski_release - cmake -DCMAKE_BUILD_TYPE=Release -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=Minkowski -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON ../../../ + cmake -DCMAKE_BUILD_TYPE=Release -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=Minkowski -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON -DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off ../../../ cmake --build . --parallel 8 cd .. mkdir build_fmks_release cd build_fmks_release - cmake -DCMAKE_BUILD_TYPE=Release -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=FMKS -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON ../../../ + cmake -DCMAKE_BUILD_TYPE=Release -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=FMKS -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON -DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off ../../../ cmake --build . --parallel 8 cd .. mkdir build_minkowski_debug cd build_minkowski_debug - cmake -DCMAKE_BUILD_TYPE=Debug -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=Minkowski -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON ../../../ + cmake -DCMAKE_BUILD_TYPE=Debug -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=Minkowski -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON -DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off ../../../ cmake --build . --parallel 8 cd .. mkdir build_fmks_debug cd build_fmks_debug - cmake -DCMAKE_BUILD_TYPE=Debug -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=FMKS -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON ../../../ + cmake -DCMAKE_BUILD_TYPE=Debug -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=FMKS -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON -DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off ../../../ cmake --build . --parallel 8 cd .. # is there a run directory that shouldn't be here? @@ -79,7 +79,7 @@ jobs: run: | mkdir build cd build - cmake -DCMAKE_BUILD_TYPE=Release -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=Minkowski -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON ../ + cmake -DCMAKE_BUILD_TYPE=Release -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=Minkowski -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON .-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off ../ cmake --build . --parallel 4 - name: Run unit tests run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 7aa9ad11c..bb8d5daca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -166,6 +166,7 @@ set(SINGULARITY_USE_FORTRAN OFF CACHE BOOL "" FORCE) set(SINGULARITY_FORCE_SUBMODULE_MODE ON CACHE BOOL "" FORCE) set(SINGULARITY_USE_KOKKOSKERNELS OFF CACHE BOOL "" FORCE) set(SINGULARITY_BUILD_CLOSURE OFF CACHE BOOL "" FORCE) +set(SINGULARITY_USE_TRUE_LOG_GRIDDING ON CACHE BOOL "" FORCE) # Kill cmake's package registry because it can interfere set(CMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY ON CACHE BOOL "" FORCE) @@ -184,6 +185,9 @@ include(cmake/Geometry.cmake) # Fluid include(cmake/Fluid.cmake) +# Transport +include(cmake/Transport.cmake) + # Phoebus src message("\nConfiguring src") add_subdirectory(src) diff --git a/cmake/Transport.cmake b/cmake/Transport.cmake new file mode 100644 index 000000000..33dd44a00 --- /dev/null +++ b/cmake/Transport.cmake @@ -0,0 +1,33 @@ +# © 2021-2025. Triad National Security, LLC. All rights reserved. This +# program was produced under U.S. Government contract 89233218CNA000001 +# for Los Alamos National Laboratory (LANL), which is operated by Triad +# National Security, LLC for the U.S. Department of Energy/National +# Nuclear Security Administration. All rights in the program are +# reserved by Triad National Security, LLC, and the U.S. Department of +# Energy/National Nuclear Security Administration. The Government is +# granted for itself and others acting on its behalf a nonexclusive, +# paid-up, irrevocable worldwide license in this material to reproduce, +# prepare derivative works, distribute copies to the public, perform +# publicly and display publicly, and to permit others to do so. + +option(PHOEBUS_DO_NU_ELECTRON "Evolve electron neutrinos" ON) +option(PHOEBUS_DO_NU_ELECTRON_ANTI "Evolve electron antineutrinos" ON) +option(PHOEBUS_DO_NU_HEAVY "Evolve heavy composite type neutrinos" ON) + +if(PHOEBUS_DO_NU_ELECTRON) + set(DO_NU_ELECTRON 1 CACHE BOOL "Electron neutrinos enabled") +else() + set(DO_NU_ELECTRON 0 CACHE BOOL "Electron neutrinos disabled") +endif() + +if(PHOEBUS_DO_NU_ELECTRON_ANTI) + set(DO_NU_ELECTRON_ANTI 1 CACHE BOOL "Electron antineutrinos enabled") +else() + set(DO_NU_ELECTRON_ANTI 0 CACHE BOOL "Electron antineutrinos disabled") +endif() + +if(PHOEBUS_DO_NU_HEAVY) + set(DO_NU_HEAVY 1 CACHE BOOL "Heavy neutrinos enabled") +else() + set(DO_NU_HEAVY 0 CACHE BOOL "Heavy neutrinos disabled") +endif() diff --git a/external/parthenon b/external/parthenon index 3225612e2..9b045aec7 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit 3225612e267456e5731f80c594d45e6457274c3f +Subproject commit 9b045aec7ed88be4aab1337ed0a9839f50974a3c diff --git a/external/singularity-eos b/external/singularity-eos index d443cb54a..e3738bc8b 160000 --- a/external/singularity-eos +++ b/external/singularity-eos @@ -1 +1 @@ -Subproject commit d443cb54a09234ac2fa0507210d6f978680124f4 +Subproject commit e3738bc8bd8d26e3b49adb302a4ae9477cdad33a diff --git a/external/singularity-opac b/external/singularity-opac index 8c1bd1a8e..b05c6e12f 160000 --- a/external/singularity-opac +++ b/external/singularity-opac @@ -1 +1 @@ -Subproject commit 8c1bd1a8ee714292fd4016e12835549337b6a0c1 +Subproject commit b05c6e12f003edae4cea38f6aa279c395aa11098 diff --git a/inputs/blandford_mckee.pin b/inputs/blandford_mckee.pin index 06d92a41e..bd8c15ed5 100644 --- a/inputs/blandford_mckee.pin +++ b/inputs/blandford_mckee.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -50,14 +50,14 @@ nghost = 4 nx1 = 512 # Number of zones in X1-direction x1min = 0 # minimum value of X1 x1max = 1 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 3.14159265359 # maximum value of X2. Pi -ix2_bc = reflecting # Inner-X2 boundary condition flag -ox2_bc = reflecting # Outer-X2 boundary condition flag +ix2_bc = reflect # Inner-X2 boundary condition flag +ox2_bc = reflect # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/blast_wave.pin b/inputs/blast_wave.pin index b504a28e8..6f38722ea 100644 --- a/inputs/blast_wave.pin +++ b/inputs/blast_wave.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -18,8 +18,6 @@ problem = sedov -ix2_bc = reflect -ox2_bc = reflect problem_id = sedov # problem ID: basename of output filenames @@ -55,14 +53,14 @@ nghost = 4 nx1 = 512 # Number of zones in X1-direction x1min = 0.0 # minimum value of X1 x1max = 0.3 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 3.14159265359 # maximum value of X2. Pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = reflect # Inner-X2 boundary condition flag +ox2_bc = reflect # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/blast_wave_2d.pin b/inputs/blast_wave_2d.pin index 8b8a2ed48..0c8080dd9 100644 --- a/inputs/blast_wave_2d.pin +++ b/inputs/blast_wave_2d.pin @@ -74,8 +74,8 @@ num_threads = 1 # maximum number of OMP threads bc_vars = conserved -nx1 = 32 -nx2 = 32 +nx1 = 64 +nx2 = 64 nx3 = 1 diff --git a/inputs/bondi.pin b/inputs/bondi.pin index bd8e8b081..a15ce64a1 100644 --- a/inputs/bondi.pin +++ b/inputs/bondi.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -12,10 +12,6 @@ problem = bondi -ix1_bc = outflow -ox1_bc = outflow -ix2_bc = reflect -ox2_bc = reflect problem_id = bondi # problem ID: basename of output filenames @@ -51,14 +47,14 @@ nghost = 4 nx1 = 256 # Number of zones in X1-direction x1min = 0.59 # minimum value of X1 x1max = 5 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 1 #3.14159265359 #1 # maximum value of X2. Pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = reflect # Inner-X2 boundary condition flag +ox2_bc = reflect # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/homogeneous_sphere.pin b/inputs/homogeneous_sphere.pin index 8f9bbd4fe..2b406d03f 100644 --- a/inputs/homogeneous_sphere.pin +++ b/inputs/homogeneous_sphere.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -12,8 +12,6 @@ problem = homogeneous_sphere -ix1_bc = reflect -ox1_bc = outflow problem_id = hom_sph # problem ID: basename of output filenames @@ -39,17 +37,17 @@ nghost = 4 #refinement = adaptive #numlevel = 3 -nx1 = 128 # Number of zones in X1-direction -x1min = 0 # minimum value of X1 -x1max = 5 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +nx1 = 128 # Number of zones in X1-direction +x1min = 0 # minimum value of X1 +x1max = 5 # maximum value of X1 +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag -nx2 = 1 # Number of zones in X2-direction -x2min = 0 # minimum value of X2 -x2max = 3.14159265359 # maximum value of X2 -ix2_bc = periodic # Inner-X2 boundary condition flag -ox2_bc = periodic # Outer-X2 boundary condition flag +nx2 = 1 # Number of zones in X2-direction +x2min = 0 # minimum value of X2 +x2max = 3.14159265359 # maximum value of X2 +ix2_bc = periodic # Inner-X2 boundary condition flag +ox2_bc = periodic # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = -1 # minimum value of X3 @@ -98,10 +96,6 @@ nu_min = 1.e15 nu_max = 1.e22 nu_bins = 200 absorption = true -do_nu_electron = true -do_nu_electron_anti = false -do_nu_heavy = false - B_fake = 1.0 use_B_fake = true @@ -110,7 +104,7 @@ type = gray gray_kappa = 1.e1 -J = 0.01 +J = 0.1 Hx = 0.01 Hy = 0.0 Hz = 0.0 diff --git a/inputs/homologous.pin b/inputs/homologous.pin index ca1327244..74b11e05b 100644 --- a/inputs/homologous.pin +++ b/inputs/homologous.pin @@ -1,4 +1,4 @@ -# © 2021-2022. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -14,10 +14,6 @@ problem = homologous -ix1_bc = reflect -ox1_bc = reflect -ix2_bc = reflect -ox2_bc = reflect problem_id = homologous # problem ID: basename of output filenames @@ -64,14 +60,14 @@ numlevel = 3 nx1 = 10000 # Number of zones in X1-direction x1min = 0 # minimum value of X1 x1max = 0.10086475522209183 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = reflect # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 3.14159265359 # maximum value of X2. Pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = reflect # Inner-X2 boundary condition flag +ox2_bc = reflect # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/leptoneq.pin b/inputs/leptoneq.pin index 303f85dca..2a6cc09f5 100644 --- a/inputs/leptoneq.pin +++ b/inputs/leptoneq.pin @@ -1,3 +1,15 @@ +# © 2021-2025. Triad National Security, LLC. All rights reserved. This +# program was produced under U.S. Government contract 89233218CNA000001 +# for Los Alamos National Laboratory (LANL), which is operated by Triad +# National Security, LLC for the U.S. Department of Energy/National +# Nuclear Security Administration. All rights in the program are +# reserved by Triad National Security, LLC, and the U.S. Department of +# Energy/National Nuclear Security Administration. The Government is +# granted for itself and others acting on its behalf a nonexclusive, +# paid-up, irrevocable worldwide license in this material to reproduce, +# prepare derivative works, distribute copies to the public, perform +# publicly and display publicly, and to permit others to do so. + problem = leptoneq diff --git a/inputs/mhd_shocktube.pin b/inputs/mhd_shocktube.pin index c7aab1cec..aea0f505e 100644 --- a/inputs/mhd_shocktube.pin +++ b/inputs/mhd_shocktube.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -12,8 +12,6 @@ problem = shock_tube -ix1_bc = outflow -ox1_bc = outflow problem_id = sod # problem ID: basename of output filenames @@ -43,13 +41,13 @@ dt_init_fact = 1.e-6 #numlevel = 3 nghost = 4 -nx1 = 256 # Number of zones in X1-direction -x1min = 0 # minimum value of X1 -x1max = 1 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +nx1 = 256 # Number of zones in X1-direction +x1min = 0 # minimum value of X1 +x1max = 1 # maximum value of X1 +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag -nx2 = 1 # Number of zones in X2-direction +nx2 = 1 # Number of zones in X2-direction x2min = -1 # minimum value of X2 x2max = 1 # maximum value of X2 ix2_bc = periodic # Inner-X2 boundary condition flag diff --git a/inputs/progenitor.pin b/inputs/progenitor.pin index 39ad7fe7b..9af901330 100644 --- a/inputs/progenitor.pin +++ b/inputs/progenitor.pin @@ -1,4 +1,4 @@ -# © 2021-2022. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -14,10 +14,6 @@ problem = homologous -ix1_bc = reflect -ox1_bc = reflect -ix2_bc = reflect -ox2_bc = reflect problem_id = homologous # problem ID: basename of output filenames @@ -73,14 +69,14 @@ bc_vars = conserved nx1 = 8000 # Number of zones in X1-direction x1min = 0 # minimum value of X1 x1max = 1. # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = reflect # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 3.14159265359 # maximum value of X2. Pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = reflect # Inner-X2 boundary condition flag +ox2_bc = reflect # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/radiation_advection.pin b/inputs/radiation_advection.pin index 155f07845..6ec4c4176 100644 --- a/inputs/radiation_advection.pin +++ b/inputs/radiation_advection.pin @@ -108,9 +108,6 @@ nu_min = 1.e-2 nu_max = 1.e2 nu_bins = 10 absorption = true -do_nu_electron = true -do_nu_electron_anti = false -do_nu_heavy = false src_solver = oned diff --git a/inputs/radiation_equilibration.pin b/inputs/radiation_equilibration.pin index cfbb5c7ff..e8f9d5bc5 100644 --- a/inputs/radiation_equilibration.pin +++ b/inputs/radiation_equilibration.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -12,8 +12,6 @@ problem = radiation_equilibration -bc_ix1 = outflow -bc_ox1 = outflow problem_id = rad_eql # problem ID: basename of output filenames @@ -101,9 +99,6 @@ nu_min = 1.e-2 nu_max = 1.e2 nu_bins = 10 absorption = true -do_nu_electron = true -do_nu_electron_anti = false -do_nu_heavy = false type = gray diff --git a/inputs/rhs_tester.pin b/inputs/rhs_tester.pin index 43d68120a..e72e1ab23 100644 --- a/inputs/rhs_tester.pin +++ b/inputs/rhs_tester.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -14,10 +14,6 @@ problem = rhs_tester -ix1_bc = reflect -ox1_bc = outflow -ix2_bc = reflect -ox2_bc = reflect problem_id = test # problem ID: basename of output filenames @@ -50,17 +46,17 @@ nghost = 4 #refinement = adaptive #numlevel = 3 -nx1 = 64 # Number of zones in X1-direction -x1min = 0 # minimum value of X1 -x1max = 1 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +nx1 = 64 # Number of zones in X1-direction +x1min = 0 # minimum value of X1 +x1max = 1 # maximum value of X1 +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 3.14159265359 # maximum value of X2. Pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = reflect # Inner-X2 boundary condition flag +ox2_bc = reflect # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/rotor.pin b/inputs/rotor.pin index ef6aa1e27..186cb87c0 100644 --- a/inputs/rotor.pin +++ b/inputs/rotor.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -12,8 +12,6 @@ problem = rotor -ix1_bc = outflow -ox1_bc = outflow problem_id = mhd_rotor # problem ID: basename of output filenames diff --git a/inputs/shocktube.pin b/inputs/shocktube.pin index dbb49ad15..ca8d9dfec 100644 --- a/inputs/shocktube.pin +++ b/inputs/shocktube.pin @@ -12,8 +12,6 @@ problem = shock_tube -ix1_bc = outflow -ox1_bc = outflow problem_id = sod # problem ID: basename of output filenames @@ -45,8 +43,8 @@ nghost = 4 nx1 = 512 # Number of zones in X1-direction x1min = 0 # minimum value of X1 x1max = 1 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = -1 # minimum value of X2 diff --git a/inputs/shocktube_nonrel.pin b/inputs/shocktube_nonrel.pin index af43eff6d..fab1b67c3 100644 --- a/inputs/shocktube_nonrel.pin +++ b/inputs/shocktube_nonrel.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -12,8 +12,6 @@ problem = shock_tube -ix1_bc = outflow -ox1_bc = outflow problem_id = sod # problem ID: basename of output filenames @@ -42,13 +40,13 @@ nghost = 4 #refinement = adaptive #numlevel = 3 -nx1 = 512 # Number of zones in X1-direction -x1min = 0 # minimum value of X1 -x1max = 1 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +nx1 = 512 # Number of zones in X1-direction +x1min = 0 # minimum value of X1 +x1max = 1 # maximum value of X1 +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag -nx2 = 1 # Number of zones in X2-direction +nx2 = 1 # Number of zones in X2-direction x2min = -1 # minimum value of X2 x2max = 1 # maximum value of X2 ix2_bc = periodic # Inner-X2 boundary condition flag diff --git a/inputs/shocktube_stellarcollapse.pin b/inputs/shocktube_stellarcollapse.pin index 4a96c2c48..4dec468a2 100644 --- a/inputs/shocktube_stellarcollapse.pin +++ b/inputs/shocktube_stellarcollapse.pin @@ -15,8 +15,6 @@ problem = shock_tube -ix1_bc = outflow -ox1_bc = outflow problem_id = sod # problem ID: basename of output filenames @@ -45,13 +43,13 @@ nghost = 4 #refinement = adaptive #numlevel = 3 -nx1 = 512 # Number of zones in X1-direction -x1min = 0 # minimum value of X1 -x1max = 1 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +nx1 = 512 # Number of zones in X1-direction +x1min = 0 # minimum value of X1 +x1max = 1 # maximum value of X1 +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag -nx2 = 1 # Number of zones in X2-direction +nx2 = 1 # Number of zones in X2-direction x2min = -1 # minimum value of X2 x2max = 1 # maximum value of X2 ix2_bc = periodic # Inner-X2 boundary condition flag diff --git a/inputs/shocktube_stellarcollapse_faketable.pin b/inputs/shocktube_stellarcollapse_faketable.pin index a36eb05bd..757b3f9ed 100644 --- a/inputs/shocktube_stellarcollapse_faketable.pin +++ b/inputs/shocktube_stellarcollapse_faketable.pin @@ -19,8 +19,6 @@ problem = shock_tube -ix1_bc = outflow -ox1_bc = outflow problem_id = sod # problem ID: basename of output filenames @@ -49,11 +47,11 @@ nghost = 4 #refinement = adaptive #numlevel = 3 -nx1 = 512 # Number of zones in X1-direction -x1min = 0 # minimum value of X1 -x1max = 1 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +nx1 = 512 # Number of zones in X1-direction +x1min = 0 # minimum value of X1 +x1max = 1 # maximum value of X1 +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = -1 # minimum value of X2 diff --git a/inputs/spherical_shock_tube.pin b/inputs/spherical_shock_tube.pin index 9865d978f..9070fd4dd 100644 --- a/inputs/spherical_shock_tube.pin +++ b/inputs/spherical_shock_tube.pin @@ -24,10 +24,6 @@ problem = shock_tube -ix1_bc = reflect -ox1_bc = outflow -ix2_bc = reflect -ox2_bc = reflect problem_id = sedov # problem ID: basename of output filenames @@ -79,14 +75,14 @@ nghost = 4 nx1 = 512 # Number of zones in X1-direction x1min = 0 # minimum value of X1 x1max = 1 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 3.14159265359 # maximum value of X2. Pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = reflect # Inner-X2 boundary condition flag +ox2_bc = reflect # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/torus.pin b/inputs/torus.pin index 0341d22c4..eff3fc183 100644 --- a/inputs/torus.pin +++ b/inputs/torus.pin @@ -1,10 +1,17 @@ +# © 2021-2025. Triad National Security, LLC. All rights reserved. This +# program was produced under U.S. Government contract 89233218CNA000001 +# for Los Alamos National Laboratory (LANL), which is operated by Triad +# National Security, LLC for the U.S. Department of Energy/National +# Nuclear Security Administration. All rights in the program are +# reserved by Triad National Security, LLC, and the U.S. Department of +# Energy/National Nuclear Security Administration. The Government is +# granted for itself and others acting on its behalf a nonexclusive, +# paid-up, irrevocable worldwide license in this material to reproduce, +# prepare derivative works, distribute copies to the public, perform +# publicly and display publicly, and to permit others to do so. problem = torus -ix1_bc = outflow -ox1_bc = outflow -ix2_bc = outflow -ox2_bc = outflow bc_vars = primitive @@ -59,14 +66,14 @@ nghost = 4 nx1 = 256 # Number of zones in X1-direction x1min = 0.59 # minimum value of X1; overwritten for the torus problem x1max = 3.69 # maximum value of X1; set by /r_outer for the torus problem -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 256 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 1 # maximum value of X2. X2 = 1 -> theta = pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = gr_outflow # Inner-X2 boundary condition flag +ox2_bc = gr_outflow # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/torus_cutout.pin b/inputs/torus_cutout.pin index f651812ee..0957c9346 100644 --- a/inputs/torus_cutout.pin +++ b/inputs/torus_cutout.pin @@ -1,10 +1,17 @@ +# © 2021-2025. Triad National Security, LLC. All rights reserved. This +# program was produced under U.S. Government contract 89233218CNA000001 +# for Los Alamos National Laboratory (LANL), which is operated by Triad +# National Security, LLC for the U.S. Department of Energy/National +# Nuclear Security Administration. All rights in the program are +# reserved by Triad National Security, LLC, and the U.S. Department of +# Energy/National Nuclear Security Administration. The Government is +# granted for itself and others acting on its behalf a nonexclusive, +# paid-up, irrevocable worldwide license in this material to reproduce, +# prepare derivative works, distribute copies to the public, perform +# publicly and display publicly, and to permit others to do so. problem = torus -ix1_bc = outflow -ox1_bc = outflow -ix2_bc = outflow -ox2_bc = outflow problem_id = torus # problem ID: basename of output filenames @@ -49,14 +56,14 @@ nghost = 4 nx1 = 128 # Number of zones in X1-direction x1min = 0.59 # minimum value of X1; overwritten for the torus problem x1max = 3.69 # maximum value of X1; set by /r_outer for the torus problem -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 128 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 1 # maximum value of X2. X2 = 1 -> theta = pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = gr_outflow # Inner-X2 boundary condition flag +ox2_bc = gr_outflow # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/torus_rad.pin b/inputs/torus_rad.pin index 484eed76a..0d618646a 100644 --- a/inputs/torus_rad.pin +++ b/inputs/torus_rad.pin @@ -1,10 +1,17 @@ +# © 2021-2025. Triad National Security, LLC. All rights reserved. This +# program was produced under U.S. Government contract 89233218CNA000001 +# for Los Alamos National Laboratory (LANL), which is operated by Triad +# National Security, LLC for the U.S. Department of Energy/National +# Nuclear Security Administration. All rights in the program are +# reserved by Triad National Security, LLC, and the U.S. Department of +# Energy/National Nuclear Security Administration. The Government is +# granted for itself and others acting on its behalf a nonexclusive, +# paid-up, irrevocable worldwide license in this material to reproduce, +# prepare derivative works, distribute copies to the public, perform +# publicly and display publicly, and to permit others to do so. problem = torus -ix1_bc = outflow -ox1_bc = outflow -ix2_bc = outflow -ox2_bc = outflow bc_vars = primitive @@ -58,15 +65,15 @@ nx1 = 256 # Number of zones in X1-direction #nx1 = 128 x1min = 0.59 # minimum value of X1; overwritten for the torus problem x1max = 3.69 # maximum value of X1; set by /r_outer for the torus problem -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 256 # Number of zones in X2-direction #nx2 = 128 x2min = 0 # minimum value of X2 x2max = 1 # maximum value of X2. X2 = 1 -> theta = pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = gr_outflow # Inner-X2 boundary condition flag +ox2_bc = gr_outflow # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction #nx3 = 64 diff --git a/inputs/tov-static.pin b/inputs/tov-static.pin index 0b5118356..7e4a11896 100644 --- a/inputs/tov-static.pin +++ b/inputs/tov-static.pin @@ -1,4 +1,4 @@ -# © 2021-2022. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -18,8 +18,6 @@ problem = tov -ix1_bc = reflect -ox1_bc = outflow problem_id = tov # problem ID: basename of output filenames @@ -62,8 +60,8 @@ nghost = 4 nx1 = 1024 # Number of zones in X1-direction x1min = 0 # minimum value of X1 x1max = 250 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 diff --git a/inputs/tov.pin b/inputs/tov.pin index 10aae4b72..54dc69803 100644 --- a/inputs/tov.pin +++ b/inputs/tov.pin @@ -1,4 +1,4 @@ -# © 2021-2022. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -14,10 +14,6 @@ problem = tov -ix1_bc = reflect -ox1_bc = outflow -ix2_bc = reflect -ox2_bc = reflect problem_id = tov # problem ID: basename of output filenames @@ -64,14 +60,14 @@ nghost = 4 nx1 = 1024 # Number of zones in X1-direction x1min = 0 # minimum value of X1 x1max = 250 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 3.14159265359 # maximum value of X2. Pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = reflect # Inner-X2 boundary condition flag +ox2_bc = reflect # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/tov_cart.pin b/inputs/tov_cart.pin index 1b6e43f18..178bdf5b4 100644 --- a/inputs/tov_cart.pin +++ b/inputs/tov_cart.pin @@ -19,16 +19,6 @@ problem = tov # change inner BCs to reflect # and change domain to [0,xmax] -ix1_bc = outflow -ox1_bc = outflow - -ix2_bc = outflow -ox2_bc = outflow - -ix3_bc = outflow -ox3_bc = outflow - - problem_id = tov # problem ID: basename of output filenames @@ -76,20 +66,20 @@ nghost = 4 nx1 = 200 # Number of zones in X1-direction x1min = -100 # minimum value of X1 x1max = 100 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 200 # Number of zones in X2-direction x2min = -100 # minimum value of X2 x2max = 100 # maximum value of X2. Pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = gr_outflow # Inner-X2 boundary condition flag +ox2_bc = gr_outflow # Outer-X2 boundary condition flag nx3 = 200 # Number of zones in X3-direction x3min = -100 # minimum value of X3 x3max = 100 # maximum value of X3. 2*pi -ix3_bc = user # Inner-X3 boundary condition flag -ox3_bc = user # Outer-X3 boundary condition flfgag +ix3_bc = gr_outflow # Inner-X3 boundary condition flag +ox3_bc = gr_outflow # Outer-X3 boundary condition flfgag num_threads = 1 # maximum number of OMP threads diff --git a/src/compile_constants.hpp.in b/src/compile_constants.hpp.in index 4fdf0db20..b81b147bd 100644 --- a/src/compile_constants.hpp.in +++ b/src/compile_constants.hpp.in @@ -16,6 +16,11 @@ #define NCONS_MAX (@MAX_NUMBER_CONSERVED_VARS@) +// NOTE(BLB) This will have to change if we support additional species +#define DO_NU_ELECTRON (@DO_NU_ELECTRON@) +#define DO_NU_ELECTRON_ANTI (@DO_NU_ELECTRON_ANTI@) +#define DO_NU_HEAVY (@DO_NU_HEAVY@) + #define PHOEBUS_GEOMETRY Geometry::@PHOEBUS_GEOMETRY@ #define GEOMETRY_MESH @GEOMETRY_MESH@ #define GEOMETRY_MESH_BLOCK @GEOMETRY_MESH_BLOCK@ @@ -26,4 +31,6 @@ #define CON2PRIM_STATISTICS (@CON2PRIM_STATISTICS@) +constexpr static int PHOEBUS_NUM_SPECIES = DO_NU_ELECTRON + DO_NU_ELECTRON_ANTI + DO_NU_HEAVY; + #endif //COMPILE_CONSTANTS_HPP_ diff --git a/src/fixup/fixup.cpp b/src/fixup/fixup.cpp index f2dbc0c4a..8026d6928 100644 --- a/src/fixup/fixup.cpp +++ b/src/fixup/fixup.cpp @@ -15,7 +15,6 @@ #include "fixup.hpp" -#include #include #include "analysis/history.hpp" @@ -718,6 +717,13 @@ TaskStatus ApplyFloors(T *rc) { } template TaskStatus ApplyFloors>(MeshBlockData *rc); +template <> +TaskStatus ApplyFloors>(MeshData *md) { + for (const auto &mbd : md->GetAllBlockData()) { + ApplyFloors(mbd.get()); + } + return TaskStatus::complete; +} TaskStatus FixFluxes(MeshBlockData *rc) { using parthenon::BoundaryFace; @@ -731,6 +737,8 @@ TaskStatus FixFluxes(MeshBlockData *rc) { const std::string ox1_bc = fluid->Param("ox1_bc"); const std::string ix2_bc = fluid->Param("ix2_bc"); const std::string ox2_bc = fluid->Param("ox2_bc"); + const std::string ix3_bc = fluid->Param("ix3_bc"); + const std::string ox3_bc = fluid->Param("ox3_bc"); auto rad = pmb->packages.Get("radiation"); int num_species = 0; @@ -750,7 +758,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { // x1-direction if (pmb->boundary_flag[BoundaryFace::inner_x1] == BoundaryFlag::user) { - if (ix1_bc == "outflow") { + if (ix1_bc == "gr_outflow") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes(std::vector({c::density::name()}), std::vector({c::density::name()}), @@ -786,7 +794,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { } } if (pmb->boundary_flag[BoundaryFace::outer_x1] == BoundaryFlag::user) { - if (ox1_bc == "outflow") { + if (ox1_bc == "gr_outflow") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes(std::vector({c::density::name()}), std::vector({c::density::name()}), @@ -825,7 +833,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { // x2-direction if (pmb->boundary_flag[BoundaryFace::inner_x2] == BoundaryFlag::user) { - if (ix2_bc == "outflow") { + if (ix2_bc == "gr_outflow") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes(std::vector({c::density::name()}), std::vector({c::density::name()}), @@ -907,7 +915,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { } } if (pmb->boundary_flag[BoundaryFace::outer_x2] == BoundaryFlag::user) { - if (ox2_bc == "outflow") { + if (ox2_bc == "gr_outflow") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes(std::vector({c::density::name()}), std::vector({c::density::name()}), @@ -993,7 +1001,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { if (ndim == 2) return TaskStatus::complete; // x3-direction - if (pmb->boundary_flag[BoundaryFace::inner_x3] == BoundaryFlag::outflow) { + if (ix3_bc == "gr_outflow") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes(std::vector({c::density::name()}), @@ -1004,7 +1012,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { v.flux(X3DIR, crho, k, j, i) = std::min(v.flux(X3DIR, crho, k, j, i), 0.0); }); - } else if (pmb->boundary_flag[BoundaryFace::inner_x3] == BoundaryFlag::reflect) { + } else if (ix3_bc == "reflect") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes( std::vector({c::density::name(), c::energy::name(), cr::E::name()}), @@ -1025,7 +1033,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { } }); } - if (pmb->boundary_flag[BoundaryFace::outer_x3] == BoundaryFlag::outflow) { + if (ox3_bc == "gr_outflow") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes(std::vector({c::density::name()}), @@ -1036,7 +1044,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { v.flux(X3DIR, crho, k, j, i) = std::max(v.flux(X3DIR, crho, k, j, i), 0.0); }); - } else if (pmb->boundary_flag[BoundaryFace::outer_x3] == BoundaryFlag::reflect) { + } else if (ox3_bc == "reflect") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes( std::vector({c::density::name(), c::energy::name(), cr::E::name()}), diff --git a/src/fixup/fixup_c2p.cpp b/src/fixup/fixup_c2p.cpp index 5e7147bc9..a0b098a3d 100644 --- a/src/fixup/fixup_c2p.cpp +++ b/src/fixup/fixup_c2p.cpp @@ -15,7 +15,6 @@ #include "fixup.hpp" -#include #include #include "fluid/con2prim_robust.hpp" @@ -384,5 +383,13 @@ TaskStatus ConservedToPrimitiveFixup(T *rc) { template TaskStatus ConservedToPrimitiveFixup>(MeshBlockData *rc); +// template TaskStatus +template <> +TaskStatus ConservedToPrimitiveFixup>(MeshData *md) { + for (const auto &mbd : md->GetAllBlockData()) { + ConservedToPrimitiveFixup(mbd.get()); + } + return TaskStatus::complete; +} } // namespace fixup diff --git a/src/fixup/fixup_netfield.cpp b/src/fixup/fixup_netfield.cpp index 3195b0a75..ca4787564 100644 --- a/src/fixup/fixup_netfield.cpp +++ b/src/fixup/fixup_netfield.cpp @@ -15,7 +15,6 @@ #include "fixup.hpp" -#include #include #include diff --git a/src/fixup/fixup_particles.cpp b/src/fixup/fixup_particles.cpp index 341fd2551..79f1cfe26 100644 --- a/src/fixup/fixup_particles.cpp +++ b/src/fixup/fixup_particles.cpp @@ -38,11 +38,12 @@ TaskStatus PurgeParticles(MeshBlockData *rc, const std::string swarmName) /* only do this when FMKS is used. */ if constexpr (std::is_same::value) { auto *pmb = rc->GetParentPointer(); - auto &swarm = pmb->swarm_data.Get()->Get(swarmName); + // auto &swarm = rc->swarm_data.Get()->Get(swarmName); + auto &swarm = rc->GetSwarmData()->Get(swarmName); - auto &x = swarm->Get("x").Get(); - auto &y = swarm->Get("y").Get(); - auto &z = swarm->Get("z").Get(); + auto &x = swarm->Get(swarm_position::x::name()).Get(); + auto &y = swarm->Get(swarm_position::y::name()).Get(); + auto &z = swarm->Get(swarm_position::z::name()).Get(); auto swarm_d = swarm->GetDeviceContext(); diff --git a/src/fixup/fixup_radc2p.cpp b/src/fixup/fixup_radc2p.cpp index 3ec3b3d75..6d1831505 100644 --- a/src/fixup/fixup_radc2p.cpp +++ b/src/fixup/fixup_radc2p.cpp @@ -15,7 +15,6 @@ #include "fixup.hpp" -#include #include #include "fluid/con2prim_robust.hpp" @@ -252,5 +251,12 @@ TaskStatus RadConservedToPrimitiveFixup(T *rc) { template TaskStatus RadConservedToPrimitiveFixup>(MeshBlockData *rc); +template <> +TaskStatus RadConservedToPrimitiveFixup>(MeshData *md) { + for (const auto &mbd : md->GetAllBlockData()) { + RadConservedToPrimitiveFixup(mbd.get()); + } + return TaskStatus::complete; +} } // namespace fixup diff --git a/src/fixup/fixup_src.cpp b/src/fixup/fixup_src.cpp index 54f7feac9..73ca6951c 100644 --- a/src/fixup/fixup_src.cpp +++ b/src/fixup/fixup_src.cpp @@ -15,7 +15,6 @@ #include "fixup.hpp" -#include #include #include "fluid/con2prim_robust.hpp" diff --git a/src/fluid/fluid.cpp b/src/fluid/fluid.cpp index 6e8be2b36..a23fa3edd 100644 --- a/src/fluid/fluid.cpp +++ b/src/fluid/fluid.cpp @@ -183,6 +183,10 @@ std::shared_ptr Initialize(ParameterInput *pin) { params.Add("ix2_bc", ix2_bc); const std::string ox2_bc = pin->GetString("phoebus", "ox2_bc"); params.Add("ox2_bc", ox2_bc); + const std::string ix3_bc = pin->GetString("phoebus", "ix3_bc"); + params.Add("ix3_bc", ix3_bc); + const std::string ox3_bc = pin->GetString("phoebus", "ox3_bc"); + params.Add("ox3_bc", ox3_bc); int ndim = 1; if (pin->GetInteger("parthenon/mesh", "nx3") > 1) @@ -768,6 +772,7 @@ TaskStatus CalculateFluxes(MeshBlockData *rc) { } TaskStatus FluxCT(MeshBlockData *rc) { + using parthenon::MakePackDescriptor; Mesh *pmesh = rc->GetMeshPointer(); auto &fluid = pmesh->packages.Get("fluid"); if (!fluid->Param("mhd") || !fluid->Param("active") || @@ -780,49 +785,54 @@ TaskStatus FluxCT(MeshBlockData *rc) { IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); IndexRange kb = rc->GetBoundsK(IndexDomain::interior); - auto f1 = rc->Get(fluid_cons::bfield::name()).flux[X1DIR]; - auto f2 = rc->Get(fluid_cons::bfield::name()).flux[X2DIR]; - auto f3 = rc->Get(fluid_cons::bfield::name()).flux[X3DIR]; + auto &resolved_pkgs = pmesh->resolved_packages; + static auto desc = MakePackDescriptor( + resolved_pkgs.get(), {}, {parthenon::PDOpt::WithFluxes}); + auto v = desc.GetPack(rc); auto emf = rc->Get(internal_variables::emf::name()).data; if (ndim == 2) { parthenon::par_for( DEFAULT_LOOP_PATTERN, "FluxCT::EMF::2D", DevExecSpace(), kb.s, kb.e, jb.s, jb.e + 1, ib.s, ib.e + 1, KOKKOS_LAMBDA(const int k, const int j, const int i) { - emf(k, j, i) = 0.25 * (f1(1, k, j, i) + f1(1, k, j - 1, i) - f2(0, k, j, i) - - f2(0, k, j, i - 1)); + emf(k, j, i) = + 0.25 * (v.flux(0, X1DIR, 1, k, j, i) + v.flux(0, X1DIR, 1, k, j - 1, i) - + v.flux(0, X2DIR, 0, k, j, i) - v.flux(0, X2DIR, 0, k, j, i - 1)); }); parthenon::par_for( DEFAULT_LOOP_PATTERN, "FluxCT::Flux::2D", DevExecSpace(), kb.s, kb.e, jb.s, jb.e + 1, ib.s, ib.e + 1, KOKKOS_LAMBDA(const int k, const int j, const int i) { - f1(0, k, j, i) = 0.0; - f1(1, k, j, i) = 0.5 * (emf(k, j, i) + emf(k, j + 1, i)); - f2(0, k, j, i) = -0.5 * (emf(k, j, i) + emf(k, j, i + 1)); - f2(1, k, j, i) = 0.0; + v.flux(0, X1DIR, 0, k, j, i) = 0.0; + v.flux(0, X1DIR, 1, k, j, i) = 0.5 * (emf(k, j, i) + emf(k, j + 1, i)); + v.flux(0, X2DIR, 0, k, j, i) = -0.5 * (emf(k, j, i) + emf(k, j, i + 1)); + v.flux(0, X2DIR, 1, k, j, i) = 0.0; }); } else if (ndim == 3) { parthenon::par_for( DEFAULT_LOOP_PATTERN, "FluxCT::EMF::3D", DevExecSpace(), kb.s, kb.e + 1, jb.s, jb.e + 1, ib.s, ib.e + 1, KOKKOS_LAMBDA(const int k, const int j, const int i) { - emf(0, k, j, i) = 0.25 * (f2(2, k, j, i) + f2(2, k - 1, j, i) - f3(1, k, j, i) - - f3(1, k, j - 1, i)); - emf(1, k, j, i) = -0.25 * (f1(2, k, j, i) + f1(2, k - 1, j, i) - - f3(0, k, j, i) - f3(0, k, j, i - 1)); - emf(2, k, j, i) = 0.25 * (f1(1, k, j, i) + f1(1, k, j - 1, i) - f2(0, k, j, i) - - f2(0, k, j, i - 1)); + emf(0, k, j, i) = + 0.25 * (v.flux(0, X2DIR, 2, k, j, i) + v.flux(0, X2DIR, 2, k - 1, j, i) - + v.flux(0, X3DIR, 1, k, j, i) - v.flux(0, X3DIR, 1, k, j - 1, i)); + emf(1, k, j, i) = + -0.25 * (v.flux(0, X1DIR, 2, k, j, i) + v.flux(0, X1DIR, 2, k - 1, j, i) - + v.flux(0, X3DIR, 0, k, j, i) - v.flux(0, X3DIR, 0, k, j, i - 1)); + emf(2, k, j, i) = + 0.25 * (v.flux(0, X1DIR, 1, k, j, i) + v.flux(0, X1DIR, 1, k, j - 1, i) - + v.flux(0, X2DIR, 0, k, j, i) - v.flux(0, X2DIR, 0, k, j, i - 1)); }); parthenon::par_for( DEFAULT_LOOP_PATTERN, "FluxCT::Flux::3D", DevExecSpace(), kb.s, kb.e + 1, jb.s, jb.e + 1, ib.s, ib.e + 1, KOKKOS_LAMBDA(const int k, const int j, const int i) { - f1(0, k, j, i) = 0.0; - f1(1, k, j, i) = 0.5 * (emf(2, k, j, i) + emf(2, k, j + 1, i)); - f1(2, k, j, i) = -0.5 * (emf(1, k, j, i) + emf(1, k + 1, j, i)); - f2(0, k, j, i) = -0.5 * (emf(2, k, j, i) + emf(2, k, j, i + 1)); - f2(1, k, j, i) = 0.0; - f2(2, k, j, i) = 0.5 * (emf(0, k, j, i) + emf(0, k + 1, j, i)); - f3(0, k, j, i) = 0.5 * (emf(1, k, j, i) + emf(1, k, j, i + 1)); - f3(1, k, j, i) = -0.5 * (emf(0, k, j, i) + emf(0, k, j + 1, i)); - f3(2, k, j, i) = 0.0; + v.flux(0, X1DIR, 0, k, j, i) = 0.0; + v.flux(0, X1DIR, 1, k, j, i) = 0.5 * (emf(2, k, j, i) + emf(2, k, j + 1, i)); + v.flux(0, X1DIR, 2, k, j, i) = -0.5 * (emf(1, k, j, i) + emf(1, k + 1, j, i)); + v.flux(0, X2DIR, 0, k, j, i) = -0.5 * (emf(2, k, j, i) + emf(2, k, j, i + 1)); + v.flux(0, X2DIR, 1, k, j, i) = 0.0; + v.flux(0, X2DIR, 2, k, j, i) = 0.5 * (emf(0, k, j, i) + emf(0, k + 1, j, i)); + v.flux(0, X3DIR, 0, k, j, i) = 0.5 * (emf(1, k, j, i) + emf(1, k, j, i + 1)); + v.flux(0, X3DIR, 1, k, j, i) = -0.5 * (emf(0, k, j, i) + emf(0, k, j + 1, i)); + v.flux(0, X3DIR, 2, k, j, i) = 0.0; }); } diff --git a/src/fluid/riemann.hpp b/src/fluid/riemann.hpp index 02500d4ee..87267df88 100644 --- a/src/fluid/riemann.hpp +++ b/src/fluid/riemann.hpp @@ -70,8 +70,16 @@ class FluxState { } static void FluxVars(const std::string &var) { flux_vars.push_back(var); } - static std::vector ReconVars() { return recon_vars; } - static std::vector FluxVars() { return flux_vars; } + static std::vector ReconVars() { + for (auto &v : recon_vars) { + } + return recon_vars; + } + static std::vector FluxVars() { + for (auto &v : flux_vars) { + } + return flux_vars; + } KOKKOS_FORCEINLINE_FUNCTION int NumConserved() const { return ncons; } @@ -229,12 +237,12 @@ class FluxState { pye(imap[fluid_prim::ye::name()].second), prs(imap[fluid_prim::pressure::name()].first), gm1(imap[fluid_prim::gamma1::name()].first), - crho(imap[fluid_cons::density::name()].first), - cmom_lo(imap[fluid_cons::momentum::name()].first), - ceng(imap[fluid_cons::energy::name()].first), - cb_lo(imap[fluid_cons::bfield::name()].first), - cb_hi(imap[fluid_cons::bfield::name()].second), - cye(imap[fluid_cons::ye::name()].first), + crho(imap[fluid_prim::density::name()].first), + cmom_lo(imap[fluid_prim::velocity::name()].first), + ceng(imap[fluid_prim::energy::name()].first), + cb_lo(imap[fluid_prim::bfield::name()].first), + cb_hi(imap[fluid_prim::bfield::name()].second), + cye(imap[fluid_prim::ye::name()].first), ncons(5 + (pb_hi - pb_lo + 1) + (cye > 0)) { PARTHENON_REQUIRE_THROWS( ncons <= NCONS_MAX, diff --git a/src/geometry/geometry.cpp b/src/geometry/geometry.cpp index 2ceff06d0..9d79ef428 100644 --- a/src/geometry/geometry.cpp +++ b/src/geometry/geometry.cpp @@ -54,8 +54,10 @@ std::shared_ptr Initialize(ParameterInput *pin) { Metadata gcoord_cell = Metadata({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, cell_shape); // TODO(JMM): Make this actual node-centered data when available + // TODO: Warning: g.n.coord output broken in Parthenon currently std::vector node_shape = {dims.nx1 + 1, dims.nx2 + 1, dims.nx3 + 1, 4}; - Metadata gcoord_node = Metadata({Metadata::Derived, Metadata::OneCopy}, node_shape); + Metadata gcoord_node = + Metadata({Metadata::Derived, Metadata::OneCopy, Metadata::None}, node_shape); geometry->AddField(geometric_variables::cell_coords::name(), gcoord_cell); geometry->AddField(geometric_variables::node_coords::name(), gcoord_node); diff --git a/src/microphysics/opac_phoebus/opac_phoebus.cpp b/src/microphysics/opac_phoebus/opac_phoebus.cpp index 62d807fed..b7ffe3aa5 100644 --- a/src/microphysics/opac_phoebus/opac_phoebus.cpp +++ b/src/microphysics/opac_phoebus/opac_phoebus.cpp @@ -144,8 +144,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { } { - auto opacity_host = - params.Get("h.opacity_baseunits"); + auto opacity_host = params.Get("h.opacity"); const Real YeMin = pin->GetOrAddReal("mean_opacity", "yemin", 0.1); const Real YeMax = pin->GetOrAddReal("mean_opacity", "yemax", 0.5); const int NYe = pin->GetOrAddInteger("mean_opacity", "nye", 10); @@ -159,10 +158,10 @@ std::shared_ptr Initialize(ParameterInput *pin) { const Real lNuMin = std::log10(pin->GetOrAddReal("mean_opacity", "numin", 0.1)); const Real lNuMax = std::log10(pin->GetOrAddReal("mean_opacity", "numax", 10.)); const int NNu = pin->GetOrAddInteger("mean_opacity", "nnu", 100); - MeanOpacity mean_opac_host = - MeanOpacityScaleFree(opacity_host, lRhoMin, lRhoMax, NRho, lTMin, lTMax, NT, - YeMin, YeMax, NYe, lNuMin, lNuMax, NNu); - auto mean_opac_device = mean_opac_host.GetOnDevice(); + auto mean_opac_host = + MeanOpacityBase(opacity_host, lRhoMin, lRhoMax, NRho, lTMin, lTMax, NT, YeMin, + YeMax, NYe, lNuMin, lNuMax, NNu); + MeanOpacity mean_opac_device = mean_opac_host.GetOnDevice(); params.Add("h.mean_opacity", mean_opac_host); params.Add("d.mean_opacity", mean_opac_device); } else { @@ -176,13 +175,13 @@ std::shared_ptr Initialize(ParameterInput *pin) { const Real lNuMin = std::log10(pin->GetOrAddReal("mean_opacity", "numin", 1.e10)); const Real lNuMax = std::log10(pin->GetOrAddReal("mean_opacity", "numax", 1.e24)); const int NNu = pin->GetOrAddInteger("mean_opacity", "nnu", 100); - MeanOpacityCGS cgs_mean_opacity = - MeanOpacityCGS(opacity_host, lRhoMin, lRhoMax, NRho, lTMin, lTMax, NT, YeMin, - YeMax, NYe, lNuMin, lNuMax, NNu); - MeanOpacity mean_opac_host = - MeanNonCGSUnits(std::forward(cgs_mean_opacity), - time_unit, mass_unit, length_unit, temp_unit); - auto mean_opac_device = mean_opac_host.GetOnDevice(); + auto cgs_mean_opacity = + MeanOpacityBase(opacity_host, lRhoMin, lRhoMax, NRho, lTMin, lTMax, NT, YeMin, + YeMax, NYe, lNuMin, lNuMax, NNu); + auto mean_opac_host = MeanNonCGSUnits( + std::forward(cgs_mean_opacity), time_unit, mass_unit, + length_unit, temp_unit); + MeanOpacity mean_opac_device = mean_opac_host.GetOnDevice(); params.Add("h.mean_opacity", mean_opac_host); params.Add("d.mean_opacity", mean_opac_device); } diff --git a/src/pgen/advection.cpp b/src/pgen/advection.cpp index bc1eae066..ca6929058 100644 --- a/src/pgen/advection.cpp +++ b/src/pgen/advection.cpp @@ -104,48 +104,79 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { }); fluid::PrimitiveToConserved(rc.get()); +} - /* tracer init section */ - if (do_tracers) { - auto &sc = pmb->swarm_data.Get(); - auto &swarm = pmb->swarm_data.Get()->Get("tracers"); - auto rng_pool = tracer_pkg->Param("rng_pool"); - - const Real &x_min = pmb->coords.Xf<1>(ib.s); - const Real &y_min = pmb->coords.Xf<2>(jb.s); - const Real &z_min = pmb->coords.Xf<3>(kb.s); - const Real &x_max = pmb->coords.Xf<1>(ib.e + 1); - const Real &y_max = pmb->coords.Xf<2>(jb.e + 1); - const Real &z_max = pmb->coords.Xf<3>(kb.e + 1); - - // as for num_tracers on each block... will get too many on multiple blocks - // TODO: distribute amongst blocks. - const auto num_tracers_total = tracer_pkg->Param("num_tracers"); - const int number_block = num_tracers_total; - - auto new_particles_context = swarm->AddEmptyParticles(number_block); - - auto &x = swarm->Get("x").Get(); - auto &y = swarm->Get("y").Get(); - auto &z = swarm->Get("z").Get(); - auto &id = swarm->Get("id").Get(); - - auto swarm_d = swarm->GetDeviceContext(); - - const int gid = pmb->gid; - const int max_active_index = new_particles_context.GetNewParticlesMaxIndex(); - pmb->par_for( - "ProblemGenerator::Torus::DistributeTracers", 0, max_active_index, - KOKKOS_LAMBDA(const int n) { - if (swarm_d.IsActive(n)) { +void PostInitializationModifier(ParameterInput *pin, Mesh *pmesh) { + + const int ndim = pmesh->ndim; + for (auto &pmb : pmesh->block_list) { + auto &rc = pmb->meshblock_data.Get(); + auto tracer_pkg = pmb->packages.Get("tracers"); + bool do_tracers = tracer_pkg->Param("active"); + const Real rin = pin->GetOrAddReal("advection", "rin", 0.1); + const Real v_inner = ndim == 3 ? (4. / 3.) * M_PI * std::pow(rin, 3.) + : ndim == 2 ? M_PI * rin * rin + : rin; + + auto geom = Geometry::GetCoordinateSystem(rc.get()); + auto coords = pmb->coords; + if (do_tracers) { + const auto num_tracers_total = tracer_pkg->Param("num_tracers"); + auto rng_pool = tracer_pkg->Param("rng_pool"); + auto &swarm = rc->GetSwarmData()->Get("tracers"); + + auto ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + auto jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + auto kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + Real number_block = 0.0; + + // Get fraction of block containing ye sphere + pmb->par_reduce( + "Phoebus::ProblemGenerator::Torus::BlockTracerNumber", kb.s, kb.e, jb.s, jb.e, + ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i, + Real &number_block_reduce) { + const Real dx1 = coords.Dxc<1>(k, j, i); + const Real dx2 = coords.Dxc<2>(k, j, i); + const Real dx3 = coords.Dxc<3>(k, j, i); + const Real x1 = coords.Xc<1>(k, j, i); + const Real x2 = coords.Xc<2>(k, j, i); + const Real x3 = coords.Xc<3>(k, j, i); + + if (x1 * x1 + x2 * x2 + x3 * x3 < rin * rin) { + Real vol_block = dx1 * dx2 * dx3; + number_block_reduce += vol_block; + } + }, + Kokkos::Sum(number_block)); + number_block /= v_inner; + number_block = number_block * num_tracers_total; + const int num_tracers_block = (int)number_block; + + // distribute + auto new_particles_context = swarm->AddEmptyParticles(number_block); + + auto &x = swarm->Get(swarm_position::x::name()).Get(); + auto &y = swarm->Get(swarm_position::y::name()).Get(); + auto &z = swarm->Get(swarm_position::z::name()).Get(); + auto &id = swarm->Get("id").Get(); + + auto swarm_d = swarm->GetDeviceContext(); + + const int gid = pmb->gid; + const int max_active_index = new_particles_context.GetNewParticlesMaxIndex(); + pmb->par_for( + "ProblemGenerator::Advection::DistributeTracers", 0, max_active_index, + KOKKOS_LAMBDA(const int new_n) { + const int n = new_particles_context.GetNewParticleIndex(new_n); auto rng_gen = rng_pool.get_state(); // sample in ye ball Real r2 = 1.0 + rin * rin; // init > rin^2 while (r2 > rin * rin) { - x(n) = x_min + rng_gen.drand() * (x_max - x_min); - y(n) = y_min + rng_gen.drand() * (y_max - y_min); - z(n) = z_min + rng_gen.drand() * (z_max - z_min); + x(n) = -rin + rng_gen.drand() * 2.0 * rin; // x \in [-rin, +rin] + y(n) = -rin + rng_gen.drand() * 2.0 * rin; + z(n) = -rin + rng_gen.drand() * 2.0 * rin; r2 = x(n) * x(n) + y(n) * y(n) + z(n) * z(n); } id(n) = num_tracers_total * gid + n; @@ -153,9 +184,10 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { bool on_current_mesh_block = true; swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); rng_pool.free_state(rng_gen); - } - }); + }); + } } -} + +} // PostInitializationModifier } // namespace advection diff --git a/src/pgen/pgen.hpp b/src/pgen/pgen.hpp index 5ae89d6dc..a6d94c1e7 100644 --- a/src/pgen/pgen.hpp +++ b/src/pgen/pgen.hpp @@ -61,6 +61,7 @@ using namespace parthenon::package::prelude; // the name here #define FOREACH_POSTINIT_MODIFIER \ POSTINIT_MODIFIER(phoebus) \ + POSTINIT_MODIFIER(advection) \ POSTINIT_MODIFIER(torus) /* diff --git a/src/pgen/torus.cpp b/src/pgen/torus.cpp index 6db236c9f..f481a3a7a 100644 --- a/src/pgen/torus.cpp +++ b/src/pgen/torus.cpp @@ -630,8 +630,7 @@ void PostInitializationModifier(ParameterInput *pin, Mesh *pmesh) { // tracer initialization. if (do_tracers) { - auto &sc = pmb->swarm_data.Get(); - auto &swarm = pmb->swarm_data.Get()->Get("tracers"); + auto &swarm = rc->GetSwarmData()->Get("tracers"); auto rng_pool = tracer_pkg->Param("rng_pool"); const auto num_tracers_total = tracer_pkg->Param("num_tracers"); @@ -669,9 +668,9 @@ void PostInitializationModifier(ParameterInput *pin, Mesh *pmesh) { auto new_particles_context = swarm->AddEmptyParticles(num_tracers); - auto &x = swarm->Get("x").Get(); - auto &y = swarm->Get("y").Get(); - auto &z = swarm->Get("z").Get(); + auto &x = swarm->Get(swarm_position::x::name()).Get(); + auto &y = swarm->Get(swarm_position::y::name()).Get(); + auto &z = swarm->Get(swarm_position::z::name()).Get(); auto &mass = swarm->Get("mass").Get(); auto &id = swarm->Get("id").Get(); diff --git a/src/phoebus_boundaries/phoebus_boundaries.cpp b/src/phoebus_boundaries/phoebus_boundaries.cpp index e440ddaa9..0000d6e48 100644 --- a/src/phoebus_boundaries/phoebus_boundaries.cpp +++ b/src/phoebus_boundaries/phoebus_boundaries.cpp @@ -33,6 +33,8 @@ using namespace parthenon::package::prelude; namespace Boundaries { +void SwarmNoWorkBC(std::shared_ptr &swarm) {} + parthenon::TopologicalElement CC = parthenon::TopologicalElement::CC; // Copied out of Parthenon, with slight modification @@ -84,8 +86,9 @@ void GenericBC(std::shared_ptr> &rc, bool coarse) { const bool rescale = fluid->Param("bc_vars") == "conserved"; // Do the thing + const bool fine = false; // NOTE(BLB): will need changing for fine fields pmb->par_for_bndry( - label, nb, domain, CC, coarse, + label, nb, domain, CC, coarse, fine, KOKKOS_LAMBDA(const int &l, const int &k, const int &j, const int &i) { int kref, jref, iref, sgn; if (TYPE == BCType::Reflect) { @@ -128,9 +131,10 @@ void OutflowInnerX1(std::shared_ptr> &rc, bool coarse) { auto &fluid = rc->GetMeshPointer()->packages.Get("fluid"); std::string bc_vars = fluid->Param("bc_vars"); + const bool fine = false; if (bc_vars == "conserved") { pmb->par_for_bndry( - "OutflowInnerX1Cons", nb, domain, CC, coarse, + "OutflowInnerX1Cons", nb, domain, CC, coarse, fine, KOKKOS_LAMBDA(const int &l, const int &k, const int &j, const int &i) { Real detg_ref = geom.DetGamma(CellLocation::Cent, k, j, ref); Real detg = geom.DetGamma(CellLocation::Cent, k, j, i); @@ -139,7 +143,7 @@ void OutflowInnerX1(std::shared_ptr> &rc, bool coarse) { }); } else if (bc_vars == "primitive") { pmb->par_for_bndry( - "OutflowInnerX1Prim", nb, domain, CC, coarse, + "OutflowInnerX1Prim", nb, domain, CC, coarse, fine, KOKKOS_LAMBDA(const int &l, const int &k, const int &j, const int &i) { q(l, k, j, i) = q(l, k, j, ref); }); @@ -165,8 +169,9 @@ void PolarInnerX2(std::shared_ptr> &rc, bool coarse) { const auto idx_pvel = imap.GetFlatIdx(fluid_prim::velocity::name(), false); const auto idx_pb = imap.GetFlatIdx(fluid_prim::bfield::name(), false); + const bool fine = false; pmb->par_for_bndry( - "PolarInnerX2Prim", nb, domain, CC, coarse, + "PolarInnerX2Prim", nb, domain, CC, coarse, fine, KOKKOS_LAMBDA(const int &l, const int &k, const int &j, const int &i) { const int jref = -j + 2 * j0 - 1; if (l == idx_pvel(1)) { @@ -199,8 +204,9 @@ void PolarOuterX2(std::shared_ptr> &rc, bool coarse) { const auto idx_pb = imap.GetFlatIdx(fluid_prim::bfield::name(), false); const std::string label = "PolarOuterX2Prim"; + const bool fine = false; pmb->par_for_bndry( - label, nb, domain, CC, coarse, + label, nb, domain, CC, coarse, fine, KOKKOS_LAMBDA(const int &l, const int &k, const int &j, const int &i) { const int jref = -j + 2 * (j0 + 1) - 1; if (l == idx_pvel(1)) { @@ -235,9 +241,10 @@ void OutflowOuterX1(std::shared_ptr> &rc, bool coarse) { std::string bc_vars = fluid->Param("bc_vars"); const int num_species = rad->Param("active") ? rad->Param("num_species") : 0; + const bool fine = false; if (bc_vars == "conserved") { pmb->par_for_bndry( - "OutflowOuterX1Cons", nb, IndexDomain::outer_x1, CC, coarse, + "OutflowOuterX1Cons", nb, IndexDomain::outer_x1, CC, coarse, fine, KOKKOS_LAMBDA(const int &l, const int &k, const int &j, const int &i) { Real detg_ref = geom.DetGamma(CellLocation::Cent, k, j, ref); Real detg = geom.DetGamma(CellLocation::Cent, k, j, i); @@ -246,7 +253,7 @@ void OutflowOuterX1(std::shared_ptr> &rc, bool coarse) { }); } else if (bc_vars == "primitive") { pmb->par_for_bndry( - "OutflowOuterX1Prim", nb, domain, CC, coarse, + "OutflowOuterX1Prim", nb, domain, CC, coarse, fine, KOKKOS_LAMBDA(const int &l, const int &k, const int &j, const int &i) { q(l, k, j, i) = q(l, k, j, ref); }); @@ -335,6 +342,9 @@ TaskStatus ConvertBoundaryConditions(std::shared_ptr> &rc) { // TODO(BRR) Is this always true? const bool coarse = false; + // NOTE(BLB): setting fine to false. Will require refactoring + // if we use fine fields in the future, ala parthenon/pull/991 + const bool fine = false; PackIndexMap imap; std::vector vars{fluid_prim::velocity::name(), @@ -348,7 +358,7 @@ TaskStatus ConvertBoundaryConditions(std::shared_ptr> &rc) { const int num_species = pkg_rad->Param("active") ? pkg_rad->Param("num_species") : 0; pmb->par_for_bndry( - "OutflowOuterX1PrimFixup", nb1, domain, CC, coarse, + "OutflowOuterX1PrimFixup", nb1, domain, CC, coarse, fine, KOKKOS_LAMBDA(const int &dummy, const int &k, const int &j, const int &i) { // Enforce u^1 >= 0 Real vcon[3] = {q(pv_lo, k, j, i), q(pv_lo + 1, k, j, i), @@ -460,20 +470,23 @@ void ProcessBoundaryConditions(parthenon::ParthenonManager &pman) { const std::string rad_method = pman.pinput->GetOrAddString("radiation", "method", "None"); + // TODO: logic for gr outflow et/ if (typeid(PHOEBUS_GEOMETRY) == typeid(Geometry::FMKS)) { bool derefine_poles = pman.pinput->GetOrAddBoolean("coordinates", "derefine_poles", true); if (derefine_poles) { const std::string ix2_bc = - pman.pinput->GetOrAddString("phoebus", "ix2_bc", "outflow"); + pman.pinput->GetOrAddString("phoebus", "ix2_bc", "gr_outflow"); const std::string ox2_bc = - pman.pinput->GetOrAddString("phoebus", "ox2_bc", "outflow"); + pman.pinput->GetOrAddString("phoebus", "ox2_bc", "gr_outflow"); PARTHENON_REQUIRE( ix2_bc != "reflect" && ix2_bc != "polar" && ox2_bc != "reflect" && ox2_bc != "polar", "Polar and Reflecting X2 BCs not supported for \"derefine_poles = true\"!"); } } + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::inner_x3, "swarm_no_bc", &Boundaries::SwarmNoWorkBC); for (int d = 1; d <= 3; ++d) { // outer = 0 for inner face, outer = 1 for outer face @@ -481,66 +494,51 @@ void ProcessBoundaryConditions(parthenon::ParthenonManager &pman) { auto &face = inner_outer[outer]; const std::string name = face + "x" + std::to_string(d) + "_bc"; const std::string parth_bc = pman.pinput->GetString("parthenon/mesh", name); - PARTHENON_REQUIRE(parth_bc == "user" || parth_bc == "periodic", - "Only \"user\" and \"periodic\" allowed for parthenon/mesh/" + - name); - - const std::string bc = pman.pinput->GetOrAddString("phoebus", name, "outflow"); - if (bc == "reflect") { - pman.app_input->boundary_conditions[loc[d - 1][outer]] = reflect[d - 1][outer]; - } else if (bc == "polar") { - PARTHENON_REQUIRE(d == 2, "Polar boundary conditions only supported in X2!"); - pman.app_input->boundary_conditions[loc[d - 1][outer]] = polar[d - 1][outer]; - } else if (bc == "outflow") { - pman.app_input->boundary_conditions[loc[d - 1][outer]] = outflow[d - 1][outer]; - if (d == 1) { - if (outer == 0) { - if (rad_method == "mocmc") { - pman.app_input - ->swarm_boundary_conditions[parthenon::BoundaryFace::inner_x1] = - Boundaries::SetSwarmNoWorkBC; - } else { - pman.app_input - ->swarm_boundary_conditions[parthenon::BoundaryFace::inner_x1] = - Boundaries::SetSwarmIX1Outflow; - } - } else if (outer == 1) { - if (rad_method == "mocmc") { - pman.app_input - ->swarm_boundary_conditions[parthenon::BoundaryFace::outer_x1] = - Boundaries::SetSwarmNoWorkBC; - } else { - pman.app_input - ->swarm_boundary_conditions[parthenon::BoundaryFace::outer_x1] = - Boundaries::SetSwarmOX1Outflow; - } - } + + const std::string bc = pman.pinput->GetOrAddString("phoebus", name, "gr_outflow"); + pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "reflect", + reflect[d - 1][outer]); + if (d == 2) { // polar BC only supported for X2 + pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "polar", + polar[d - 1][outer]); + } + pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "gr_outflow", + outflow[d - 1][outer]); + if (d == 1) { + if (outer == 0) { + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::inner_x1, "swarm_no_bc", + &Boundaries::SwarmNoWorkBC); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::inner_x1, "swarm_outflow", + &parthenon::BoundaryFunction::SwarmOutflowInnerX1); + } else if (outer == 1) { + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::outer_x1, "swarm_no_bc", + &Boundaries::SwarmNoWorkBC); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::outer_x1, "swarm_outflow", + &parthenon::BoundaryFunction::SwarmOutflowOuterX1); } - if (d == 2) { - if (outer == 0) { - if (rad_method == "mocmc") { - pman.app_input - ->swarm_boundary_conditions[parthenon::BoundaryFace::inner_x2] = - Boundaries::SetSwarmNoWorkBC; - } else { - pman.app_input - ->swarm_boundary_conditions[parthenon::BoundaryFace::inner_x2] = - Boundaries::SetSwarmIX2Outflow; - } - } else if (outer == 1) { - if (rad_method == "mocmc") { - pman.app_input - ->swarm_boundary_conditions[parthenon::BoundaryFace::outer_x2] = - Boundaries::SetSwarmNoWorkBC; - } else { - pman.app_input - ->swarm_boundary_conditions[parthenon::BoundaryFace::outer_x2] = - Boundaries::SetSwarmOX2Outflow; - } - } + } + if (d == 2) { + if (outer == 0) { + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::inner_x2, "swarm_no_bc", + &Boundaries::SwarmNoWorkBC); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::inner_x2, "swarm_outflow", + &parthenon::BoundaryFunction::SwarmOutflowInnerX1); + } else if (outer == 1) { + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::outer_x2, "swarm_no_bc", + &Boundaries::SwarmNoWorkBC); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::outer_x2, "swarm_outflow", + &parthenon::BoundaryFunction::SwarmOutflowOuterX2); } - } // periodic boundaries, which are handled by parthenon, so no need to set anything - } + } + } // periodic boundaries, which are handled by parthenon, so no need to set anything } } diff --git a/src/phoebus_boundaries/phoebus_boundaries.hpp b/src/phoebus_boundaries/phoebus_boundaries.hpp index f6d69c961..dbcf72e77 100644 --- a/src/phoebus_boundaries/phoebus_boundaries.hpp +++ b/src/phoebus_boundaries/phoebus_boundaries.hpp @@ -18,11 +18,14 @@ #include +#include +#include #include #include #include #include using namespace parthenon::package::prelude; +using namespace parthenon::BoundaryFunction; namespace Boundaries { @@ -46,28 +49,20 @@ void OutflowOuterX3(std::shared_ptr> &rc, bool coarse); void ReflectInnerX3(std::shared_ptr> &rc, bool coarse); void ReflectOuterX3(std::shared_ptr> &rc, bool coarse); -class ParticleBoundNoWork : public parthenon::ParticleBound { - public: - KOKKOS_INLINE_FUNCTION void - Apply(const int n, double &x, double &y, double &z, - const parthenon::SwarmDeviceContext &swarm_d) const override {} -}; - -inline auto SetSwarmIX1Outflow() { - return parthenon::DeviceAllocate(); -} -inline auto SetSwarmOX1Outflow() { - return parthenon::DeviceAllocate(); -} -inline auto SetSwarmIX2Outflow() { - return parthenon::DeviceAllocate(); -} -inline auto SetSwarmOX2Outflow() { - return parthenon::DeviceAllocate(); -} -inline auto SetSwarmNoWorkBC() { - return parthenon::DeviceAllocate(); -} +void SwarmNoWorkBC(std::shared_ptr &swarm); + +// inline auto SetSwarmIX1Outflow() { +// return parthenon::BoundaryFunction::OutflowInnerX1; +// } +// inline auto SetSwarmOX1Outflow() { +// return parthenon::BoundaryFunction::OutflowOuterX1; +// } +// inline auto SetSwarmIX2Outflow() { +// return parthenon::BoundaryFunction::OutflowInnerX2; +// } +// inline auto SetSwarmOX2Outflow() { +// return parthenon::BoundaryFunction::OutflowOuterX2; +// } TaskStatus ConvertBoundaryConditions(std::shared_ptr> &rc); diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 998c9679f..743032f58 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -50,6 +50,37 @@ using namespace Geometry; namespace phoebus { +TaskStatus MeshResetCommunication(MeshData *md) { + for (const auto &mbd : md->GetAllBlockData()) { + auto &sc = mbd->GetSwarmData(); + sc->ResetCommunication(); + } + + return TaskStatus::complete; +} + +TaskStatus MeshSend(MeshData *md) { + for (const auto &mbd : md->GetAllBlockData()) { + auto &sc = mbd->GetSwarmData(); + sc->Send(BoundaryCommSubset::all); + } + + return TaskStatus::complete; +} + +TaskStatus MeshReceive(MeshData *md) { + TaskStatus status = TaskStatus::complete; + for (const auto &mbd : md->GetAllBlockData()) { + auto &sc = mbd->GetSwarmData(); + auto local_status = sc->Receive(BoundaryCommSubset::all); + if (local_status == TaskStatus::incomplete) { + status = TaskStatus::incomplete; + } + } + + return status; +} + // *************************************************// // define the application driver. in this case, *// // that mostly means defining the MakeTaskList *// @@ -282,6 +313,7 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { } // Goal: make async regions go away + // TODO(BLB): move moments code to packs / MeshData TaskRegion &async_region_1 = tc.AddRegion(num_independent_task_lists); for (int ib = 0; ib < num_independent_task_lists; ib++) { auto pmb = blocks[ib].get(); @@ -320,24 +352,32 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { auto fix_flux = tl.AddTask(sndrcv_flux_depend, fixup::FixFluxes, sc0.get()); sndrcv_flux_depend = sndrcv_flux_depend | fix_flux; } + } - if (rad_mocmc_active) { - using MDT = std::remove_pointer::type; + if (rad_mocmc_active) { + TaskRegion &sync_region_mocmc = tc.AddRegion(num_partitions); + for (int n = 0; n < num_partitions; n++) { + auto &tl = sync_region_mocmc[n]; + + auto &base = pmesh->mesh_data.GetOrAdd("base", n); + using MDT = std::remove_pointer::type; // TODO(BRR) stage_name[stage - 1]? - auto &sd0 = pmb->swarm_data.Get(stage_name[integrator->nstages]); + auto reset_comms = tl.AddTask(none, MeshResetCommunication, base.get()); auto samples_transport = - tl.AddTask(none, radiation::MOCMCTransport, sc0.get(), dt); - auto send = tl.AddTask(samples_transport, &SwarmContainer::Send, sd0.get(), - BoundaryCommSubset::all); - auto receive = - tl.AddTask(send, &SwarmContainer::Receive, sd0.get(), BoundaryCommSubset::all); + tl.AddTask(reset_comms, radiation::MOCMCTransport, base.get(), beta * dt); + auto send = tl.AddTask(samples_transport, MeshSend, base.get()); + auto receive = tl.AddTask(samples_transport | send, MeshReceive, base.get()); + + auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], n); + // TODO(BRR) stage_name[stage - 1]? auto sample_bounds = - tl.AddTask(receive, radiation::MOCMCSampleBoundaries, sc0.get()); - auto sample_recon = - tl.AddTask(sample_bounds, radiation::MOCMCReconstruction, sc0.get()); + tl.AddTask(none, radiation::MOCMCSampleBoundaries, base.get(), sc1.get()); + auto sample_recon = tl.AddTask(sample_bounds, radiation::MOCMCReconstruction, + base.get(), sc1.get()); auto eddington = - tl.AddTask(sample_recon, radiation::MOCMCEddington, sc0.get()); + tl.AddTask(sample_recon, radiation::MOCMCEddington, base.get(), sc1.get()); + TaskID geom_src(0); geom_src = geom_src | eddington; } } @@ -524,43 +564,37 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { } // Fix up flux failures - TaskRegion &async_region_2 = tc.AddRegion(num_independent_task_lists); - for (int ib = 0; ib < num_independent_task_lists; ib++) { - auto pmb = blocks[ib].get(); - auto &tl = async_region_2[ib]; - - // first make other useful containers - auto &base = pmb->meshblock_data.Get(); + TaskRegion &sync_region_fixup = tc.AddRegion(num_partitions); + for (int ib = 0; ib < num_partitions; ib++) { + auto &tl = sync_region_fixup[ib]; // pull out the container we'll use to get fluxes and/or compute RHSs - auto &sc1 = pmb->meshblock_data.Get(stage_name[stage]); - using MDT = std::remove_pointer::type; + auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], ib); // fill in derived fields auto fill_derived = - tl.AddTask(none, parthenon::Update::FillDerived>, sc1.get()); + tl.AddTask(none, parthenon::Update::FillDerived>, sc1.get()); - auto fixup = tl.AddTask( - fill_derived, fixup::ConservedToPrimitiveFixup>, sc1.get()); + auto fixup = tl.AddTask(fill_derived, + fixup::ConservedToPrimitiveFixup>, sc1.get()); - auto radfixup = tl.AddTask( - fixup, fixup::RadConservedToPrimitiveFixup>, sc1.get()); + auto radfixup = + tl.AddTask(fixup, fixup::RadConservedToPrimitiveFixup>, sc1.get()); - auto floors = - tl.AddTask(radfixup, fixup::ApplyFloors>, sc1.get()); + auto floors = tl.AddTask(radfixup, fixup::ApplyFloors>, sc1.get()); + auto &base = pmesh->mesh_data.GetOrAdd("base", ib); + using MDT = std::remove_pointer::type; TaskID gas_rad_int(0); if (rad_mocmc_active) { - auto impl_update = - tl.AddTask(floors, radiation::MOCMCFluidSource>, sc1.get(), - beta * dt, fluid_active); - auto impl_edd = tl.AddTask( - impl_update, radiation::MOCMCEddington>, sc1.get()); + auto impl_update = tl.AddTask(floors, radiation::MOCMCFluidSource, base.get(), + sc1.get(), beta * dt, fluid_active); + auto impl_edd = + tl.AddTask(impl_update, radiation::MOCMCEddington, base.get(), sc1.get()); gas_rad_int = gas_rad_int | impl_edd; } else if (rad_moments_active) { - auto impl_update = - tl.AddTask(floors, radiation::MomentFluidSource>, sc1.get(), - beta * dt, fluid_active); + auto impl_update = tl.AddTask(floors, radiation::MomentFluidSource, sc1.get(), + beta * dt, fluid_active); gas_rad_int = gas_rad_int | impl_update; } @@ -568,7 +602,7 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { // Only apply floors because MomentFluidSource already ensured that a sensible state // was returned auto floors = - tl.AddTask(gas_rad_int, fixup::ApplyFloors>, sc1.get()); + tl.AddTask(gas_rad_int, fixup::ApplyFloors>, sc1.get()); } } @@ -580,27 +614,35 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { for (int i = 0; i < blocks.size(); i++) { auto &tl = sync_region_tr[0]; auto &pmb = blocks[i]; - auto &sc = pmb->swarm_data.Get(); + auto &sc = pmb->meshblock_data.Get()->GetSwarmData(); auto reset_comms = tl.AddTask(none, &SwarmContainer::ResetCommunication, sc.get()); } } - TaskRegion &async_region_tr = tc.AddRegion(blocks.size()); - for (int n = 0; n < blocks.size(); n++) { + TaskRegion &async_region_tr = tc.AddRegion(num_partitions); + for (int n = 0; n < num_partitions; n++) { auto &tl = async_region_tr[n]; auto &pmb = blocks[n]; - auto &sc = pmb->swarm_data.Get(); + auto &base = pmesh->mesh_data.GetOrAdd("base", n); auto &mbd0 = pmb->meshblock_data.Get(stage_name[stage]); - auto tracerAdvect = tl.AddTask(none, tracers::AdvectTracers, mbd0.get(), dt); + + auto tracerAdvect = tl.AddTask(none, tracers::AdvectTracers, base.get(), dt); auto tracerPurge = tl.AddTask(tracerAdvect, fixup::PurgeParticles, mbd0.get(), swarmName); + } - auto send = tl.AddTask(tracerPurge, &SwarmContainer::Send, sc.get(), - BoundaryCommSubset::all); + TaskRegion &async_region_tr_comm = tc.AddRegion(blocks.size()); + for (int n = 0; n < blocks.size(); n++) { + auto &tl = async_region_tr_comm[n]; + auto &pmb = blocks[n]; + auto &sc = pmb->meshblock_data.Get()->GetSwarmData(); + auto send = + tl.AddTask(none, &SwarmContainer::Send, sc.get(), BoundaryCommSubset::all); auto receive = tl.AddTask(send, &SwarmContainer::Receive, sc.get(), BoundaryCommSubset::all); + auto defrag = tl.AddTask(receive, &SwarmContainer::Defrag, sc.get(), 0.9); } } @@ -932,7 +974,7 @@ TaskListStatus PhoebusDriver::MonteCarloStep() { auto &pmb = blocks[i]; auto &tl = async_region0[i]; auto &mbd0 = pmb->meshblock_data.Get(stage_name[integrator->nstages]); - auto &sc0 = pmb->swarm_data.Get(stage_name[integrator->nstages]); + auto &sc0 = pmb->meshblock_data.Get()->GetSwarmData(); auto sample_particles = tl.AddTask(none, radiation::MonteCarloSourceParticles, pmb.get(), mbd0.get(), sc0.get(), t0, dt); auto transport_particles = @@ -1023,7 +1065,8 @@ TaskListStatus PhoebusDriver::MonteCarloStep() { * Fills Tracers * Computes entropy for output **/ -void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { +void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin, + const parthenon::SimTime &time) { auto tracer_pkg = pmb->packages.Get("tracers"); bool do_tracers = tracer_pkg->Param("active"); diff --git a/src/phoebus_driver.hpp b/src/phoebus_driver.hpp index 921e06905..65a358556 100644 --- a/src/phoebus_driver.hpp +++ b/src/phoebus_driver.hpp @@ -44,7 +44,8 @@ class PhoebusDriver : public EvolutionDriver { }; parthenon::Packages_t ProcessPackages(std::unique_ptr &pin); -void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin); +void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin, + const parthenon::SimTime &time); } // namespace phoebus diff --git a/src/phoebus_utils/variables.hpp b/src/phoebus_utils/variables.hpp index 2c127ea2e..0a71b9a9e 100644 --- a/src/phoebus_utils/variables.hpp +++ b/src/phoebus_utils/variables.hpp @@ -14,6 +14,8 @@ #ifndef PHOEBUS_UTILS_VARIABLES_HPP_ #define PHOEBUS_UTILS_VARIABLES_HPP_ +#include "compile_constants.hpp" + #include #include @@ -41,6 +43,26 @@ static std::string name() { return #varstring; } \ } +#define TENSOR_SWARM(type, ns, varname, ...) \ + struct varname : public parthenon::swarm_variable_names::base_t { \ + template \ + KOKKOS_INLINE_FUNCTION varname(Ts &&...args) \ + : parthenon::swarm_variable_names::base_t( \ + std::forward(args)...) {} \ + static std::string name() { return #ns "." #varname; } \ + } + +#define TENSOR_VARIABLE(ns, varname, ...) \ + struct varname : public parthenon::variable_names::base_t { \ + template \ + KOKKOS_INLINE_FUNCTION varname(Ts &&...args) \ + : parthenon::variable_names::base_t( \ + std::forward(args)...) {} \ + static std::string name() { return #ns "." #varname; } \ + } + +using parthenon::variable_names::ANYDIM; + namespace fluid_prim { VARIABLE(p, density); VARIABLE(p, velocity); @@ -64,26 +86,26 @@ VARIABLE(c, ye); namespace radmoment_prim { VARIABLE(r.p, J); -VARIABLE(r.p, H); +TENSOR_VARIABLE(r.p, H, ANYDIM, 3); // (num_species, 3) } // namespace radmoment_prim namespace radmoment_cons { VARIABLE(r.c, E); -VARIABLE(r.c, F); +TENSOR_VARIABLE(r.c, F, ANYDIM, 3); // (num_species, 3) } // namespace radmoment_cons namespace radmoment_internal { VARIABLE(r.i, xi); VARIABLE(r.i, phi); -VARIABLE(r.i, ql); -VARIABLE(r.i, qr); +TENSOR_VARIABLE(r.i, ql, ANYDIM, 3); // (num_species, nrecon) +TENSOR_VARIABLE(r.i, qr, ANYDIM, 3); // (num_species, nrecon) VARIABLE(r.i, ql_v); VARIABLE(r.i, qr_v); VARIABLE(r.i, dJ); VARIABLE(r.i, kappaJ); VARIABLE(r.i, kappaH); VARIABLE(r.i, JBB); -VARIABLE(r.i, tilPi); +TENSOR_VARIABLE(r.i, tilPi, PHOEBUS_NUM_SPECIES, 3, 3); VARIABLE(r.i, kappaH_mean); VARIABLE(r.i, c2pfail); VARIABLE(r.i, srcfail); @@ -96,6 +118,16 @@ VARIABLE(mocmc.i, Inu1); VARIABLE(mocmc.i, jinvs); } // namespace mocmc_internal +namespace mocmc_core { +SWARM_VARIABLE(Real, mocmc.c, t); +SWARM_VARIABLE(Real, mocmc.c, mu_lo); +SWARM_VARIABLE(Real, mocmc.c, mu_hi); +SWARM_VARIABLE(Real, mocmc.c, phi_lo); +SWARM_VARIABLE(Real, mocmc.c, phi_hi); +SWARM_VARIABLE(Real, mocmc.c, ncov); +TENSOR_SWARM(Real, mocmc.c, Inuinv, ANYDIM, PHOEBUS_NUM_SPECIES); +} // namespace mocmc_core + namespace internal_variables { VARIABLE_NONS(face_signal_speed); VARIABLE_NONS(cell_signal_speed); @@ -118,6 +150,48 @@ VARIABLE_CUSTOM(cell_coords, g.c.coord); VARIABLE_CUSTOM(node_coords, g.n.coord); } // namespace geometric_variables +namespace monte_carlo_internal { +VARIABLE(monte_carlo.i, dNdlnu_max); +VARIABLE(monte_carlo.i, dN); +VARIABLE(monte_carlo.i, Ns); +TENSOR_VARIABLE(monte_carlo.i, dNdlnu, ANYDIM, PHOEBUS_NUM_SPECIES); +} // namespace monte_carlo_internal + +// TODO(BLB) Can probably move k0..k3 to k(mu) +namespace monte_carlo_core { +SWARM_VARIABLE(Real, monte_carlo.c, t); +SWARM_VARIABLE(Real, monte_carlo.c, k0); +SWARM_VARIABLE(Real, monte_carlo.c, k1); +SWARM_VARIABLE(Real, monte_carlo.c, k2); +SWARM_VARIABLE(Real, monte_carlo.c, k3); +SWARM_VARIABLE(Real, monte_carlo.c, weight); +SWARM_VARIABLE(int, monte_carlo.c, species); +} // namespace monte_carlo_core + +namespace tracer_variables { +SWARM_VARIABLE(Real, tr, rho); +SWARM_VARIABLE(Real, tr, temperature); +SWARM_VARIABLE(Real, tr, ye); +SWARM_VARIABLE(Real, tr, entropy); +SWARM_VARIABLE(Real, tr, pressure); +SWARM_VARIABLE(Real, tr, energy); +SWARM_VARIABLE(Real, tr, vel_x); +SWARM_VARIABLE(Real, tr, vel_y); +SWARM_VARIABLE(Real, tr, vel_z); +SWARM_VARIABLE(Real, tr, lorentz); +SWARM_VARIABLE(Real, tr, lapse); +SWARM_VARIABLE(Real, tr, detgamma); +SWARM_VARIABLE(Real, tr, shift_x); +SWARM_VARIABLE(Real, tr, shift_y); +SWARM_VARIABLE(Real, tr, shift_z); +SWARM_VARIABLE(Real, tr, mass); +SWARM_VARIABLE(Real, tr, bernoulli); +SWARM_VARIABLE(Real, tr, B_x); +SWARM_VARIABLE(Real, tr, B_y); +SWARM_VARIABLE(Real, tr, B_z); +TENSOR_SWARM(Real, tr, test, ANYDIM, PHOEBUS_NUM_SPECIES); +} // namespace tracer_variables + namespace diagnostic_variables { VARIABLE_NONS(divb); VARIABLE_NONS(ratio_divv_cs); diff --git a/src/radiation/geodesics.hpp b/src/radiation/geodesics.hpp index fcfda22f2..6996e1931 100644 --- a/src/radiation/geodesics.hpp +++ b/src/radiation/geodesics.hpp @@ -35,17 +35,51 @@ void GetXSource(Real &Kcon0, Real &Kcon1, Real &Kcon2, Real &Kcon3, Real src[NDF src[3] = Kcon3 / Kcon0; } -KOKKOS_INLINE_FUNCTION -void GetKSource(Real &X0, Real &X1, Real &X2, Real &X3, Real &Kcov0, Real &Kcov1, - Real &Kcov2, Real &Kcov3, Real &Kcon0, - const Geometry::CoordSysMeshBlock &geom, Real source[4]) { - SPACETIMELOOP(mu) { source[mu] = 0.; } +template +KOKKOS_INLINE_FUNCTION void GetKSource(Real &X0, Real &X1, Real &X2, Real &X3, + Real &Kcov0, Real &Kcov1, Real &Kcov2, Real &Kcov3, + Real &Kcon0, Geom &geom, Real source[4]) { + // TODO(BLB): include options for more geometries to exploit spacetime symmetries + if constexpr (std::is_same::value) { // flat space + SPACETIMELOOP(mu) { source[mu] = 0.0; } + } else { // generic + // TODO(BLB): hook into metric machinery instead of doing it here + Real Kcov[NDFULL] = {Kcov0, Kcov1, Kcov2, Kcov3}; + Real Xm[NDFULL], Xp[NDFULL]; + Real Gconm[NDFULL][NDFULL], Gconp[NDFULL][NDFULL]; + Real dG; + + constexpr static Real DELTA = 1.0e-6; + + SPACETIMELOOP(mu) { + source[mu] = 0.; + Xp[0] = X0; + Xp[1] = X1; + Xp[2] = X2; + Xp[3] = X3; + SPACETIMELOOP(nu) { Xm[nu] = Xp[nu]; } + + Xm[mu] -= DELTA; + Xp[mu] += DELTA; + + geom.SpacetimeMetricInverse(Xm[0], Xm[1], Xm[2], Xm[3], Gconm); + geom.SpacetimeMetricInverse(Xp[0], Xp[1], Xp[2], Xp[3], Gconp); + + SPACETIMELOOP2(nu, kap) { + dG = (Gconp[nu][kap] - Gconm[nu][kap]) / (Xp[mu] - Xm[mu]); + source[mu] += Kcov[nu] * Kcov[kap] * dG; + } + + source[mu] *= -1.0 / (2.0 * Kcon0); + } + } } -KOKKOS_INLINE_FUNCTION -void PushParticle(Real &X0, Real &X1, Real &X2, Real &X3, Real &Kcov0, Real &Kcov1, - Real &Kcov2, Real &Kcov3, const Real &dt, - const Geometry::CoordSysMeshBlock &geom) { +template +KOKKOS_INLINE_FUNCTION void PushParticle(Real &X0, Real &X1, Real &X2, Real &X3, + Real &Kcov0, Real &Kcov1, Real &Kcov2, + Real &Kcov3, const Real &dt, Geom &geom) { Real c1[NDFULL], c2[NDFULL], d1[NDFULL], d2[NDFULL]; Real Xtmp[NDFULL], Kcontmp[NDFULL], Kcovtmp[NDFULL]; Real Kcov[NDFULL] = {Kcov0, Kcov1, Kcov2, Kcov3}; diff --git a/src/radiation/mocmc.cpp b/src/radiation/mocmc.cpp index 509a24151..51fb36ffe 100644 --- a/src/radiation/mocmc.cpp +++ b/src/radiation/mocmc.cpp @@ -55,7 +55,7 @@ template void MOCMCInitSamples(T *rc) { auto *pmb = rc->GetParentPointer(); - auto &sc = pmb->swarm_data.Get(); + auto &sc = rc->GetSwarmData(); auto &swarm = sc->Get("mocmc"); StateDescriptor *rad = pmb->packages.Get("radiation").get(); auto rng_pool = rad->Param("rng_pool"); @@ -160,11 +160,11 @@ void MOCMCInitSamples(T *rc) { // }, // result); - const auto &x = swarm->template Get("x").Get(); - const auto &y = swarm->template Get("y").Get(); - const auto &z = swarm->template Get("z").Get(); - const auto &ncov = swarm->template Get("ncov").Get(); - const auto &Inuinv = swarm->template Get("Inuinv").Get(); + const auto &x = swarm->template Get(swarm_position::x::name()).Get(); + const auto &y = swarm->template Get(swarm_position::y::name()).Get(); + const auto &z = swarm->template Get(swarm_position::z::name()).Get(); + const auto &ncov = swarm->template Get(mocmc_core::ncov::name()).Get(); + const auto &Inuinv = swarm->template Get(mocmc_core::Inuinv::name()).Get(); auto swarm_d = swarm->GetDeviceContext(); @@ -225,22 +225,20 @@ void MOCMCInitSamples(T *rc) { }); // Initialize eddington tensor and opacities for first step - MOCMCReconstruction(rc); - MOCMCEddington(rc); - MOCMCFluidSource(rc, 0., false); // Update opacities for asymptotic fluxes + MOCMCReconstruction(rc, rc); + MOCMCEddington(rc, rc); + MOCMCFluidSource(rc, rc, 0., false); // Update opacities for asymptotic fluxes } template -TaskStatus MOCMCSampleBoundaries(T *rc) { +TaskStatus MOCMCSampleBoundaries(T *rc_base, T *rc) { auto *pmb = rc->GetParentPointer(); - auto &sc = pmb->swarm_data.Get(); - auto &swarm = sc->Get("mocmc"); StateDescriptor *rad = pmb->packages.Get("radiation").get(); const auto geom = Geometry::GetCoordinateSystem(rc); - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + IndexRange ib = rc->GetBoundsI(IndexDomain::interior); + IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); + IndexRange kb = rc->GetBoundsK(IndexDomain::interior); std::vector variables{pr::J::name(), pf::velocity::name(), ir::tilPi::name()}; @@ -251,15 +249,14 @@ TaskStatus MOCMCSampleBoundaries(T *rc) { auto opac_pkg = pmb->packages.Get("opacity"); const auto opac = opac_pkg->template Param("opacities"); - const auto &x = swarm->template Get("x").Get(); - const auto &y = swarm->template Get("y").Get(); - const auto &z = swarm->template Get("z").Get(); - const auto &ncov = swarm->template Get("ncov").Get(); - const auto &Inuinv = swarm->template Get("Inuinv").Get(); - const auto &mu_lo = swarm->template Get("mu_lo").Get(); - const auto &mu_hi = swarm->template Get("mu_hi").Get(); - const auto &phi_lo = swarm->template Get("phi_lo").Get(); - const auto &phi_hi = swarm->template Get("phi_hi").Get(); + static constexpr auto swarm_name = "mocmc"; + static const auto desc_mocmc = + MakeSwarmPackDescriptor( + swarm_name); + auto pack_mocmc = desc_mocmc.GetPack(rc_base); + auto pv = imap.GetFlatIdx(pf::velocity::name()); auto iTilPi = imap.GetFlatIdx(ir::tilPi::name()); auto iJ = imap.GetFlatIdx(pr::J::name()); @@ -273,8 +270,6 @@ TaskStatus MOCMCSampleBoundaries(T *rc) { species_d[s] = species[s]; } - auto swarm_d = swarm->GetDeviceContext(); - // TODO(BRR) provide *all* MOCMC boundaries auto ix1_bc = rad->Param("ix1_bc"); @@ -289,26 +284,33 @@ TaskStatus MOCMCSampleBoundaries(T *rc) { // ox1_temp = rad->Param("ox1_temp"); //} - pmb->par_for( - "Temporary MOCMC boundaries", 0, swarm->GetMaxActiveIndex(), - KOKKOS_LAMBDA(const int n) { + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "MOCMC::SampleBoundaries", DevExecSpace(), 0, + pack_mocmc.GetMaxFlatIndex(), KOKKOS_LAMBDA(const int idx) { + const auto [b, n] = pack_mocmc.GetBlockParticleIndices(idx); + const auto swarm_d = pack_mocmc.GetContext(b); if (swarm_d.IsActive(n)) { + Real &x = pack_mocmc(b, swarm_position::x(), n); + Real &y = pack_mocmc(b, swarm_position::y(), n); + Real &z = pack_mocmc(b, swarm_position::z(), n); // Store zone before reflections int i, j, k; - swarm_d.Xtoijk(x(n), y(n), z(n), i, j, k); + swarm_d.Xtoijk(x, y, z, i, j, k); - if (x(n) < swarm_d.x_min_global_) { + if (x < swarm_d.x_min_global_) { // Reflect particle across boundary - x(n) = swarm_d.x_min_global_ + (swarm_d.x_min_global_ - x(n)); + x = swarm_d.x_min_global_ + (swarm_d.x_min_global_ - x); // TODO(BRR) normalized in relativity? - ncov(1, n) = -ncov(1, n); + pack_mocmc(b, mocmc_core::ncov(1), n) = + -pack_mocmc(b, mocmc_core::ncov(1), n); for (int s = 0; s < num_species; s++) { Real temp = 0.; if (ix1_bc == MOCMCBoundaries::outflow) { // Temperature from J in ghost zone - temp = opac.TemperatureFromEnergyDensity(v(iJ(s), k, j, i), species_d[s]); + temp = + opac.TemperatureFromEnergyDensity(v(b, iJ(s), k, j, i), species_d[s]); } else { // Fixed temperature temp = ix1_temp; @@ -317,7 +319,7 @@ TaskStatus MOCMCSampleBoundaries(T *rc) { // Reset intensities for (int nubin = 0; nubin < nu_bins; nubin++) { const Real nu = nusamp(nubin); - Inuinv(nubin, s, n) = + pack_mocmc(b, mocmc_core::Inuinv(nubin, s), n) = std::max(robust::SMALL(), opac.ThermalDistributionOfTNu( temp, species_d[s], nu)) / std::pow(nu, 3); @@ -325,21 +327,22 @@ TaskStatus MOCMCSampleBoundaries(T *rc) { } } - if (x(n) > swarm_d.x_max_global_) { + if (x > swarm_d.x_max_global_) { // Reflect particle across boundary - x(n) = swarm_d.x_max_global_ - (x(n) - swarm_d.x_max_global_); - ncov(1, n) = -ncov(1, n); + x = swarm_d.x_max_global_ - (x - swarm_d.x_max_global_); + pack_mocmc(b, mocmc_core::ncov(1), n) = + -pack_mocmc(b, mocmc_core::ncov(1), n); // Reset intensities for (int nubin = 0; nubin < nu_bins; nubin++) { for (int s = 0; s < num_species; s++) { - Inuinv(nubin, s, n) = robust::SMALL(); + pack_mocmc(b, mocmc_core::Inuinv(nubin, s), n) = robust::SMALL(); } } } bool on_current_mesh_block = true; - swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); + swarm_d.GetNeighborBlockIndex(n, x, y, z, on_current_mesh_block); } }); @@ -347,67 +350,72 @@ TaskStatus MOCMCSampleBoundaries(T *rc) { } template -TaskStatus MOCMCReconstruction(T *rc) { +TaskStatus MOCMCReconstruction(T *rc_base, T *rc) { auto *pmb = rc->GetParentPointer(); - auto &sc = pmb->swarm_data.Get(); - auto &swarm = sc->Get("mocmc"); StateDescriptor *rad = pmb->packages.Get("radiation").get(); + Real num_total = 0; + // Sort and accumulate total active particles + // Sorting must be done before forming packs. + for (int b = 0; b <= rc->NumBlocks() - 1; b++) { + rc_base->GetSwarmData(b)->Get("mocmc")->SortParticlesByCell(); + num_total += rc_base->GetSwarmData(b)->Get("mocmc")->GetNumActive(); + } + rad->UpdateParam("num_total", num_total); + const auto geom = Geometry::GetCoordinateSystem(rc); - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + IndexRange ib = rc->GetBoundsI(IndexDomain::interior); + IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); + IndexRange kb = rc->GetBoundsK(IndexDomain::interior); std::vector variables{pf::velocity::name(), ir::tilPi::name()}; PackIndexMap imap; auto v = rc->PackVariables(variables, imap); - const auto &x = swarm->template Get("x").Get(); - const auto &y = swarm->template Get("y").Get(); - const auto &z = swarm->template Get("z").Get(); - const auto &ncov = swarm->template Get("ncov").Get(); - const auto &Inuinv = swarm->template Get("Inuinv").Get(); - const auto &mu_lo = swarm->template Get("mu_lo").Get(); - const auto &mu_hi = swarm->template Get("mu_hi").Get(); - const auto &phi_lo = swarm->template Get("phi_lo").Get(); - const auto &phi_hi = swarm->template Get("phi_hi").Get(); + static constexpr auto swarm_name = "mocmc"; + static const auto desc_mocmc = + MakeSwarmPackDescriptor( + swarm_name); + auto pack_mocmc = desc_mocmc.GetPack(rc_base); auto pv = imap.GetFlatIdx(pf::velocity::name()); auto iTilPi = imap.GetFlatIdx(ir::tilPi::name()); auto nusamp = rad->Param>("nusamp"); const int nu_bins = rad->Param("nu_bins"); - swarm->SortParticlesByCell(); - auto swarm_d = swarm->GetDeviceContext(); - - rad->UpdateParam("num_total", swarm->GetNumActive()); - auto mocmc_recon = rad->Param("mocmc_recon"); parthenon::par_for( - DEFAULT_LOOP_PATTERN, "MOCMC::kdgrid", DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, - ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { + DEFAULT_LOOP_PATTERN, "MOCMC::kdgrid", DevExecSpace(), 0, rc->NumBlocks() - 1, kb.s, + kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto swarm_d = pack_mocmc.GetContext(b); const int nsamp = swarm_d.GetParticleCountPerCell(k, j, i); for (int n = 0; n < nsamp; n++) { const int nswarm = swarm_d.GetFullIndex(k, j, i, n); if (n == 0) { - mu_lo(nswarm) = 0.0; - mu_hi(nswarm) = 2.0; - phi_lo(nswarm) = 0.0; - phi_hi(nswarm) = 2.0 * M_PI; + pack_mocmc(b, mocmc_core::mu_lo(), nswarm) = 0.0; + pack_mocmc(b, mocmc_core::mu_hi(), nswarm) = 2.0; + pack_mocmc(b, mocmc_core::phi_lo(), nswarm) = 0.0; + pack_mocmc(b, mocmc_core::phi_hi(), nswarm) = 2.0 * M_PI; continue; } Real cov_g[4][4]; - geom.SpacetimeMetric(CellLocation::Cent, k, j, i, cov_g); - const Real vpcon[] = {v(pv(0), k, j, i), v(pv(1), k, j, i), v(pv(2), k, j, i)}; + geom.SpacetimeMetric(CellLocation::Cent, b, k, j, i, cov_g); + const Real vpcon[] = {v(b, pv(0), k, j, i), v(b, pv(1), k, j, i), + v(b, pv(2), k, j, i)}; const Real trial[4] = {0., 1., 0., 0.}; Real ucon[4]; - GetFourVelocity(vpcon, geom, CellLocation::Cent, k, j, i, ucon); + GetFourVelocity(vpcon, geom, CellLocation::Cent, b, k, j, i, ucon); Geometry::Tetrads tetrads(ucon, trial, cov_g); - Real ncov_coord[4] = {ncov(0, nswarm), ncov(1, nswarm), ncov(2, nswarm), - ncov(3, nswarm)}; + Real ncov_coord[4] = {pack_mocmc(b, mocmc_core::ncov(0), nswarm), + pack_mocmc(b, mocmc_core::ncov(1), nswarm), + pack_mocmc(b, mocmc_core::ncov(2), nswarm), + pack_mocmc(b, mocmc_core::ncov(3), nswarm)}; Real ncov_tetrad[4]; tetrads.CoordToTetradCov(ncov_coord, ncov_tetrad); @@ -417,40 +425,53 @@ TaskStatus MOCMCReconstruction(T *rc) { for (int m = 0; m < n; m++) { const int mswarm = swarm_d.GetFullIndex(k, j, i, m); PARTHENON_DEBUG_REQUIRE(mswarm != nswarm, "Comparing the same particle!"); - if (mu > mu_lo(mswarm) && mu < mu_hi(mswarm) && phi > phi_lo(mswarm) && - phi < phi_hi(mswarm)) { - Real mcov_tetrad[4] = {-1., ncov(1, mswarm), ncov(2, mswarm), - ncov(3, mswarm)}; - Real mu0 = mu_hi(mswarm) - mu_lo(mswarm); - Real phi0 = phi_hi(mswarm) - phi_lo(mswarm); + if (mu > pack_mocmc(b, mocmc_core::mu_lo(), mswarm) && + mu < pack_mocmc(b, mocmc_core::mu_hi(), mswarm) && + phi > pack_mocmc(b, mocmc_core::phi_lo(), mswarm) && + phi < pack_mocmc(b, mocmc_core::phi_hi(), mswarm)) { + Real mcov_tetrad[4] = {-1., pack_mocmc(b, mocmc_core::ncov(1), mswarm), + pack_mocmc(b, mocmc_core::ncov(2), mswarm), + pack_mocmc(b, mocmc_core::ncov(3), mswarm)}; + Real mu0 = pack_mocmc(b, mocmc_core::mu_hi(), mswarm) - + pack_mocmc(b, mocmc_core::mu_lo(), mswarm); + Real phi0 = pack_mocmc(b, mocmc_core::phi_hi(), mswarm) - + pack_mocmc(b, mocmc_core::phi_lo(), mswarm); if (mu0 > phi0) { const Real mu_m = 1.0 - ncov_tetrad[1]; mu0 = 0.5 * (mu + mu_m); if (mu < mu0) { - mu_lo(nswarm) = mu_lo(mswarm); - mu_hi(nswarm) = mu0; - mu_lo(mswarm) = mu0; + pack_mocmc(b, mocmc_core::mu_lo(), nswarm) = + pack_mocmc(b, mocmc_core::mu_lo(), mswarm); + pack_mocmc(b, mocmc_core::mu_hi(), nswarm) = mu0; + pack_mocmc(b, mocmc_core::mu_lo(), mswarm) = mu0; } else { - mu_lo(nswarm) = mu0; - mu_hi(nswarm) = mu_hi(mswarm); - mu_hi(mswarm) = mu0; + pack_mocmc(b, mocmc_core::mu_lo(), nswarm) = mu0; + pack_mocmc(b, mocmc_core::mu_hi(), nswarm) = + pack_mocmc(b, mocmc_core::mu_hi(), mswarm); + pack_mocmc(b, mocmc_core::mu_hi(), mswarm) = mu0; } - phi_lo(nswarm) = phi_lo(mswarm); - phi_hi(nswarm) = phi_hi(mswarm); + pack_mocmc(b, mocmc_core::phi_lo(), nswarm) = + pack_mocmc(b, mocmc_core::phi_lo(), mswarm); + pack_mocmc(b, mocmc_core::phi_hi(), nswarm) = + pack_mocmc(b, mocmc_core::phi_hi(), mswarm); } else { const Real phi_m = atan2(ncov_tetrad[3], ncov_tetrad[2]); phi0 = 0.5 * (phi + phi_m); if (phi < phi0) { - phi_lo(nswarm) = phi_lo(mswarm); - phi_hi(nswarm) = phi0; - phi_lo(mswarm) = phi0; + pack_mocmc(b, mocmc_core::phi_lo(), nswarm) = + pack_mocmc(b, mocmc_core::phi_lo(), mswarm); + pack_mocmc(b, mocmc_core::phi_hi(), nswarm) = phi0; + pack_mocmc(b, mocmc_core::phi_lo(), mswarm) = phi0; } else { - phi_lo(nswarm) = phi0; - phi_hi(nswarm) = phi_hi(mswarm); - phi_hi(mswarm) = phi0; + pack_mocmc(b, mocmc_core::phi_lo(), nswarm) = phi0; + pack_mocmc(b, mocmc_core::phi_hi(), nswarm) = + pack_mocmc(b, mocmc_core::phi_hi(), mswarm); + pack_mocmc(b, mocmc_core::phi_hi(), mswarm) = phi0; } - mu_lo(nswarm) = mu_lo(mswarm); - mu_hi(nswarm) = mu_hi(mswarm); + pack_mocmc(b, mocmc_core::mu_lo(), nswarm) = + pack_mocmc(b, mocmc_core::mu_lo(), mswarm); + pack_mocmc(b, mocmc_core::mu_hi(), nswarm) = + pack_mocmc(b, mocmc_core::mu_hi(), mswarm); } break; } // if inside @@ -464,56 +485,51 @@ TaskStatus MOCMCReconstruction(T *rc) { template TaskStatus MOCMCTransport(T *rc, const Real dt) { auto *pmb = rc->GetParentPointer(); - auto &sc = pmb->swarm_data.Get(); - auto &swarm = sc->Get("mocmc"); auto geom = Geometry::GetCoordinateSystem(rc); - auto &t = swarm->template Get("t").Get(); - auto &x = swarm->template Get("x").Get(); - auto &y = swarm->template Get("y").Get(); - auto &z = swarm->template Get("z").Get(); - auto &ncov = swarm->template Get("ncov").Get(); - auto swarm_d = swarm->GetDeviceContext(); + static constexpr auto swarm_name = "mocmc"; + static const auto desc_mocmc = + MakeSwarmPackDescriptor(swarm_name); + auto pack_mocmc = desc_mocmc.GetPack(rc); - pmb->par_for( - "MOCMC::Transport", 0, swarm->GetMaxActiveIndex(), KOKKOS_LAMBDA(const int n) { + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "MOCMC::Transport", DevExecSpace(), 0, + pack_mocmc.GetMaxFlatIndex(), KOKKOS_LAMBDA(const int idx) { + const auto [b, n] = pack_mocmc.GetBlockParticleIndices(idx); + const auto swarm_d = pack_mocmc.GetContext(b); if (swarm_d.IsActive(n)) { - Real y0 = y(n); - Real z0 = z(n); - PushParticle(t(n), x(n), y(n), z(n), ncov(0, n), ncov(1, n), ncov(2, n), - ncov(3, n), dt, geom); + Real &t = pack_mocmc(b, mocmc_core::t(), n); + Real &x = pack_mocmc(b, swarm_position::x(), n); + Real &y = pack_mocmc(b, swarm_position::y(), n); + Real &z = pack_mocmc(b, swarm_position::z(), n); + Real y0 = y; + Real z0 = z; + PushParticle(t, x, y, z, pack_mocmc(b, mocmc_core::ncov(0), n), + pack_mocmc(b, mocmc_core::ncov(1), n), + pack_mocmc(b, mocmc_core::ncov(2), n), + pack_mocmc(b, mocmc_core::ncov(3), n), dt, geom); bool on_current_mesh_block = true; - swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); + swarm_d.GetNeighborBlockIndex(n, x, y, z, on_current_mesh_block); } }); return TaskStatus::complete; } -// TODO(BRR): Hack to get around current lack of support for packing parthenon swarms -template <> -TaskStatus MOCMCFluidSource(MeshData *rc, const Real dt, const bool update_fluid) { - for (int n = 0; n < rc->NumBlocks(); n++) { - MOCMCFluidSource(rc->GetBlockData(n).get(), dt, update_fluid); - } - return TaskStatus::complete; -} - template -TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { +TaskStatus MOCMCFluidSource(T *rc_base, T *rc, const Real dt, const bool update_fluid) { // Assume particles are already sorted from MOCMCReconstruction call! auto *pmb = rc->GetParentPointer(); - auto &sc = pmb->swarm_data.Get(); - auto &swarm = sc->Get("mocmc"); StateDescriptor *rad = pmb->packages.Get("radiation").get(); // Meshblock geometry const auto geom = Geometry::GetCoordinateSystem(rc); - const IndexRange &ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - const IndexRange &jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - const IndexRange &kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + const IndexRange &ib = rc->GetBoundsI(IndexDomain::interior); + const IndexRange &jb = rc->GetBoundsJ(IndexDomain::interior); + const IndexRange &kb = rc->GetBoundsK(IndexDomain::interior); // Microphysics auto opac = pmb->packages.Get("opacity"); @@ -556,13 +572,12 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { cye = imap[cf::ye::name()].first; } - const auto &ncov = swarm->template Get("ncov").Get(); - const auto &mu_lo = swarm->template Get("mu_lo").Get(); - const auto &mu_hi = swarm->template Get("mu_hi").Get(); - const auto &phi_lo = swarm->template Get("phi_lo").Get(); - const auto &phi_hi = swarm->template Get("phi_hi").Get(); - const auto &Inuinv = swarm->template Get("Inuinv").Get(); - const auto swarm_d = swarm->GetDeviceContext(); + static constexpr auto swarm_name = "mocmc"; + static const auto desc_mocmc = + MakeSwarmPackDescriptor( + swarm_name); + auto pack_mocmc = desc_mocmc.GetPack(rc_base); auto nusamp = rad->Param>("nusamp"); const int nu_bins = rad->Param("nu_bins"); @@ -574,19 +589,21 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { species_d[s] = species[s]; } - const int iblock = 0; // No meshblockpacks right now int nspec = idx_E.DimSize(1); if (true) { // update = lagged parthenon::par_for( - DEFAULT_LOOP_PATTERN, "MOCMC::FluidSource", DevExecSpace(), kb.s, kb.e, jb.s, - jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { + DEFAULT_LOOP_PATTERN, "MOCMC::FluidSource", DevExecSpace(), 0, + rc->NumBlocks() - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto swarm_d = pack_mocmc.GetContext(b); Real cov_g[4][4]; - geom.SpacetimeMetric(CellLocation::Cent, iblock, k, j, i, cov_g); - Real alpha = geom.Lapse(CellLocation::Cent, iblock, k, j, i); + geom.SpacetimeMetric(CellLocation::Cent, b, k, j, i, cov_g); + Real alpha = geom.Lapse(CellLocation::Cent, b, k, j, i); Real con_beta[3]; - geom.ContravariantShift(CellLocation::Cent, iblock, k, j, i, con_beta); - const Real vpcon[] = {v(pv(0), k, j, i), v(pv(1), k, j, i), v(pv(2), k, j, i)}; + geom.ContravariantShift(CellLocation::Cent, b, k, j, i, con_beta); + const Real vpcon[] = {v(b, pv(0), k, j, i), v(b, pv(1), k, j, i), + v(b, pv(2), k, j, i)}; const Real W = phoebus::GetLorentzFactor(vpcon, cov_g); const Real ucon[4] = {W / alpha, vpcon[0] - con_beta[0] * W / alpha, vpcon[1] - con_beta[1] * W / alpha, @@ -596,23 +613,22 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { const int nsamp = swarm_d.GetParticleCountPerCell(k, j, i); // Set up the background state - Vec con_v{{v(iblock, pv(0), k, j, i), v(iblock, pv(1), k, j, i), - v(iblock, pv(2), k, j, i)}}; + Vec con_v{{v(b, pv(0), k, j, i), v(b, pv(1), k, j, i), v(b, pv(2), k, j, i)}}; Tens2 cov_gamma; - geom.Metric(CellLocation::Cent, iblock, k, j, i, cov_gamma.data); - Real alpha = geom.Lapse(CellLocation::Cent, iblock, k, j, i); - Real sdetgam = geom.DetGamma(CellLocation::Cent, iblock, k, j, i); - LocalThreeGeometry g(geom, CellLocation::Cent, iblock, k, j, i); - Real Estar = v(iblock, idx_E(ispec), k, j, i) / sdetgam; - Vec cov_Fstar{v(iblock, idx_F(ispec, 0), k, j, i) / sdetgam, - v(iblock, idx_F(ispec, 1), k, j, i) / sdetgam, - v(iblock, idx_F(ispec, 2), k, j, i) / sdetgam}; + geom.Metric(CellLocation::Cent, b, k, j, i, cov_gamma.data); + Real alpha = geom.Lapse(CellLocation::Cent, b, k, j, i); + Real sdetgam = geom.DetGamma(CellLocation::Cent, b, k, j, i); + LocalThreeGeometry g(geom, CellLocation::Cent, b, k, j, i); + Real Estar = v(b, idx_E(ispec), k, j, i) / sdetgam; + Vec cov_Fstar{v(b, idx_F(ispec, 0), k, j, i) / sdetgam, + v(b, idx_F(ispec, 1), k, j, i) / sdetgam, + v(b, idx_F(ispec, 2), k, j, i) / sdetgam}; Tens2 con_tilPi; SPACELOOP2(ii, jj) { - con_tilPi(ii, jj) = v(iblock, iTilPi(ispec, ii, jj), k, j, i); + con_tilPi(ii, jj) = v(b, iTilPi(ispec, ii, jj), k, j, i); } - Real JBB = opac_d.EnergyDensityFromTemperature(v(iblock, pT, k, j, i), - species_d[ispec]); + Real JBB = + opac_d.EnergyDensityFromTemperature(v(b, pT, k, j, i), species_d[ispec]); ClosureMOCMC<> c(con_v, &g); @@ -621,15 +637,19 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { // Calculate Inu0 for (int bin = 0; bin < nu_bins; bin++) { - v(iblock, Inu0(ispec, bin), k, j, i) = 0.; + v(b, Inu0(ispec, bin), k, j, i) = 0.; } for (int n = 0; n < nsamp; n++) { const int nswarm = swarm_d.GetFullIndex(k, j, i, n); - const Real dOmega = (mu_hi(nswarm) - mu_lo(nswarm)) * - (phi_hi(nswarm) - phi_lo(nswarm)) / (4. * M_PI); + const Real dOmega = (pack_mocmc(b, mocmc_core::mu_hi(), nswarm) - + pack_mocmc(b, mocmc_core::mu_lo(), nswarm)) * + (pack_mocmc(b, mocmc_core::phi_hi(), nswarm) - + pack_mocmc(b, mocmc_core::phi_lo(), nswarm)) / + (4. * M_PI); for (int bin = 0; bin < nu_bins; bin++) { - v(iblock, Inu0(ispec, bin), k, j, i) += - Inuinv(bin, ispec, nswarm) * pow(nusamp(bin), 3) * dOmega; + v(b, Inu0(ispec, bin), k, j, i) += + pack_mocmc(b, mocmc_core::Inuinv(bin, ispec), nswarm) * + pow(nusamp(bin), 3) * dOmega; } } @@ -640,42 +660,39 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { Real Itot = 0.; for (int bin = 0; bin < nu_bins; bin++) { kappaJ += opac_d.AngleAveragedAbsorptionCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nusamp(bin)) * - v(iblock, Inu0(ispec, bin), k, j, i) * nusamp(bin); + v(b, pdens, k, j, i), v(b, pT, k, j, i), v(b, pye, k, j, i), + species_d[ispec], nusamp(bin)) * + v(b, Inu0(ispec, bin), k, j, i) * nusamp(bin); kappaH += opac_d.TotalScatteringCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nusamp(bin)) * - v(iblock, Inu0(ispec, bin), k, j, i) * nusamp(bin); - Itot += v(iblock, Inu0(ispec, bin), k, j, i) * nusamp(bin); + v(b, pdens, k, j, i), v(b, pT, k, j, i), v(b, pye, k, j, i), + species_d[ispec], nusamp(bin)) * + v(b, Inu0(ispec, bin), k, j, i) * nusamp(bin); + Itot += v(b, Inu0(ispec, bin), k, j, i) * nusamp(bin); } // Trapezoidal rule kappaJ -= 0.5 * opac_d.AngleAveragedAbsorptionCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nusamp(0)) * - v(iblock, Inu0(ispec, 0), k, j, i) * nusamp(0); - kappaJ -= - 0.5 * - opac_d.AngleAveragedAbsorptionCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nusamp(nu_bins - 1)) * - v(iblock, Inu0(ispec, nu_bins - 1), k, j, i) * nusamp(nu_bins - 1); + v(b, pdens, k, j, i), v(b, pT, k, j, i), v(b, pye, k, j, i), + species_d[ispec], nusamp(0)) * + v(b, Inu0(ispec, 0), k, j, i) * nusamp(0); + kappaJ -= 0.5 * + opac_d.AngleAveragedAbsorptionCoefficient( + v(b, pdens, k, j, i), v(b, pT, k, j, i), v(b, pye, k, j, i), + species_d[ispec], nusamp(nu_bins - 1)) * + v(b, Inu0(ispec, nu_bins - 1), k, j, i) * nusamp(nu_bins - 1); + kappaH -= 0.5 * + opac_d.TotalScatteringCoefficient( + v(b, pdens, k, j, i), v(b, pT, k, j, i), v(b, pye, k, j, i), + species_d[ispec], nusamp(0)) * + v(b, Inu0(ispec, 0), k, j, i) * nusamp(0); kappaH -= 0.5 * opac_d.TotalScatteringCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nusamp(0)) * - v(iblock, Inu0(ispec, 0), k, j, i) * nusamp(0); - kappaH -= - 0.5 * - opac_d.TotalScatteringCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nusamp(nu_bins - 1)) * - v(iblock, Inu0(ispec, nu_bins - 1), k, j, i) * nusamp(nu_bins - 1); - Itot -= 0.5 * - (v(iblock, Inu0(ispec, 0), k, j, i) * nusamp(0) + - v(iblock, Inu0(ispec, nu_bins - 1), k, j, i) * nusamp(nu_bins - 1)); + v(b, pdens, k, j, i), v(b, pT, k, j, i), v(b, pye, k, j, i), + species_d[ispec], nusamp(nu_bins - 1)) * + v(b, Inu0(ispec, nu_bins - 1), k, j, i) * nusamp(nu_bins - 1); + Itot -= 0.5 * (v(b, Inu0(ispec, 0), k, j, i) * nusamp(0) + + v(b, Inu0(ispec, nu_bins - 1), k, j, i) * nusamp(nu_bins - 1)); kappaJ = robust::ratio(kappaJ, Itot); kappaH = robust::ratio(kappaH, Itot) + kappaJ; @@ -683,24 +700,24 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { Real tauH = alpha * dt * kappaH; // Store kappaH for asymptotic fluxes - v(iblock, idx_kappaH(ispec), k, j, i) = kappaH; + v(b, idx_kappaH(ispec), k, j, i) = kappaH; auto status = c.LinearSourceUpdate(Estar, cov_Fstar, con_tilPi, JBB, tauJ, tauH, &dE, &cov_dF); // Add source corrections to conserved radiation variables - v(iblock, idx_E(ispec), k, j, i) += sdetgam * dE; + v(b, idx_E(ispec), k, j, i) += sdetgam * dE; for (int idir = 0; idir < 3; ++idir) { - v(iblock, idx_F(ispec, idir), k, j, i) += sdetgam * cov_dF(idir); + v(b, idx_F(ispec, idir), k, j, i) += sdetgam * cov_dF(idir); } // Add source corrections to conserved fluid variables if (update_fluid) { - v(iblock, cye, k, j, i) -= sdetgam * 0.0; - v(iblock, ceng, k, j, i) -= sdetgam * dE; - v(iblock, cmom_lo + 0, k, j, i) -= sdetgam * cov_dF(0); - v(iblock, cmom_lo + 1, k, j, i) -= sdetgam * cov_dF(1); - v(iblock, cmom_lo + 2, k, j, i) -= sdetgam * cov_dF(2); + v(b, cye, k, j, i) -= sdetgam * 0.0; + v(b, ceng, k, j, i) -= sdetgam * dE; + v(b, cmom_lo + 0, k, j, i) -= sdetgam * cov_dF(0); + v(b, cmom_lo + 1, k, j, i) -= sdetgam * cov_dF(1); + v(b, cmom_lo + 2, k, j, i) -= sdetgam * cov_dF(2); } // Update sample intensities @@ -709,7 +726,9 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { Real nu_fluid0 = nusamp(0); Real nu_lab0 = 0.; - SPACETIMELOOP(nu) { nu_lab0 -= ncov(nu, nswarm) * ucon[nu]; } + SPACETIMELOOP(nu) { + nu_lab0 -= pack_mocmc(b, mocmc_core::ncov(nu), nswarm) * ucon[nu]; + } nu_lab0 *= nusamp(0); const Real shift = (log(nusamp(0)) - log(nu_lab0)) / dlnu; @@ -723,11 +742,11 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { const Real nu_fluid = nusamp(bin); const Real alphainv_s = nu_fluid * opac_d.TotalScatteringCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nu_fluid); - v(iblock, ijinvs(ispec, bin), k, j, i) = - v(iblock, Inu0(ispec, bin), k, j, i) / - (nu_fluid * nu_fluid * nu_fluid) * alphainv_s; + v(b, pdens, k, j, i), v(b, pT, k, j, i), + v(b, pye, k, j, i), species_d[ispec], nu_fluid); + v(b, ijinvs(ispec, bin), k, j, i) = v(b, Inu0(ispec, bin), k, j, i) / + (nu_fluid * nu_fluid * nu_fluid) * + alphainv_s; } for (int bin = 0; bin < nu_bins; bin++) { @@ -736,27 +755,27 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { const Real ds = dt / ucon[0] / nu_fluid; const Real alphainv_a = nu_fluid * opac_d.TotalScatteringCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nu_fluid); + v(b, pdens, k, j, i), v(b, pT, k, j, i), + v(b, pye, k, j, i), species_d[ispec], nu_fluid); const Real alphainv_s = nu_fluid * opac_d.TotalScatteringCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nu_fluid); - const Real jinv_a = - opac_d.ThermalDistributionOfTNu(v(iblock, pT, k, j, i), - species_d[ispec], nu_fluid) / - (nu_fluid * nu_fluid * nu_fluid) * alphainv_a; + v(b, pdens, k, j, i), v(b, pT, k, j, i), + v(b, pye, k, j, i), species_d[ispec], nu_fluid); + const Real jinv_a = opac_d.ThermalDistributionOfTNu( + v(b, pT, k, j, i), species_d[ispec], nu_fluid) / + (nu_fluid * nu_fluid * nu_fluid) * alphainv_a; // Interpolate invariant scattering emissivity to lab frame Real jinv_s = 0.; interp.GetIndicesAndWeights(bin, nubin_shift, nubin_wgt); for (int isup = 0; isup < interp.StencilSize(); isup++) { - jinv_s += nubin_wgt[isup] * - v(iblock, ijinvs(ispec, nubin_shift[isup]), k, j, i); + jinv_s += + nubin_wgt[isup] * v(b, ijinvs(ispec, nubin_shift[isup]), k, j, i); } - Inuinv(bin, ispec, nswarm) = - (Inuinv(bin, ispec, nswarm) + ds * (jinv_a + jinv_s)) / + pack_mocmc(b, mocmc_core::Inuinv(bin, ispec), nswarm) = + (pack_mocmc(b, mocmc_core::Inuinv(bin, ispec), nswarm) + + ds * (jinv_a + jinv_s)) / (1. + ds * (alphainv_a + alphainv_s)); } } @@ -773,28 +792,17 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { return TaskStatus::complete; } -// TODO(BRR): Hack to get around current lack of support for packing parthenon swarms -template <> -TaskStatus MOCMCEddington(MeshData *rc) { - for (int n = 0; n < rc->NumBlocks(); n++) { - MOCMCEddington(rc->GetBlockData(n).get()); - } - return TaskStatus::complete; -} - template -TaskStatus MOCMCEddington(T *rc) { +TaskStatus MOCMCEddington(T *rc_base, T *rc) { // Assume list is sorted! namespace ir = radmoment_internal; auto *pmb = rc->GetParentPointer(); - auto &sc = pmb->swarm_data.Get(); - auto &swarm = sc->Get("mocmc"); StateDescriptor *rad = pmb->packages.Get("radiation").get(); - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + IndexRange ib = rc->GetBoundsI(IndexDomain::interior); + IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); + IndexRange kb = rc->GetBoundsK(IndexDomain::interior); auto geom = Geometry::GetCoordinateSystem(rc); @@ -802,12 +810,12 @@ TaskStatus MOCMCEddington(T *rc) { PackIndexMap imap; auto v = rc->PackVariables(variables, imap); - const auto &Inuinv = swarm->template Get("Inuinv").Get(); - const auto &ncov = swarm->template Get("ncov").Get(); - const auto &mu_lo = swarm->template Get("mu_lo").Get(); - const auto &mu_hi = swarm->template Get("mu_hi").Get(); - const auto &phi_lo = swarm->template Get("phi_lo").Get(); - const auto &phi_hi = swarm->template Get("phi_hi").Get(); + static constexpr auto swarm_name = "mocmc"; + static const auto desc_mocmc = + MakeSwarmPackDescriptor( + swarm_name); + auto pack_mocmc = desc_mocmc.GetPack(rc_base); auto iTilPi = imap.GetFlatIdx(ir::tilPi::name()); const auto pvel_lo = imap[pf::velocity::name()].first; @@ -816,18 +824,16 @@ TaskStatus MOCMCEddington(T *rc) { auto nusamp = rad->Param>("nusamp"); auto num_species = rad->Param("num_species"); - // TODO(BRR) block packing eventually - const int iblock = 0; - - auto swarm_d = swarm->GetDeviceContext(); parthenon::par_for( - DEFAULT_LOOP_PATTERN, "MOCMC::kdgrid", DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, - ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { + DEFAULT_LOOP_PATTERN, "MOCMC::kdgrid", DevExecSpace(), 0, rc->NumBlocks() - 1, kb.s, + kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto swarm_d = pack_mocmc.GetContext(b); // initialize eddington to zero for (int s = 0; s < num_species; s++) { for (int ii = 0; ii < 3; ii++) { for (int jj = ii; jj < 3; jj++) { - v(iTilPi(s, ii, jj), k, j, i) = 0.0; + v(b, iTilPi(s, ii, jj), k, j, i) = 0.0; } } } @@ -835,12 +841,12 @@ TaskStatus MOCMCEddington(T *rc) { const int nsamp = swarm_d.GetParticleCountPerCell(k, j, i); Real cov_g[4][4]; - geom.SpacetimeMetric(CellLocation::Cent, iblock, k, j, i, cov_g); - Real alpha = geom.Lapse(CellLocation::Cent, iblock, k, j, i); + geom.SpacetimeMetric(CellLocation::Cent, b, k, j, i, cov_g); + Real alpha = geom.Lapse(CellLocation::Cent, b, k, j, i); Real con_beta[3]; - geom.ContravariantShift(CellLocation::Cent, iblock, k, j, i, con_beta); - const Real vpcon[] = {v(pvel_lo, k, j, i), v(pvel_lo + 1, k, j, i), - v(pvel_lo + 2, k, j, i)}; + geom.ContravariantShift(CellLocation::Cent, b, k, j, i, con_beta); + const Real vpcon[] = {v(b, pvel_lo, k, j, i), v(b, pvel_lo + 1, k, j, i), + v(b, pvel_lo + 2, k, j, i)}; const Real W = phoebus::GetLorentzFactor(vpcon, cov_g); const Real ucon[4] = {W / alpha, vpcon[0] - con_beta[0] * W / alpha, vpcon[1] - con_beta[1] * W / alpha, @@ -851,7 +857,9 @@ TaskStatus MOCMCEddington(T *rc) { Real nu_fluid0 = nusamp(0); Real nu_lab0 = 0.; - SPACETIMELOOP(nu) { nu_lab0 -= ncov(nu, nswarm) * ucon[nu]; } + SPACETIMELOOP(nu) { + nu_lab0 -= pack_mocmc(b, mocmc_core::ncov(nu), nswarm) * ucon[nu]; + } nu_lab0 *= nusamp(0); const Real shift = (log(nusamp(0)) - log(nu_lab0)) / dlnu; @@ -866,26 +874,33 @@ TaskStatus MOCMCEddington(T *rc) { for (int s = 0; s < num_species; s++) { interp.GetIndicesAndWeights(nubin, nubin_shift, nubin_wgt); for (int isup = 0; isup < interp.StencilSize(); isup++) { - I[s] += nubin_wgt[isup] * Inuinv(nubin_shift[isup], s, nswarm) * + I[s] += nubin_wgt[isup] * + pack_mocmc(b, mocmc_core::Inuinv(nubin_shift[isup], s), nswarm) * pow(nusamp(nubin), 4); } } } Real wgts[6]; - kdgrid::integrate_ninj_domega_quad(mu_lo(nswarm), mu_hi(nswarm), phi_lo(nswarm), - phi_hi(nswarm), wgts); + kdgrid::integrate_ninj_domega_quad(pack_mocmc(b, mocmc_core::mu_lo(), nswarm), + pack_mocmc(b, mocmc_core::mu_hi(), nswarm), + pack_mocmc(b, mocmc_core::phi_lo(), nswarm), + pack_mocmc(b, mocmc_core::phi_hi(), nswarm), + wgts); for (int ii = 0; ii < 3; ii++) { for (int jj = ii; jj < 3; jj++) { const int ind = Geometry::Utils::Flatten2(ii, jj, 3); for (int s = 0; s < num_species; s++) { - v(iTilPi(s, ii, jj), k, j, i) += wgts[ind] * I[s]; + v(b, iTilPi(s, ii, jj), k, j, i) += wgts[ind] * I[s]; } } } for (int s = 0; s < num_species; s++) { - energy[s] += (mu_hi(nswarm) - mu_lo(nswarm)) * - (phi_hi(nswarm) - phi_lo(nswarm)) * I[s]; + energy[s] += (pack_mocmc(b, mocmc_core::mu_hi(), nswarm) - + pack_mocmc(b, mocmc_core::mu_lo(), nswarm)) * + (pack_mocmc(b, mocmc_core::phi_hi(), nswarm) - + pack_mocmc(b, mocmc_core::phi_lo(), nswarm)) * + I[s]; } } @@ -893,17 +908,17 @@ TaskStatus MOCMCEddington(T *rc) { for (int s = 0; s < num_species; s++) { for (int ii = 0; ii < 3; ii++) { for (int jj = ii; jj < 3; jj++) { - v(iTilPi(s, ii, jj), k, j, i) /= energy[s]; + v(b, iTilPi(s, ii, jj), k, j, i) /= energy[s]; } - v(iTilPi(s, ii, ii), k, j, i) -= 1. / 3.; + v(b, iTilPi(s, ii, ii), k, j, i) -= 1. / 3.; } } } for (int s = 0; s < num_species; s++) { - v(iTilPi(s, 1, 0), k, j, i) = v(iTilPi(s, 0, 1), k, j, i); - v(iTilPi(s, 2, 0), k, j, i) = v(iTilPi(s, 0, 2), k, j, i); - v(iTilPi(s, 2, 1), k, j, i) = v(iTilPi(s, 1, 2), k, j, i); + v(b, iTilPi(s, 1, 0), k, j, i) = v(b, iTilPi(s, 0, 1), k, j, i); + v(b, iTilPi(s, 2, 0), k, j, i) = v(b, iTilPi(s, 0, 2), k, j, i); + v(b, iTilPi(s, 2, 1), k, j, i) = v(b, iTilPi(s, 1, 2), k, j, i); } }); @@ -920,16 +935,21 @@ TaskStatus MOCMCUpdateParticleCount(Mesh *pmesh, std::vector *resolution) return TaskStatus::complete; } -template TaskStatus MOCMCSampleBoundaries>(MeshBlockData *); -template TaskStatus MOCMCReconstruction>(MeshBlockData *); -template TaskStatus MOCMCEddington>(MeshBlockData *rc); +template TaskStatus MOCMCSampleBoundaries>(MeshData *rc_base, + MeshData *rc); +// template TaskStatus MOCMCSampleBoundaries>(MeshData *); +template TaskStatus MOCMCReconstruction>(MeshData *rc_base, + MeshData *rc); +// template TaskStatus MOCMCReconstruction>(MeshData *); +template TaskStatus MOCMCEddington>(MeshData *rc_base, + MeshData *rc); +// template TaskStatus MOCMCEddington>(MeshData *rc); template void MOCMCInitSamples>(MeshBlockData *); -template TaskStatus MOCMCTransport>(MeshBlockData *rc, - const Real dt); +template TaskStatus MOCMCTransport>(MeshData *rc, const Real dt); // template TaskStatus MOCMCFluidSource>(MeshData *rc, const Real dt, -// const bool update_fluid); -template TaskStatus MOCMCFluidSource>(MeshBlockData *rc, - const Real dt, - const bool update_fluid); +// const bool update_fluid); +template TaskStatus MOCMCFluidSource>(MeshData *rc_base, + MeshData *rc, const Real dt, + const bool update_fluid); } // namespace radiation diff --git a/src/radiation/moments.cpp b/src/radiation/moments.cpp index aa1633cd5..0c0cdc962 100644 --- a/src/radiation/moments.cpp +++ b/src/radiation/moments.cpp @@ -62,39 +62,24 @@ TaskStatus MomentCon2PrimImpl(T *rc) { namespace cr = radmoment_cons; namespace pr = radmoment_prim; namespace ir = radmoment_internal; + namespace pf = fluid_prim; Mesh *pmesh = rc->GetMeshPointer(); StateDescriptor *rad = pmesh->packages.Get("radiation").get(); + const int n_species = rad->Param("num_species"); IndexRange ib = rc->GetBoundsI(IndexDomain::entire); IndexRange jb = rc->GetBoundsJ(IndexDomain::entire); IndexRange kb = rc->GetBoundsK(IndexDomain::entire); + using parthenon::MakePackDescriptor; + auto &resolved_pkgs = pmesh->resolved_packages; - std::vector variables{cr::E::name(), - cr::F::name(), - pr::J::name(), - pr::H::name(), - fluid_prim::velocity::name(), - ir::xi::name(), - ir::phi::name(), - ir::c2pfail::name(), - ir::tilPi::name()}; - PackIndexMap imap; - auto v = rc->PackVariables(variables, imap); - - auto cE = imap.GetFlatIdx(cr::E::name()); - auto pJ = imap.GetFlatIdx(pr::J::name()); - auto cF = imap.GetFlatIdx(cr::F::name()); - auto pH = imap.GetFlatIdx(pr::H::name()); - auto pv = imap.GetFlatIdx(fluid_prim::velocity::name()); - auto iTilPi = imap.GetFlatIdx(ir::tilPi::name(), false); - auto specB = cE.GetBounds(1); - auto dirB = pH.GetBounds(2); - - auto iXi = imap.GetFlatIdx(ir::xi::name()); - auto iPhi = imap.GetFlatIdx(ir::phi::name()); + static auto desc = + MakePackDescriptor(resolved_pkgs.get()); - auto ifail = imap[ir::c2pfail::name()].first; + auto v = desc.GetPack(rc); + const int nblocks = v.GetNBlocks(); auto geom = Geometry::GetCoordinateSystem(rc); const Real pi = acos(-1); @@ -104,21 +89,22 @@ TaskStatus MomentCon2PrimImpl(T *rc) { parthenon::par_for( DEFAULT_LOOP_PATTERN, "RadMoments::Con2Prim", DevExecSpace(), 0, - v.GetDim(5) - 1, // Loop over meshblocks - specB.s, specB.e, // Loop over species - kb.s, kb.e, // z-loop - jb.s, jb.e, // y-loop - ib.s, ib.e, // x-loop + rc->NumBlocks() - 1, // Loop over meshblocks + 0, n_species - 1, // Loop over species + kb.s, kb.e, // z-loop + jb.s, jb.e, // y-loop + ib.s, ib.e, // x-loop KOKKOS_LAMBDA(const int b, const int ispec, const int k, const int j, const int i) { Tens2 cov_gamma; geom.Metric(CellLocation::Cent, b, k, j, i, cov_gamma.data); const Real isdetgam = 1.0 / geom.DetGamma(CellLocation::Cent, b, k, j, i); - const Real vp[3] = {v(b, pv(0), k, j, i), v(b, pv(1), k, j, i), - v(b, pv(2), k, j, i)}; + const Real vp[3] = {v(b, pf::velocity(0), k, j, i), + v(b, pf::velocity(1), k, j, i), + v(b, pf::velocity(2), k, j, i)}; const Real W = phoebus::GetLorentzFactor(vp, cov_gamma.data); - Vec con_v{{v(b, pv(0), k, j, i) / W, v(b, pv(1), k, j, i) / W, - v(b, pv(2), k, j, i) / W}}; + Vec con_v{{v(b, pf::velocity(0), k, j, i) / W, v(b, pf::velocity(1), k, j, i) / W, + v(b, pf::velocity(2), k, j, i) / W}}; typename CLOSURE::LocalGeometryType g(geom, CellLocation::Cent, b, k, j, i); CLOSURE c(con_v, &g, closure_runtime_params); @@ -126,44 +112,46 @@ TaskStatus MomentCon2PrimImpl(T *rc) { Real J; Vec covH; Tens2 conTilPi; - Real E = v(b, cE(ispec), k, j, i) * isdetgam; - Vec covF = {{v(b, cF(ispec, 0), k, j, i) * isdetgam, - v(b, cF(ispec, 1), k, j, i) * isdetgam, - v(b, cF(ispec, 2), k, j, i) * isdetgam}}; - - if (iTilPi.IsValid()) { - SPACELOOP2(ii, jj) { conTilPi(ii, jj) = v(b, iTilPi(ispec, ii, jj), k, j, i); } + Real E = v(b, cr::E(ispec), k, j, i) * isdetgam; + Vec covF = {{v(b, cr::F(ispec, 0), k, j, i) * isdetgam, + v(b, cr::F(ispec, 1), k, j, i) * isdetgam, + v(b, cr::F(ispec, 2), k, j, i) * isdetgam}}; + + if (v.Contains(b, ir::tilPi())) { + SPACELOOP2(ii, jj) { + conTilPi(ii, jj) = v(b, ir::tilPi(ispec, ii, jj), k, j, i); + } } else { Real xi = 0.0; Real phi = pi; // TODO(BRR) Remove STORE_GUESS parameter and instead check if closure type is // M1? if (STORE_GUESS) { - xi = v(b, iXi(ispec), k, j, i); - phi = 1.0001 * v(b, iPhi(ispec), k, j, i); + xi = v(b, ir::xi(ispec), k, j, i); + phi = 1.0001 * v(b, ir::phi(ispec), k, j, i); } c.GetConTilPiFromCon(E, covF, xi, phi, &conTilPi); if (STORE_GUESS) { - v(b, iXi(ispec), k, j, i) = xi; - v(b, iPhi(ispec), k, j, i) = phi; + v(b, ir::xi(ispec), k, j, i) = xi; + v(b, ir::phi(ispec), k, j, i) = phi; } } auto status = c.Con2Prim(E, covF, conTilPi, &J, &covH); if (status == ClosureStatus::modified) { c.Prim2Con(J, covH, conTilPi, &E, &covF); - v(b, cE(ispec), k, j, i) = E / isdetgam; - SPACELOOP(ii) { v(b, cF(ispec, ii), k, j, i) = covF(ii) / isdetgam; } + v(b, cr::E(ispec), k, j, i) = E / isdetgam; + SPACELOOP(ii) { v(b, cr::F(ispec, ii), k, j, i) = covF(ii) / isdetgam; } status = ClosureStatus::success; } - v(b, pJ(ispec), k, j, i) = J; - for (int idir = dirB.s; idir <= dirB.e; ++idir) { // Loop over directions + v(b, pr::J(ispec), k, j, i) = J; + for (int idir = 0; idir <= 2; ++idir) { // Loop over directions // Use the scaled value of the rest frame flux for reconstruction - v(b, pH(ispec, idir), k, j, i) = robust::ratio(covH(idir), J); + v(b, pr::H(ispec, idir), k, j, i) = robust::ratio(covH(idir), J); } - v(b, ifail, k, j, i) = + v(b, ir::c2pfail(), k, j, i) = (status == ClosureStatus::success ? FailFlags::success : FailFlags::fail); }); @@ -199,38 +187,31 @@ TaskStatus MomentPrim2ConImpl(T *rc, IndexDomain domain) { namespace cr = radmoment_cons; namespace pr = radmoment_prim; namespace ir = radmoment_internal; + namespace pf = fluid_prim; + Mesh *pmesh = rc->GetMeshPointer(); + StateDescriptor *rad = pmesh->packages.Get("radiation").get(); IndexRange ib = rc->GetBoundsI(domain); IndexRange jb = rc->GetBoundsJ(domain); IndexRange kb = rc->GetBoundsK(domain); - std::vector variables{cr::E::name(), cr::F::name(), pr::J::name(), - pr::H::name(), fluid_prim::velocity::name()}; - if (programming::is_specialization_of::value) { - variables.push_back(ir::tilPi::name()); - } - PackIndexMap imap; - auto v = rc->PackVariables(variables, imap); - - auto cE = imap.GetFlatIdx(cr::E::name()); - auto pJ = imap.GetFlatIdx(pr::J::name()); - auto cF = imap.GetFlatIdx(cr::F::name()); - auto pH = imap.GetFlatIdx(pr::H::name()); - auto pv = imap.GetFlatIdx(fluid_prim::velocity::name()); - auto iTilPi = imap.GetFlatIdx(ir::tilPi::name(), false); + auto &resolved_pkgs = pmesh->resolved_packages; + static auto desc = + MakePackDescriptor( + resolved_pkgs.get()); - auto specB = cE.GetBounds(1); - auto dirB = pH.GetBounds(2); + auto v = desc.GetPack(rc); auto geom = Geometry::GetCoordinateSystem(rc); + const int n_species = rad->Param("num_species"); parthenon::par_for( DEFAULT_LOOP_PATTERN, "RadMoments::Prim2Con", DevExecSpace(), 0, - v.GetDim(5) - 1, // Loop over meshblocks - specB.s, specB.e, // Loop over species - kb.s, kb.e, // z-loop - jb.s, jb.e, // y-loop - ib.s, ib.e, // x-loop + rc->NumBlocks() - 1, // Loop over meshblocks + 0, n_species - 1, // Loop over species + kb.s, kb.e, // z-loop + jb.s, jb.e, // y-loop + ib.s, ib.e, // x-loop KOKKOS_LAMBDA(const int b, const int ispec, const int k, const int j, const int i) { // Set up the background const Real sdetgam = geom.DetGamma(CellLocation::Cent, b, k, j, i); @@ -238,8 +219,9 @@ TaskStatus MomentPrim2ConImpl(T *rc, IndexDomain domain) { geom.Metric(CellLocation::Cent, b, k, j, i, cov_gamma.data); typename CLOSURE::LocalGeometryType g(geom, CellLocation::Cent, b, k, j, i); - const Real con_vp[3] = {v(b, pv(0), k, j, i), v(b, pv(1), k, j, i), - v(b, pv(2), k, j, i)}; + const Real con_vp[3] = {v(b, pf::velocity(0), k, j, i), + v(b, pf::velocity(1), k, j, i), + v(b, pf::velocity(2), k, j, i)}; const Real W = phoebus::GetLorentzFactor(con_vp, cov_gamma.data); Vec con_v{{con_vp[0] / W, con_vp[1] / W, con_vp[2] / W}}; @@ -248,12 +230,15 @@ TaskStatus MomentPrim2ConImpl(T *rc, IndexDomain domain) { Real E; Vec covF; Tens2 conTilPi; - Real J = v(b, pJ(ispec), k, j, i); - Vec covH = {{v(b, pH(ispec, 0), k, j, i) * J, v(b, pH(ispec, 1), k, j, i) * J, - v(b, pH(ispec, 2), k, j, i) * J}}; - - if (iTilPi.IsValid()) { - SPACELOOP2(ii, jj) { conTilPi(ii, jj) = v(b, iTilPi(ispec, ii, jj), k, j, i); } + Real J = v(b, pr::J(ispec), k, j, i); + Vec covH = {{v(b, pr::H(ispec, 0), k, j, i) * J, + v(b, pr::H(ispec, 1), k, j, i) * J, + v(b, pr::H(ispec, 2), k, j, i) * J}}; + + if (v.Contains(b, ir::tilPi())) { + SPACELOOP2(ii, jj) { + conTilPi(ii, jj) = v(b, ir::tilPi(ispec, ii, jj), k, j, i); + } } else { c.GetConTilPiFromPrim(J, covH, &conTilPi); } @@ -262,9 +247,9 @@ TaskStatus MomentPrim2ConImpl(T *rc, IndexDomain domain) { c.Prim2Con(J, covH, conTilPi, &E, &covF); - v(b, cE(ispec), k, j, i) = sdetgam * E; - for (int idir = dirB.s; idir <= dirB.e; ++idir) { - v(b, cF(ispec, idir), k, j, i) = sdetgam * covF(idir); + v(b, cr::E(ispec), k, j, i) = sdetgam * E; + for (int idir = 0; idir <= 2; ++idir) { + v(b, cr::F(ispec, idir), k, j, i) = sdetgam * covF(idir); } }); @@ -513,6 +498,7 @@ TaskStatus CalculateFluxesImpl(T *rc) { namespace pr = radmoment_prim; namespace ir = radmoment_internal; + // TODO(@astrtobarker) move to sparse packs PackIndexMap imap_ql, imap_qr, imap; std::vector vars{ir::ql::name(), ir::qr::name(), ir::ql_v::name(), ir::qr_v::name(), ir::dJ::name(), ir::kappaH::name()}; @@ -520,6 +506,9 @@ TaskStatus CalculateFluxesImpl(T *rc) { auto v = rc->PackVariablesAndFluxes(vars, flxs, imap); + PackIndexMap imap_fl; + auto v_fl = rc->PackVariablesAndFluxes(flxs, flxs, imap_fl); + auto idx_qlv = imap.GetFlatIdx(ir::ql_v::name()); auto idx_qrv = imap.GetFlatIdx(ir::qr_v::name()); auto idx_ql = imap.GetFlatIdx(ir::ql::name()); @@ -527,8 +516,8 @@ TaskStatus CalculateFluxesImpl(T *rc) { auto idx_dJ = imap.GetFlatIdx(ir::dJ::name()); auto idx_kappaH = imap.GetFlatIdx(ir::kappaH::name()); - auto idx_Ff = imap.GetFlatIdx(cr::F::name()); - auto idx_Ef = imap.GetFlatIdx(cr::E::name()); + auto idx_Ff = imap_fl.GetFlatIdx(cr::F::name()); + auto idx_Ef = imap_fl.GetFlatIdx(cr::E::name()); auto num_species = rad_pkg->Param("num_species"); @@ -737,14 +726,14 @@ TaskStatus CalculateFluxesImpl(T *rc) { 0.5 * sdetgam * (conFl(idir) + conFr(idir) + sigspeed * (El - Er)); SPACELOOP(ii) { - v.flux(idir_in, idx_Ff(ispec, ii), k, j, i) = + v_fl.flux(idir_in, idx_Ff(ispec, ii), k, j, i) = 0.5 * sdetgam * (Pl(idir, ii) + Pr(idir, ii) + sigspeed * (covFl(ii) - covFr(ii))); } if (sdetgam < robust::SMALL()) { - v.flux(idir_in, idx_Ef(ispec), k, j, i) = 0.0; - SPACELOOP(ii) v.flux(idir_in, idx_Ff(ispec, ii), k, j, i) = 0.0; + v_fl.flux(idir_in, idx_Ef(ispec), k, j, i) = 0.0; + SPACELOOP(ii) v_fl.flux(idir_in, idx_Ff(ispec, ii), k, j, i) = 0.0; } } }); diff --git a/src/radiation/moments_source.cpp b/src/radiation/moments_source.cpp index 38173b8d5..4617f72f0 100644 --- a/src/radiation/moments_source.cpp +++ b/src/radiation/moments_source.cpp @@ -987,5 +987,13 @@ TaskStatus MomentFluidSource(T *rc, Real dt, bool update_fluid) { } template TaskStatus MomentFluidSource>(MeshBlockData *, Real, bool); +template <> +TaskStatus MomentFluidSource>(MeshData *md, Real dt, + bool update_fluid) { + for (const auto &mbd : md->GetAllBlockData()) { + MomentFluidSource(mbd.get(), dt, update_fluid); + } + return TaskStatus::complete; +} } // namespace radiation diff --git a/src/radiation/monte_carlo.cpp b/src/radiation/monte_carlo.cpp index 85c66f8b1..4f1b143ce 100644 --- a/src/radiation/monte_carlo.cpp +++ b/src/radiation/monte_carlo.cpp @@ -19,6 +19,7 @@ using Geometry::CoordSysMeshBlock; using Geometry::NDFULL; +using parthenon::MakePackDescriptor; namespace radiation { @@ -88,14 +89,14 @@ void SetWeight(Mesh *pmesh) { auto &phoebus_pkg = pmesh->packages.Get("phoebus"); auto &unit_conv = phoebus_pkg->Param("unit_conv"); auto rad = pmesh->packages.Get("radiation"); - const auto nusamp = rad->Param>("nusamp"); + const auto nusamp0 = rad->Param("nu_min"); auto &code_constants = phoebus_pkg->Param("code_constants"); const Real sim_vol = PhoebusUtils::GetRegionVolume(pmesh->mesh_size); const Real h_code = code_constants.h; const Real dNtot = rad->Param("tune_emission") / (std::pow(sim_vol, 1. / 3.) * 1.0); // Note: may need a dt term here? - rad->UpdateParam("wgtC", rad->Param("Jtot") / (h_code * dNtot) * nusamp[0]); + rad->UpdateParam("wgtC", rad->Param("Jtot") / (h_code * dNtot) * nusamp0); } TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, @@ -103,6 +104,8 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, namespace p = fluid_prim; namespace c = fluid_cons; namespace iv = internal_variables; + namespace mci = monte_carlo_internal; + namespace mcc = monte_carlo_core; auto opac = pmb->packages.Get("opacity"); auto rad = pmb->packages.Get("radiation"); auto swarm = sc->Get("monte_carlo"); @@ -151,23 +154,13 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, const Real h_code = code_constants.h; const Real mp_code = code_constants.mp; - std::vector vars( - {p::density::name(), p::temperature::name(), p::ye::name(), p::velocity::name(), - "dNdlnu_max", "dNdlnu", "dN", "Ns", iv::Gcov::name(), iv::Gye::name()}); - PackIndexMap imap; - auto v = rc->PackVariables(vars, imap); - const int pye = imap[p::ye::name()].first; - const int pdens = imap[p::density::name()].first; - const int ptemp = imap[p::temperature::name()].first; - const int pvlo = imap[p::velocity::name()].first; - const int pvhi = imap[p::velocity::name()].second; - const int idNdlnu = imap["dNdlnu"].first; - const int idNdlnu_max = imap["dNdlnu_max"].first; - const int idN = imap["dN"].first; - const int iNs = imap["Ns"].first; - const int Gcov_lo = imap[iv::Gcov::name()].first; - const int Gcov_hi = imap[iv::Gcov::name()].second; - const int Gye = imap[iv::Gye::name()].first; + Mesh *pmesh = rc->GetMeshPointer(); + auto &resolved_pkgs = pmesh->resolved_packages; + static auto desc = + MakePackDescriptor( + resolved_pkgs.get()); + auto v = desc.GetPack(rc); // TODO(BRR) update this dynamically somewhere else. Get a reasonable starting value Real wgtC = rad->Param("wgtC"); @@ -175,10 +168,10 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, pmb->par_for( "MonteCarloZeroFiveForce", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { - for (int mu = Gcov_lo; mu <= Gcov_lo + 3; mu++) { - v(mu, k, j, i) = 0.; + for (int mu = 0; mu <= 3; mu++) { + v(0, iv::Gcov(mu), k, j, i) = 0.; } - v(Gye, k, j, i) = 0.; + v(0, iv::Gye(), k, j, i) = 0.; }); for (int sidx = 0; sidx < num_species; sidx++) { @@ -188,9 +181,9 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, KOKKOS_LAMBDA(const int k, const int j, const int i) { auto rng_gen = rng_pool.get_state(); Real detG = geom.DetG(CellLocation::Cent, k, j, i); - const Real &dens = v(pdens, k, j, i); - const Real &temp = v(ptemp, k, j, i); - const Real &ye = v(pye, k, j, i); + const Real &dens = v(0, p::density(), k, j, i); + const Real &temp = v(0, p::temperature(), k, j, i); + const Real &ye = v(0, p::ye(), k, j, i); Real dN = 0.; Real dNdlnu_max = 0.; @@ -204,15 +197,14 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, // Note that factors of nu in numerator and denominator cancel Real dNdlnu = Jnu * d3x * detG / (h_code * wgt); - // TODO(BRR) use FlatIdx - v(idNdlnu + sidx + n * num_species, k, j, i) = dNdlnu; + v(0, mci::dNdlnu(sidx + n * num_species), k, j, i) = dNdlnu; if (dNdlnu > dNdlnu_max) { dNdlnu_max = dNdlnu; } } for (int n = 0; n <= nu_bins; n++) { - v(idNdlnu + sidx + n * num_species, k, j, i) /= dNdlnu_max; + v(0, mci::dNdlnu(sidx + n * num_species), k, j, i) /= dNdlnu_max; } // Trapezoidal rule @@ -224,7 +216,7 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, (h_code * GetWeight(wgtC, nu1)) * dlnu; dN *= d3x * detG * dt; - v(idNdlnu_max + sidx, k, j, i) = dNdlnu_max; + v(0, mci::dNdlnu_max(sidx), k, j, i) = dNdlnu_max; int Ns = static_cast(dN); if (dN - Ns > rng_gen.drand()) { @@ -232,8 +224,8 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, } // TODO(BRR) Use a ParArrayND instead of these weird static_casts - v(idN + sidx, k, j, i) = dN; - v(iNs + sidx, k, j, i) = static_cast(Ns); + v(0, mci::dN(sidx), k, j, i) = dN; + v(0, mci::Ns(sidx), k, j, i) = static_cast(Ns); rng_pool.free_state(rng_gen); }); } @@ -244,7 +236,7 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, parthenon::loop_pattern_mdrange_tag, "MonteCarloReduceParticleCreation", DevExecSpace(), 0, num_species - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int sidx, const int k, const int j, const int i, Real &dNtot) { - dNtot += v(idN + sidx, k, j, i); + dNtot += v(0, mci::dN(sidx), k, j, i); }, Kokkos::Sum(dNtot)); @@ -260,14 +252,14 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, KOKKOS_LAMBDA(const int sidx, const int k, const int j, const int i) { auto rng_gen = rng_pool.get_state(); - Real dN_upd = wgtCfac * v(idN + sidx, k, j, i); + Real dN_upd = wgtCfac * v(0, mci::dN(sidx), k, j, i); int Ns = static_cast(dN_upd); if (dN_upd - Ns > rng_gen.drand()) { Ns++; } // TODO(BRR) Use a ParArrayND instead of these weird static_casts - v(iNs + sidx, k, j, i) = static_cast(Ns); + v(0, mci::Ns(sidx), k, j, i) = static_cast(Ns); rng_pool.free_state(rng_gen); }); int Nstot = 0; @@ -275,7 +267,7 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, parthenon::loop_pattern_mdrange_tag, "MonteCarloReduceParticleCreationNs", DevExecSpace(), 0, num_species - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int sidx, const int k, const int j, const int i, int &Nstot) { - Nstot += static_cast(v(iNs + sidx, k, j, i)); + Nstot += static_cast(v(0, mci::Ns(sidx), k, j, i)); }, Kokkos::Sum(Nstot)); @@ -284,22 +276,23 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, const auto new_particles_context = swarm->AddEmptyParticles(Nstot); - auto &t = swarm->Get("t").Get(); - auto &x = swarm->Get("x").Get(); - auto &y = swarm->Get("y").Get(); - auto &z = swarm->Get("z").Get(); - auto &k0 = swarm->Get("k0").Get(); - auto &k1 = swarm->Get("k1").Get(); - auto &k2 = swarm->Get("k2").Get(); - auto &k3 = swarm->Get("k3").Get(); - auto &weight = swarm->Get("weight").Get(); - auto &swarm_species = swarm->Get("species").Get(); - auto swarm_d = swarm->GetDeviceContext(); + // monte carlo swarm pack + static constexpr auto swarm_name = "monte_carlo"; + static const auto desc_mc = + MakeSwarmPackDescriptor( + swarm_name); + auto pack_mc = desc_mc.GetPack(rc); + + // NOTE: Currently cannot pack Real and int valued swarms together, so species is + // separated + static const auto desc_species = MakeSwarmPackDescriptor(swarm_name); + auto pack_species = desc_species.GetPack(rc); // Calculate array of starting index for each zone to compute particles ParArrayND starting_index("Starting index", 3, nx_k, nx_j, nx_i); auto starting_index_h = starting_index.GetHostMirror(); - auto dN = rc->Get("Ns").data; + auto dN = rc->Get(mci::Ns::name()).data; auto dN_h = dN.GetHostMirrorAndCopy(); int index = 0; for (int sidx = 0; sidx < num_species; sidx++) { @@ -314,7 +307,7 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, } starting_index.DeepCopy(starting_index_h); - auto dNdlnu = rc->Get("dNdlnu").data; + auto dNdlnu = rc->Get(mci::dNdlnu::name()).data; // auto dNdlnu_max = rc->Get("dNdlnu_max").data; // Loop over zones and generate appropriate number of particles in each zone @@ -324,15 +317,17 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, pmb->par_for( "MonteCarloSourceParticles", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { + const auto swarm_d = pack_mc.GetContext(0); // Create tetrad transformation once per zone Real Gcov[4][4]; geom.SpacetimeMetric(CellLocation::Cent, k, j, i, Gcov); Real Ucon[4]; - Real vel[3] = {v(pvlo, k, j, i), v(pvlo + 1, k, j, i), v(pvlo + 2, k, j, i)}; + Real vel[3] = {v(0, p::velocity(0), k, j, i), v(0, p::velocity(1), k, j, i), + v(0, p::velocity(2), k, j, i)}; GetFourVelocity(vel, geom, CellLocation::Cent, k, j, i, Ucon); Geometry::Tetrads Tetrads(Ucon, Gcov); Real detG = geom.DetG(CellLocation::Cent, k, j, i); - int dNs = v(iNs + sidx, k, j, i); + int dNs = v(0, mci::Ns(sidx), k, j, i); auto rng_gen = rng_pool.get_state(); // Loop over particles to create in this zone @@ -341,15 +336,15 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, starting_index(sidx, k - kb.s, j - jb.s, i - ib.s) + n); // Set particle species - swarm_species(m) = static_cast(s); + pack_species(0, mcc::species(), m) = static_cast(s); // Create particles at initial time - t(m) = t0; + pack_mc(0, mcc::t(), m) = t0; // Create particles at zone centers - x(m) = minx_i + (i - ib.s + 0.5) * dx_i; - y(m) = minx_j + (j - jb.s + 0.5) * dx_j; - z(m) = minx_k + (k - kb.s + 0.5) * dx_k; + pack_mc(0, swarm_position::x(), m) = minx_i + (i - ib.s + 0.5) * dx_i; + pack_mc(0, swarm_position::y(), m) = minx_j + (j - jb.s + 0.5) * dx_j; + pack_mc(0, swarm_position::z(), m) = minx_k + (k - kb.s + 0.5) * dx_k; // Sample energy and set weight Real lnu; @@ -368,7 +363,7 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, } while (rng_gen.drand() > prob); Real nu = exp(lnu); - weight(m) = GetWeight(wgtC / wgtCfac, nu); + pack_mc(0, mcc::weight(), m) = GetWeight(wgtC / wgtCfac, nu); // Encode frequency and randomly sample direction Real E = nu * h_code; @@ -379,16 +374,18 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, Real K_coord[4]; Tetrads.TetradToCoordCov(K_tetrad, K_coord); - k0(m) = K_coord[0]; - k1(m) = K_coord[1]; - k2(m) = K_coord[2]; - k3(m) = K_coord[3]; + pack_mc(0, mcc::k0(), m) = K_coord[0]; + pack_mc(0, mcc::k1(), m) = K_coord[1]; + pack_mc(0, mcc::k2(), m) = K_coord[2]; + pack_mc(0, mcc::k3(), m) = K_coord[3]; - for (int mu = Gcov_lo; mu <= Gcov_lo + 3; mu++) { + for (int mu = 0; mu <= 3; mu++) { // detG is in both numerator and denominator - v(mu, k, j, i) -= 1. / (d3x * dt) * weight(m) * K_coord[mu - Gcov_lo]; + v(0, iv::Gcov(mu), k, j, i) -= + 1. / (d3x * dt) * pack_mc(0, mcc::weight(), m) * K_coord[mu]; } - v(Gye, k, j, i) -= LeptonSign(s) / (d3x * dt) * Ucon[0] * weight(m) * mp_code; + v(0, iv::Gye(), k, j, i) -= LeptonSign(s) / (d3x * dt) * Ucon[0] * + pack_mc(0, mcc::weight(), m) * mp_code; } // for n rng_pool.free_state(rng_gen); @@ -399,7 +396,8 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, pmb->par_for( "MonteCarloRemoveEmittedParticles", 0, num_species - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int sidx, const int k, const int j, const int i) { - int dNs = v(iNs + sidx, k, j, i); + const auto swarm_d = pack_mc.GetContext(0); + int dNs = v(0, mci::Ns(sidx), k, j, i); // Loop over particles to create in this zone for (int n = 0; n < static_cast(dNs); n++) { const int m = new_particles_context.GetNewParticleIndex( @@ -418,6 +416,7 @@ TaskStatus MonteCarloTransport(MeshBlock *pmb, MeshBlockData *rc, namespace p = fluid_prim; namespace c = fluid_cons; namespace iv = internal_variables; + namespace mcc = monte_carlo_core; auto rad = pmb->packages.Get("radiation"); auto swarm = sc->Get("monte_carlo"); auto opac = pmb->packages.Get("opacity"); @@ -437,17 +436,19 @@ TaskStatus MonteCarloTransport(MeshBlock *pmb, MeshBlockData *rc, const Real &dx_k = pmb->coords.Dxf<3>(pmb->cellbounds.ks(IndexDomain::interior)); const Real d4x = dx_i * dx_j * dx_k * dt; auto geom = Geometry::GetCoordinateSystem(rc); - auto &t = swarm->Get("t").Get(); - auto &x = swarm->Get("x").Get(); - auto &y = swarm->Get("y").Get(); - auto &z = swarm->Get("z").Get(); - auto &k0 = swarm->Get("k0").Get(); - auto &k1 = swarm->Get("k1").Get(); - auto &k2 = swarm->Get("k2").Get(); - auto &k3 = swarm->Get("k3").Get(); - auto &weight = swarm->Get("weight").Get(); - auto &swarm_species = swarm->Get("species").Get(); - auto swarm_d = swarm->GetDeviceContext(); + + // monte carlo swarm pack + static constexpr auto swarm_name = "monte_carlo"; + static const auto desc_mc = + MakeSwarmPackDescriptor( + swarm_name); + auto pack_mc = desc_mc.GetPack(rc); + + // NOTE: Currently cannot pack Real and int valued swarms together, so species is + // separated + static const auto desc_species = MakeSwarmPackDescriptor(swarm_name); + auto pack_species = desc_species.GetPack(rc); auto phoebus_pkg = pmb->packages.Get("phoebus"); auto &unit_conv = phoebus_pkg->Param("unit_conv"); @@ -456,46 +457,52 @@ TaskStatus MonteCarloTransport(MeshBlock *pmb, MeshBlockData *rc, const Real h_code = code_constants.h; const Real mp_code = code_constants.mp; - std::vector vars({p::density::name(), p::ye::name(), p::velocity::name(), - p::temperature::name(), iv::Gcov::name(), - iv::Gye::name()}); - PackIndexMap imap; - auto v = rc->PackVariables(vars, imap); - const int prho = imap[p::density::name()].first; - const int iye = imap[p::ye::name()].first; - const int ivlo = imap[p::velocity::name()].first; - const int ivhi = imap[p::velocity::name()].second; - const int itemp = imap[p::temperature::name()].first; - const int iGcov_lo = imap[iv::Gcov::name()].first; - const int iGcov_hi = imap[iv::Gcov::name()].second; - const int iGye = imap[iv::Gye::name()].first; + Mesh *pmesh = rc->GetMeshPointer(); + auto &resolved_pkgs = pmesh->resolved_packages; + static auto desc = MakePackDescriptor(resolved_pkgs.get()); + auto v = desc.GetPack(rc); ParArray1D num_interactions("Number interactions", 2); pmb->par_for( - "MonteCarloTransport", 0, swarm->GetMaxActiveIndex(), KOKKOS_LAMBDA(const int n) { + "MonteCarloTransport", 0, pack_mc.GetMaxFlatIndex(), KOKKOS_LAMBDA(const int idx) { + const auto [b, n] = pack_mc.GetBlockParticleIndices(idx); + const auto swarm_d = pack_mc.GetContext(b); if (swarm_d.IsActive(n)) { + Real &x = pack_mc(b, swarm_position::x(), n); + Real &y = pack_mc(b, swarm_position::y(), n); + Real &z = pack_mc(b, swarm_position::z(), n); + Real &t = pack_mc(b, mcc::t(), n); + Real &k0 = pack_mc(b, mcc::k0(), n); + Real &k1 = pack_mc(b, mcc::k1(), n); + Real &k2 = pack_mc(b, mcc::k2(), n); + Real &k3 = pack_mc(b, mcc::k3(), n); + Real &weight = pack_mc(b, mcc::weight(), n); + auto rng_gen = rng_pool.get_state(); - auto s = static_cast(swarm_species(n)); + auto s = static_cast(pack_species(b, mcc::species(), n)); // TODO(BRR) Get u^mu, evaluate -k.u - const Real nu = -k0(n) / h_code; + const Real nu = -k0 / h_code; // TODO(BRR) Get K^0 via metric - Real Kcon0 = -k0(n); + Real Kcon0 = -k0; // Real dlam = dt / Kcon0; int k, j, i; - swarm_d.Xtoijk(x(n), y(n), z(n), i, j, k); + swarm_d.Xtoijk(x, y, z, i, j, k); - Real alphanu = opacities.AbsorptionCoefficient( - v(prho, k, j, i), v(itemp, k, j, i), v(iye, k, j, i), s, nu); + Real alphanu = opacities.AbsorptionCoefficient(v(b, p::density(), k, j, i), + v(b, p::temperature(), k, j, i), + v(b, p::ye(), k, j, i), s, nu); Real dtau_abs = alphanu * dt; // c = 1 in code units - Real vel[3] = {v(ivlo, k, j, i), v(ivlo + 1, k, j, i), v(ivlo + 2, k, j, i)}; - Real W = GetLorentzFactor(vel, geom, CellLocation::Cent, k, j, i); - Real alpha = geom.Lapse(CellLocation::Cent, k, j, i); + Real vel[3] = {v(b, p::velocity(0), k, j, i), v(b, p::velocity(1), k, j, i), + v(b, p::velocity(2), k, j, i)}; + Real W = GetLorentzFactor(vel, geom, CellLocation::Cent, b, k, j, i); + Real alpha = geom.Lapse(CellLocation::Cent, b, k, j, i); Real Ucon0 = robust::ratio(W, std::abs(alpha)); bool absorbed = false; @@ -506,15 +513,12 @@ TaskStatus MonteCarloTransport(MeshBlock *pmb, MeshBlockData *rc, Real xabs = -log(rng_gen.drand()); if (xabs <= dtau_abs) { // Deposit energy-momentum and lepton number in fluid - Kokkos::atomic_add(&(v(iGcov_lo, k, j, i)), 1. / d4x * weight(n) * k0(n)); - Kokkos::atomic_add(&(v(iGcov_lo + 1, k, j, i)), - 1. / d4x * weight(n) * k1(n)); - Kokkos::atomic_add(&(v(iGcov_lo + 2, k, j, i)), - 1. / d4x * weight(n) * k2(n)); - Kokkos::atomic_add(&(v(iGcov_lo + 3, k, j, i)), - 1. / d4x * weight(n) * k3(n)); - Kokkos::atomic_add(&(v(iGye, k, j, i)), - LeptonSign(s) / d4x * weight(n) * mp_code * Ucon0); + Kokkos::atomic_add(&(v(b, iv::Gcov(0), k, j, i)), 1. / d4x * weight * k0); + Kokkos::atomic_add(&(v(b, iv::Gcov(1), k, j, i)), 1. / d4x * weight * k1); + Kokkos::atomic_add(&(v(b, iv::Gcov(2), k, j, i)), 1. / d4x * weight * k2); + Kokkos::atomic_add(&(v(b, iv::Gcov(3), k, j, i)), 1. / d4x * weight * k3); + Kokkos::atomic_add(&(v(b, iv::Gye(), k, j, i)), + LeptonSign(s) / d4x * weight * mp_code * Ucon0); absorbed = true; Kokkos::atomic_add(&(num_interactions[0]), 1.); @@ -523,10 +527,10 @@ TaskStatus MonteCarloTransport(MeshBlock *pmb, MeshBlockData *rc, } if (absorbed == false) { - PushParticle(t(n), x(n), y(n), z(n), k0(n), k1(n), k2(n), k3(n), dt, geom); + PushParticle(t, x, y, z, k0, k1, k2, k3, dt, geom); bool on_current_mesh_block = true; - swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); + swarm_d.GetNeighborBlockIndex(n, x, y, z, on_current_mesh_block); } rng_pool.free_state(rng_gen); @@ -661,33 +665,42 @@ TaskStatus MonteCarloUpdateTuning(Mesh *pmesh, std::vector *resolution, return TaskStatus::complete; } -TaskStatus MonteCarloCountCommunicatedParticles(MeshBlock *pmb, - int *particles_outstanding) { - auto &swarm = pmb->swarm_data.Get()->Get("monte_carlo"); +// TODO(BLB) make sure this works if/when monte carlo solver in driver is updated... +TaskStatus MonteCarloCountCommunicatedParticles(const BlockList_t &blocks, + const double tf_, bool *done) { + int num_unfinished = 0; + for (auto &block : blocks) { + auto sc = block->meshblock_data.Get()->GetSwarmData(); + auto swarm = sc->Get("my_particles"); + int max_active_index = swarm->GetMaxActiveIndex(); - *particles_outstanding += swarm->num_particles_sent_; + auto &t = swarm->Get("t").Get(); - // Reset communication flags - for (int n = 0; n < pmb->pbval->nneighbor; n++) { - auto &nb = pmb->pbval->neighbor[n]; - swarm->vbswarm->bd_var_.flag[nb.bufid] = BoundaryStatus::waiting; -#ifdef MPI_PARALLEL - swarm->vbswarm->bd_var_.req_send[nb.bufid] = MPI_REQUEST_NULL; -#endif // MPI_PARALLEL + auto swarm_d = swarm->GetDeviceContext(); + + const auto &tf = tf_; + + parthenon::par_reduce( + PARTHENON_AUTO_LABEL, 0, max_active_index, + KOKKOS_LAMBDA(const int n, int &num_unfinished) { + if (swarm_d.IsActive(n)) { + if (t(n) < tf) { + num_unfinished++; + } + } + }, + Kokkos::Sum(num_unfinished)); } #ifdef MPI_PARALLEL - pmb->exec_space.fence(); - for (int n = 0; n < pmb->pbval->nneighbor; n++) { - auto &nb = pmb->pbval->neighbor[n]; - if (nb.snb.rank != Globals::my_rank) { - PARTHENON_MPI_CHECK( - MPI_Wait(&(swarm->vbswarm->bd_var_.req_send[nb.bufid]), MPI_STATUS_IGNORE)); - } - swarm->vbswarm->bd_var_.req_send[nb.bufid] = MPI_REQUEST_NULL; - } + MPI_Allreduce(MPI_IN_PLACE, &num_unfinished, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); +#endif // MPI_PARALLEL -#endif + if (num_unfinished > 0) { + *done = false; + } else { + *done = true; + } return TaskStatus::complete; } @@ -697,9 +710,9 @@ TaskStatus InitializeCommunicationMesh(const std::string swarmName, // Boundary transfers on same MPI proc are blocking #ifdef MPI_PARALLEL for (auto &block : blocks) { - auto swarm = block->swarm_data.Get()->Get(swarmName); - for (int n = 0; n < block->pbval->nneighbor; n++) { - auto &nb = block->pbval->neighbor[n]; + auto swarm = block->meshblock_data.Get()->GetSwarmData()->Get(swarmName); + for (int n = 0; n < block->neighbors.size(); n++) { + auto &nb = block->neighbors[n]; #ifdef MPI_PARALLEL swarm->vbswarm->bd_var_.req_send[nb.bufid] = MPI_REQUEST_NULL; #endif @@ -710,7 +723,7 @@ TaskStatus InitializeCommunicationMesh(const std::string swarmName, // Reset boundary statuses for (auto &block : blocks) { auto &pmb = block; - auto sc = pmb->swarm_data.Get(); + auto sc = pmb->meshblock_data.Get()->GetSwarmData(); auto swarm = sc->Get(swarmName); swarm->ResetCommunication(); } diff --git a/src/radiation/radiation.cpp b/src/radiation/radiation.cpp index cb0cb633e..27bb9b8ff 100644 --- a/src/radiation/radiation.cpp +++ b/src/radiation/radiation.cpp @@ -62,12 +62,11 @@ std::shared_ptr Initialize(ParameterInput *pin) { bool is_frequency_dependent = (method == "monte_carlo" || method == "mocmc"); // Set which neutrino species to include in simulation - bool do_nu_electron = pin->GetOrAddBoolean("radiation", "do_nu_electron", true); + bool do_nu_electron = DO_NU_ELECTRON; params.Add("do_nu_electron", do_nu_electron); - bool do_nu_electron_anti = - pin->GetOrAddBoolean("radiation", "do_nu_electron_anti", true); + bool do_nu_electron_anti = DO_NU_ELECTRON_ANTI; params.Add("do_nu_electron_anti", do_nu_electron_anti); - bool do_nu_heavy = pin->GetOrAddBoolean("radiation", "do_nu_heavy", true); + bool do_nu_heavy = DO_NU_HEAVY; params.Add("do_nu_heavy", do_nu_heavy); // Boundary conditions @@ -96,17 +95,18 @@ std::shared_ptr Initialize(ParameterInput *pin) { params.Add("dlnu", dlnu); } + // TODO(BLB) With compile time num species this can be a static array. std::vector species; - if (do_nu_electron) { + if (DO_NU_ELECTRON) { species.push_back(RadiationType::NU_ELECTRON); } - if (do_nu_electron_anti) { + if (DO_NU_ELECTRON_ANTI) { species.push_back(RadiationType::NU_ELECTRON_ANTI); } - if (do_nu_heavy) { + if (DO_NU_HEAVY) { species.push_back(RadiationType::NU_HEAVY); } - const int num_species = species.size(); + static constexpr int num_species = PHOEBUS_NUM_SPECIES; params.Add("num_species", num_species); params.Add("species", species); @@ -144,20 +144,26 @@ std::shared_ptr Initialize(ParameterInput *pin) { } if (method == "mocmc") { - std::string swarm_name = "mocmc"; + static constexpr auto swarm_name = "mocmc"; Metadata swarm_metadata({Metadata::Provides}); physics->AddSwarm(swarm_name, swarm_metadata); Metadata real_swarmvalue_metadata({Metadata::Real}); - physics->AddSwarmValue("t", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("mu_lo", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("mu_hi", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("phi_lo", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("phi_hi", swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(mocmc_core::t::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(mocmc_core::mu_lo::name(), swarm_name, + real_swarmvalue_metadata); + physics->AddSwarmValue(mocmc_core::mu_hi::name(), swarm_name, + real_swarmvalue_metadata); + physics->AddSwarmValue(mocmc_core::phi_lo::name(), swarm_name, + real_swarmvalue_metadata); + physics->AddSwarmValue(mocmc_core::phi_hi::name(), swarm_name, + real_swarmvalue_metadata); Metadata fourv_swarmvalue_metadata({Metadata::Real}, std::vector{4}); - physics->AddSwarmValue("ncov", swarm_name, fourv_swarmvalue_metadata); + physics->AddSwarmValue(mocmc_core::ncov::name(), swarm_name, + fourv_swarmvalue_metadata); Metadata Inu_swarmvalue_metadata({Metadata::Real}, std::vector{num_species, nu_bins}); - physics->AddSwarmValue("Inuinv", swarm_name, Inu_swarmvalue_metadata); + physics->AddSwarmValue(mocmc_core::Inuinv::name(), swarm_name, + Inu_swarmvalue_metadata); // Boundary temperatures for outflow sample boundary conditions const std::string ix1_bc = pin->GetOrAddString("phoebus", "ix1_bc", "None"); @@ -224,28 +230,31 @@ std::shared_ptr Initialize(ParameterInput *pin) { } if (method == "monte_carlo") { - std::string swarm_name = "monte_carlo"; + namespace mci = monte_carlo_internal; + namespace mcc = monte_carlo_core; + + constexpr static auto swarm_name = "monte_carlo"; Metadata swarm_metadata({Metadata::Provides}); physics->AddSwarm(swarm_name, swarm_metadata); Metadata real_swarmvalue_metadata({Metadata::Real}); - physics->AddSwarmValue("t", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("k0", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("k1", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("k2", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("k3", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("weight", swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(mcc::t::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(mcc::k0::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(mcc::k1::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(mcc::k2::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(mcc::k3::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(mcc::weight::name(), swarm_name, real_swarmvalue_metadata); Metadata int_swarmvalue_metadata({Metadata::Integer}); - physics->AddSwarmValue("species", swarm_name, int_swarmvalue_metadata); + physics->AddSwarmValue(mcc::species::name(), swarm_name, int_swarmvalue_metadata); Metadata mspecies_scalar = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector{num_species}); - physics->AddField("dNdlnu_max", mspecies_scalar); - physics->AddField("dN", mspecies_scalar); - physics->AddField("Ns", mspecies_scalar); + physics->AddField(mci::dNdlnu_max::name(), mspecies_scalar); + physics->AddField(mci::dN::name(), mspecies_scalar); + physics->AddField(mci::Ns::name(), mspecies_scalar); std::vector dNdlnu_size{num_species, params.Get("nu_bins") + 1}; Metadata mdNdlnu = Metadata({Metadata::Cell, Metadata::OneCopy}, dNdlnu_size); - physics->AddField("dNdlnu", mdNdlnu); + physics->AddField(mci::dNdlnu::name(), mdNdlnu); // Parameters controlling automatic sampling resolution. // This system targets 1 scattering per light crossing time. @@ -551,7 +560,6 @@ TaskStatus ApplyRadiationFourForce(MeshData *rc, const double dt) { MakePackDescriptor( resolved_pkgs.get()); - PackIndexMap imap; auto v = desc.GetPack(rc); const int nblocks = v.GetNBlocks(); @@ -580,6 +588,7 @@ Real EstimateTimestepBlock(MeshBlockData *rc) { // Note that this is still used for the cooling function even though that option // contains no transport. This is useful for consistency between methods. + Mesh *pmesh = rc->GetMeshPointer(); auto pmb = rc->GetBlockPointer(); IndexRange ib = rc->GetBoundsI(IndexDomain::interior); IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); @@ -592,11 +601,9 @@ Real EstimateTimestepBlock(MeshBlockData *rc) { auto geom = Geometry::GetCoordinateSystem(rc); - PackIndexMap imap; - std::vector vars{ir::kappaH::name(), p::velocity::name()}; - auto v = rc->PackVariables(vars, imap); - auto idx_v = imap.GetFlatIdx(p::velocity::name()); - auto idx_kappaH = imap.GetFlatIdx(ir::kappaH::name(), false); + auto &resolved_pkgs = pmesh->resolved_packages; + static auto desc = MakePackDescriptor(resolved_pkgs.get()); + auto v = desc.GetPack(rc); auto num_species = rad->Param("num_species"); @@ -615,14 +622,15 @@ Real EstimateTimestepBlock(MeshBlockData *rc) { for (int ispec = 0; ispec < num_species; ispec++) { - const Real kappaH = idx_kappaH.IsValid() ? v(idx_kappaH(ispec), k, j, i) : 0.; + const Real kappaH = + v.Contains(0, ir::kappaH()) ? v(0, ir::kappaH(ispec), k, j, i) : 0.; for (int d = 0; d < ndim; d++) { // Signal speeds (assume (i.e. somewhat overestimate, esp. for large opt. // depth) cs_rad = 1) const Real sigp = alpha * std::sqrt(con_gamma(d, d)) - con_beta(d); const Real sigm = -alpha * std::sqrt(con_gamma(d, d)) - con_beta(d); - const Real asym_sigl = alpha * v(idx_v(d), k, j, i) - con_beta(d); + const Real asym_sigl = alpha * v(0, p::velocity(d), k, j, i) - con_beta(d); const Real rad_speed = std::max(std::fabs(sigm), std::fabs(sigp)); const Real asym_speed = std::fabs(asym_sigl); diff --git a/src/radiation/radiation.hpp b/src/radiation/radiation.hpp index b1935d810..83c0839b1 100644 --- a/src/radiation/radiation.hpp +++ b/src/radiation/radiation.hpp @@ -116,8 +116,8 @@ TaskStatus MonteCarloEstimateParticles(MeshBlock *pmb, MeshBlockData *rc, SwarmContainer *sc, const Real t0, const Real dt, Real *dNtot); -TaskStatus MonteCarloCountCommunicatedParticles(MeshBlock *pmb, - int *particles_outstanding); +TaskStatus MonteCarloCountCommunicatedParticles(const BlockList_t &blocks, + const double tf_, bool *done); TaskStatus InitializeCommunicationMesh(const std::string swarmName, const BlockList_t &blocks); @@ -154,16 +154,20 @@ template TaskStatus MOCMCTransport(T *rc, const Real dt); template -TaskStatus MOCMCSampleBoundaries(T *rc); +TaskStatus MOCMCSampleBoundaries(T *rc_base, T *rc); +// TaskStatus MOCMCSampleBoundaries(T *rc); template -TaskStatus MOCMCReconstruction(T *rc); +TaskStatus MOCMCReconstruction(T *rc_base, T *rc); +// TaskStatus MOCMCReconstruction(T *rc); template -TaskStatus MOCMCEddington(T *rc); +TaskStatus MOCMCEddington(T *rc_base, T *rc); +// TaskStatus MOCMCEddington(T *rc); template -TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid); +// TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid); +TaskStatus MOCMCFluidSource(T *rc_base, T *rc, const Real dt, const bool update_fluid); TaskStatus MOCMCUpdateParticleCount(Mesh *pmesh, std::vector *resolution); diff --git a/src/tracers/tracers.cpp b/src/tracers/tracers.cpp index 4d314c351..148bdfe65 100644 --- a/src/tracers/tracers.cpp +++ b/src/tracers/tracers.cpp @@ -20,6 +20,7 @@ namespace tracers { using namespace parthenon::package::prelude; +using parthenon::MakePackDescriptor; std::shared_ptr Initialize(ParameterInput *pin) { auto physics = std::make_shared("tracers"); @@ -39,93 +40,102 @@ std::shared_ptr Initialize(ParameterInput *pin) { physics->AddParam<>("rng_pool", rng_pool); // Add swarm of tracers - std::string swarm_name = "tracers"; + static constexpr auto swarm_name = "tracers"; Metadata swarm_metadata({Metadata::Provides, Metadata::None}); physics->AddSwarm(swarm_name, swarm_metadata); Metadata real_swarmvalue_metadata({Metadata::Real}); physics->AddSwarmValue("id", swarm_name, Metadata({Metadata::Integer})); // thermo variables - physics->AddSwarmValue("rho", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("temperature", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("ye", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("entropy", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("pressure", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("energy", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("vel_x", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("vel_y", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("vel_z", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("lorentz", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("lapse", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("detgamma", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("shift_x", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("shift_y", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("shift_z", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("mass", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("bernoulli", swarm_name, real_swarmvalue_metadata); + namespace tv = tracer_variables; + physics->AddSwarmValue(tv::rho::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::temperature::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::ye::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::entropy::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::pressure::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::energy::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::vel_x::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::vel_y::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::vel_z::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::lorentz::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::lapse::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::detgamma::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::shift_x::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::shift_y::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::shift_z::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::mass::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::bernoulli::name(), swarm_name, real_swarmvalue_metadata); + + // TEST + Metadata Inu_swarmvalue_metadata({Metadata::Real}, std::vector{3, 100}); + physics->AddSwarmValue(tv::test::name(), swarm_name, Inu_swarmvalue_metadata); const bool mhd = pin->GetOrAddBoolean("fluid", "mhd", false); if (mhd) { - physics->AddSwarmValue("B_x", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("B_y", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("B_z", swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::B_x::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::B_y::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::B_z::name(), swarm_name, real_swarmvalue_metadata); } return physics; } // Initialize -TaskStatus AdvectTracers(MeshBlockData *rc, const Real dt) { +TaskStatus AdvectTracers(MeshData *rc, const Real dt) { namespace p = fluid_prim; auto *pmb = rc->GetParentPointer(); - auto &sc = pmb->swarm_data.Get(); - auto &swarm = sc->Get("tracers"); - - const auto ndim = pmb->pmy_mesh->ndim; - auto &x = swarm->Get("x").Get(); - auto &y = swarm->Get("y").Get(); - auto &z = swarm->Get("z").Get(); + static const auto ndim = pmb->ndim; - auto swarm_d = swarm->GetDeviceContext(); + // tracer swarm pack + static constexpr auto swarm_name = "tracers"; + static const auto desc_tracer = + MakeSwarmPackDescriptor( + swarm_name); + auto pack_tracers = desc_tracer.GetPack(rc); - const std::vector vars = {p::velocity::name()}; + static const auto vars = {p::velocity::name()}; + static const auto desc = MakePackDescriptor(rc); PackIndexMap imap; auto pack = rc->PackVariables(vars, imap); + // TODO(BLB): move to sparse packs. requires reworking tracer_rhs const int pvel_lo = imap[p::velocity::name()].first; const int pvel_hi = imap[p::velocity::name()].second; - auto geom = Geometry::GetCoordinateSystem(rc); + const auto geom = Geometry::GetCoordinateSystem(rc); // update loop. RK2 - const int max_active_index = swarm->GetMaxActiveIndex(); - pmb->par_for( - "Advect Tracers", 0, max_active_index, KOKKOS_LAMBDA(const int n) { + parthenon::par_for( + "Advect Tracers", 0, pack_tracers.GetMaxFlatIndex(), KOKKOS_LAMBDA(const int idx) { + const auto [b, n] = pack_tracers.GetBlockParticleIndices(idx); + const auto swarm_d = pack_tracers.GetContext(b); if (swarm_d.IsActive(n)) { - int k, j, i; - swarm_d.Xtoijk(x(n), y(n), z(n), i, j, k); - Real rhs1, rhs2, rhs3; + const Real x = pack_tracers(b, swarm_position::x(), n); + const Real y = pack_tracers(b, swarm_position::y(), n); + const Real z = pack_tracers(b, swarm_position::z(), n); + // predictor - tracers_rhs(pack, geom, pvel_lo, pvel_hi, ndim, dt, x(n), y(n), z(n), rhs1, - rhs2, rhs3); - const Real kx = x(n) + 0.5 * dt * rhs1; - const Real ky = y(n) + 0.5 * dt * rhs2; - const Real kz = z(n) + 0.5 * dt * rhs3; + tracers_rhs(pack, geom, pvel_lo, pvel_hi, ndim, dt, x, y, z, rhs1, rhs2, rhs3); + const Real kx = x + 0.5 * dt * rhs1; + const Real ky = y + 0.5 * dt * rhs2; + const Real kz = z + 0.5 * dt * rhs3; // corrector tracers_rhs(pack, geom, pvel_lo, pvel_hi, ndim, dt, kx, ky, kz, rhs1, rhs2, rhs3); - x(n) += rhs1 * dt; - y(n) += rhs2 * dt; - z(n) += rhs3 * dt; + + // update positions + pack_tracers(b, swarm_position::x(), n) += rhs1 * dt; + pack_tracers(b, swarm_position::y(), n) += rhs2 * dt; + pack_tracers(b, swarm_position::z(), n) += rhs3 * dt; bool on_current_mesh_block = true; - swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); + swarm_d.GetNeighborBlockIndex(n, x, y, z, on_current_mesh_block); } }); @@ -141,46 +151,26 @@ TaskStatus AdvectTracers(MeshBlockData *rc, const Real dt) { void FillTracers(MeshBlockData *rc) { using namespace LCInterp; namespace p = fluid_prim; + namespace tv = tracer_variables; auto *pmb = rc->GetParentPointer(); auto fluid = pmb->packages.Get("fluid"); - auto &sc = pmb->swarm_data.Get(); + auto &sc = rc->GetSwarmData(); auto &swarm = sc->Get("tracers"); auto eos = pmb->packages.Get("eos")->Param("d.EOS"); const auto mhd = fluid->Param("mhd"); - // pull swarm vars - auto &x = swarm->Get("x").Get(); - auto &y = swarm->Get("y").Get(); - auto &z = swarm->Get("z").Get(); - auto &v1 = swarm->Get("vel_x").Get(); - auto &v2 = swarm->Get("vel_y").Get(); - auto &v3 = swarm->Get("vel_z").Get(); - auto B1 = v1.Get(); - auto B2 = v1.Get(); - auto B3 = v1.Get(); - if (mhd) { - B1 = swarm->Get("B_x").Get(); - B2 = swarm->Get("B_y").Get(); - B3 = swarm->Get("B_z").Get(); - } - auto &s_rho = swarm->Get("rho").Get(); - auto &s_temperature = swarm->Get("temperature").Get(); - auto &s_ye = swarm->Get("ye").Get(); - auto &s_entropy = swarm->Get("entropy").Get(); - auto &s_energy = swarm->Get("energy").Get(); - auto &s_lorentz = swarm->Get("lorentz").Get(); - auto &s_lapse = swarm->Get("lapse").Get(); - auto &s_shift_x = swarm->Get("shift_x").Get(); - auto &s_shift_y = swarm->Get("shift_y").Get(); - auto &s_shift_z = swarm->Get("shift_z").Get(); - auto &s_detgamma = swarm->Get("detgamma").Get(); - auto &s_pressure = swarm->Get("pressure").Get(); - auto &s_bernoulli = swarm->Get("bernoulli").Get(); - - auto swarm_d = swarm->GetDeviceContext(); + // tracer swarm pack + static constexpr auto swarm_name = "tracers"; + static auto desc_tracers = MakeSwarmPackDescriptor< + swarm_position::x, swarm_position::y, swarm_position::z, tv::vel_x, tv::vel_y, + tv::vel_z, tv::rho, tv::temperature, tv::ye, tv::entropy, tv::energy, tv::lorentz, + tv::lapse, tv::shift_x, tv::shift_y, tv::shift_z, tv::detgamma, tv::pressure, + tv::bernoulli, tv::B_x, tv::B_y, tv::B_z, tv::test>(swarm_name); + auto pack_tracers = desc_tracers.GetPack(rc); + // hydro vars pack std::vector vars = {p::density::name(), p::temperature::name(), p::velocity::name(), p::energy::name(), p::pressure::name()}; @@ -203,37 +193,41 @@ void FillTracers(MeshBlockData *rc) { auto geom = Geometry::GetCoordinateSystem(rc); // update loop. - const int max_active_index = swarm->GetMaxActiveIndex(); pmb->par_for( - "Fill Tracers", 0, max_active_index, KOKKOS_LAMBDA(const int n) { + "Fill Tracers", 0, pack_tracers.GetMaxFlatIndex(), KOKKOS_LAMBDA(const int idx) { + const auto [b, n] = pack_tracers.GetBlockParticleIndices(idx); + const auto swarm_d = pack_tracers.GetContext(b); + if (swarm_d.IsActive(n)) { - int k, j, i; - swarm_d.Xtoijk(x(n), y(n), z(n), i, j, k); + + const Real x = pack_tracers(b, swarm_position::x(), n); + const Real y = pack_tracers(b, swarm_position::y(), n); + const Real z = pack_tracers(b, swarm_position::z(), n); // geom quantities Real gcov4[4][4]; - geom.SpacetimeMetric(0.0, x(n), y(n), z(n), gcov4); - Real lapse = geom.Lapse(0.0, x(n), y(n), z(n)); + geom.SpacetimeMetric(0.0, x, y, z, gcov4); + Real lapse = geom.Lapse(0.0, x, y, z); Real shift[3]; - geom.ContravariantShift(0.0, x(n), y(n), z(n), shift); - const Real gdet = geom.DetGamma(0.0, x(n), y(n), z(n)); + geom.ContravariantShift(0.0, x, y, z, shift); + const Real gdet = geom.DetGamma(0.0, x, y, z); // Interpolate - const Real Wvel_X1 = LCInterp::Do(0, x(n), y(n), z(n), pack, pvel_lo); - const Real Wvel_X2 = LCInterp::Do(0, x(n), y(n), z(n), pack, pvel_lo + 1); - const Real Wvel_X3 = LCInterp::Do(0, x(n), y(n), z(n), pack, pvel_hi); + const Real Wvel_X1 = LCInterp::Do(0, x, y, z, pack, pvel_lo); + const Real Wvel_X2 = LCInterp::Do(0, x, y, z, pack, pvel_lo + 1); + const Real Wvel_X3 = LCInterp::Do(0, x, y, z, pack, pvel_hi); Real B_X1 = 0.0; Real B_X2 = 0.0; Real B_X3 = 0.0; if (mhd) { - B_X1 = LCInterp::Do(0, x(n), y(n), z(n), pack, pB_lo); - B_X2 = LCInterp::Do(0, x(n), y(n), z(n), pack, pB_lo + 1); - B_X3 = LCInterp::Do(0, x(n), y(n), z(n), pack, pB_hi); + B_X1 = LCInterp::Do(0, x, y, z, pack, pB_lo); + B_X2 = LCInterp::Do(0, x, y, z, pack, pB_lo + 1); + B_X3 = LCInterp::Do(0, x, y, z, pack, pB_hi); } - const Real rho = LCInterp::Do(0, x(n), y(n), z(n), pack, prho); - const Real temperature = LCInterp::Do(0, x(n), y(n), z(n), pack, ptemp); - const Real energy = LCInterp::Do(0, x(n), y(n), z(n), pack, penergy); - const Real pressure = LCInterp::Do(0, x(n), y(n), z(n), pack, ppres); + const Real rho = LCInterp::Do(0, x, y, z, pack, prho); + const Real temperature = LCInterp::Do(0, x, y, z, pack, ptemp); + const Real energy = LCInterp::Do(0, x, y, z, pack, penergy); + const Real pressure = LCInterp::Do(0, x, y, z, pack, ppres); const Real Wvel[] = {Wvel_X1, Wvel_X2, Wvel_X3}; const Real W = phoebus::GetLorentzFactor(Wvel, gcov4); const Real vel_X1 = Wvel_X1 / W; @@ -242,7 +236,7 @@ void FillTracers(MeshBlockData *rc) { Real ye; Real lambda[2] = {0.0, 0.0}; if (pye > 0) { - ye = LCInterp::Do(0, x(n), y(n), z(n), pack, pye); + ye = LCInterp::Do(0, x, y, z, pack, pye); lambda[1] = ye; } else { ye = 0.0; @@ -255,30 +249,31 @@ void FillTracers(MeshBlockData *rc) { const Real bernoulli = -(W / lapse) * h - 1.0; // store - s_rho(n) = rho; - s_temperature(n) = temperature; - s_ye(n) = ye; - s_energy(n) = energy; - s_entropy(n) = entropy; - v1(n) = vel_X1; - v2(n) = vel_X3; - v3(n) = vel_X2; - s_shift_x(n) = shift[0]; - s_shift_y(n) = shift[1]; - s_shift_z(n) = shift[2]; - s_lapse(n) = lapse; - s_lorentz(n) = W; - s_detgamma(n) = gdet; - s_pressure(n) = pressure; - s_bernoulli(n) = bernoulli; + pack_tracers(b, tv::rho(), n) = rho; + pack_tracers(b, tv::temperature(), n) = temperature; + pack_tracers(b, tv::ye(), n) = ye; + pack_tracers(b, tv::energy(), n) = energy; + pack_tracers(b, tv::entropy(), n) = entropy; + pack_tracers(b, tv::vel_x(), n) = vel_X1; + pack_tracers(b, tv::vel_y(), n) = vel_X2; + pack_tracers(b, tv::vel_z(), n) = vel_X3; + pack_tracers(b, tv::shift_x(), n) = shift[0]; + pack_tracers(b, tv::shift_y(), n) = shift[1]; + pack_tracers(b, tv::shift_z(), n) = shift[2]; + pack_tracers(b, tv::lapse(), n) = lapse; + pack_tracers(b, tv::lorentz(), n) = W; + pack_tracers(b, tv::detgamma(), n) = gdet; + pack_tracers(b, tv::pressure(), n) = pressure; + pack_tracers(b, tv::bernoulli(), n) = bernoulli; + pack_tracers(b, tv::test(2 + 5), n) = 1.1238; if (mhd) { - B1(n) = B_X1; - B2(n) = B_X2; - B3(n) = B_X3; + pack_tracers(b, tv::B_x(), n) = B_X1; + pack_tracers(b, tv::B_y(), n) = B_X2; + pack_tracers(b, tv::B_z(), n) = B_X3; } bool on_current_mesh_block = true; - swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); + swarm_d.GetNeighborBlockIndex(n, x, y, z, on_current_mesh_block); } }); diff --git a/src/tracers/tracers.hpp b/src/tracers/tracers.hpp index 595696d37..d4cf0d45f 100644 --- a/src/tracers/tracers.hpp +++ b/src/tracers/tracers.hpp @@ -76,7 +76,7 @@ KOKKOS_INLINE_FUNCTION void tracers_rhs(Pack &pack, Geometry &geom, const int pv } } -TaskStatus AdvectTracers(MeshBlockData *rc, const Real dt); +TaskStatus AdvectTracers(MeshData *rc, const Real dt); void FillTracers(MeshBlockData *rc); diff --git a/tst/regression/bondi.py b/tst/regression/bondi.py index c57ca6ce4..8a07083fc 100755 --- a/tst/regression/bondi.py +++ b/tst/regression/bondi.py @@ -46,6 +46,7 @@ input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args=[], geometry="FMKS", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/friedmann.py b/tst/regression/friedmann.py index 3a521fb4d..4be34c50a 100755 --- a/tst/regression/friedmann.py +++ b/tst/regression/friedmann.py @@ -39,6 +39,7 @@ variables=["p.density", "p.energy"], input_file=args.input, executable=args.executable, + cmake_extra_args=[], geometry="FLRW", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/homogeneous_sphere.py b/tst/regression/homogeneous_sphere.py index aaf3935c3..c6fe5f4a2 100755 --- a/tst/regression/homogeneous_sphere.py +++ b/tst/regression/homogeneous_sphere.py @@ -46,12 +46,14 @@ modified_inputs["radiation_advection/vx"] = 0.0 modified_inputs["radiation_advection/radius"] = 1.0 +cmake_extra_args = ["-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off", "-DPHOEBUS_DO_NU_HEAVY=Off"] code = rt.gold_comparison( variables=["r.p.J", "r.p.H"], input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args=cmake_extra_args, geometry="SphericalMinkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/linear_modes.gold b/tst/regression/linear_modes.gold index 571fe701f..29941ee79 100644 --- a/tst/regression/linear_modes.gold +++ b/tst/regression/linear_modes.gold @@ -1,1232 +1,1232 @@ -1.000005632655319499e+00 -1.000002760808383684e+00 -9.999972996949745285e-01 -9.999943565266583567e-01 -9.999962802371848980e-01 -1.000001649208487331e+00 -1.000005620483547686e+00 -1.000004150727892061e+00 -9.999989029694753562e-01 -9.999945052546052038e-01 -9.999951245617294537e-01 -9.999999666961703060e-01 -1.000004883968774960e+00 -1.000005153020244952e+00 -1.000000599734518048e+00 -9.999955005058174251e-01 -9.999944869738877085e-01 -9.999982906507698388e-01 -1.000003652558603573e+00 -1.000005628965033422e+00 -1.000002244374423199e+00 -9.999968100084866141e-01 -9.999943649241527233e-01 -9.999967457378612368e-01 -1.000002185287540257e+00 -1.000005642989404420e+00 -1.000003719999225105e+00 -9.999983506855543158e-01 -9.999943790434682178e-01 -9.999954615166134486e-01 -1.000000534120877971e+00 -1.000005228450156824e+00 -1.000004875483002431e+00 -1.000000033345461725e+00 -9.999951164123052072e-01 -9.999946323421767813e-01 -9.999988401746482403e-01 -1.000004088373919187e+00 -1.000005589093612368e+00 -1.000001709466136646e+00 -9.999963473093869126e-01 -9.999943761293018651e-01 -9.999972393619038780e-01 -1.000002700093578278e+00 -1.000005644115180781e+00 -1.000003254457284108e+00 -9.999978146264839607e-01 -9.999943568631461943e-01 -9.999958495106286982e-01 -1.000001096971994130e+00 -1.000005495535843947e+00 -1.000004875450465347e+00 -9.999994658331410324e-01 -9.999947709284359165e-01 -9.999948465842898315e-01 -9.999994003330860970e-01 -1.000004499339977482e+00 -1.000005513960985581e+00 -1.000001159938610495e+00 -9.999959108496043747e-01 -9.999944056627527722e-01 -9.999977557711768883e-01 -1.000003189886431443e+00 -1.000005636202187853e+00 -1.000002760808383684e+00 -9.999972998744335317e-01 -9.999943559924534542e-01 -9.999962802054666033e-01 -1.000001649278303040e+00 -1.000005620628910519e+00 -1.000004538623388450e+00 -9.999989029677013308e-01 -9.999945028654575463e-01 -9.999951245676423905e-01 -9.999999667078377508e-01 -1.000004883749974649e+00 -1.000005367520260258e+00 -1.000000599734518048e+00 -9.999955009783095816e-01 -9.999944860242014899e-01 -9.999982906563524843e-01 -1.000003652592456493e+00 -1.000005626612752385e+00 -1.000002760794253653e+00 -9.999968099938674193e-01 -9.999943548367293822e-01 -9.999967457329563825e-01 -1.000002185325947535e+00 -1.000005642809836504e+00 -1.000004150697972882e+00 -9.999983506855543158e-01 -9.999943783760981697e-01 -9.999954615040358430e-01 -1.000000534158342669e+00 -1.000005228311358962e+00 -1.000005153536246860e+00 -1.000000033271464250e+00 -9.999951155518908008e-01 -9.999946330534588146e-01 -9.999988401823904915e-01 -1.000004088583912543e+00 -1.000005588376624566e+00 -1.000002244346593239e+00 -9.999963473093869126e-01 -9.999943743511141570e-01 -9.999972393620715216e-01 -1.000002700174408510e+00 -1.000005643771175512e+00 -1.000003719963573623e+00 -9.999978144589464213e-01 -9.999943569558090717e-01 -9.999958495208238762e-01 -1.000001096972067183e+00 -1.000005495200696481e+00 -1.000004875516921521e+00 -1.000000033248674702e+00 -9.999947709284359165e-01 -9.999948467812824759e-01 -9.999994003578855928e-01 -1.000004499543848624e+00 -1.000005513648303479e+00 -1.000001709420340612e+00 -9.999959110738979584e-01 -9.999944112094069082e-01 -9.999977557586826604e-01 -1.000003189792319835e+00 -1.000005635582378538e+00 -1.000003254426833799e+00 -9.999972996949745285e-01 -9.999943559924534542e-01 -9.999962802051375332e-01 -1.000001649284675054e+00 -1.000005621498109232e+00 -1.000004538640674401e+00 -9.999994658109249146e-01 -9.999945035818743611e-01 -9.999951246263967253e-01 -9.999999666998068415e-01 -1.000004883316716331e+00 -1.000005367234137355e+00 -1.000001159884960744e+00 -9.999955005058174251e-01 -9.999944860242014899e-01 -9.999982906307588459e-01 -1.000003652605387927e+00 -1.000005624334296517e+00 -1.000002760792013445e+00 -9.999972997383014262e-01 -9.999943614488633337e-01 -9.999967457341174537e-01 -1.000002185393187970e+00 -1.000005642143994233e+00 -1.000004150708544870e+00 -9.999989029061402412e-01 -9.999943790434682178e-01 -9.999954615040358430e-01 -1.000000534081493475e+00 -1.000005228295131277e+00 -1.000005153196704244e+00 -1.000000599682605351e+00 -9.999951170624205021e-01 -9.999946333588105585e-01 -9.999988401642738722e-01 -1.000004088706652361e+00 -1.000005595181765905e+00 -1.000002244379371907e+00 -9.999968101646514729e-01 -9.999943761293018651e-01 -9.999972393620715216e-01 -1.000002700108178377e+00 -1.000005642882210610e+00 -1.000003720001498175e+00 -9.999983506322962512e-01 -9.999943596158442638e-01 -9.999958494945223597e-01 -1.000001096905887232e+00 -1.000005495644664899e+00 -1.000004875475132726e+00 -1.000000033312556269e+00 -9.999951170408881707e-01 -9.999948465842898315e-01 -9.999994003578855928e-01 -1.000004499522637147e+00 -1.000005512885039138e+00 -1.000001709484434675e+00 -9.999963472312672907e-01 -9.999944020628815933e-01 -9.999977557613660695e-01 -1.000003189716275331e+00 -1.000005639801760493e+00 -1.000003254451671930e+00 -9.999978145217349734e-01 -9.999943565266583567e-01 -9.999962802054666033e-01 -1.000001649284675054e+00 -1.000005620784933713e+00 -1.000004538678776589e+00 -9.999994658770350320e-01 -9.999947714133969878e-01 -9.999951245978127012e-01 -9.999999667342928111e-01 -1.000004883138554179e+00 -1.000005367418171476e+00 -1.000001159934313488e+00 -9.999959109807341484e-01 -9.999944869738877085e-01 -9.999982906563524843e-01 -1.000003652605387927e+00 -1.000005619484136021e+00 -1.000002760804144186e+00 -9.999972997261444840e-01 -9.999943567613929218e-01 -9.999967457339060672e-01 -1.000002185403360278e+00 -1.000005642411317064e+00 -1.000004150689643767e+00 -9.999989030539646606e-01 -9.999945040795561457e-01 -9.999954615166134486e-01 -1.000000534158342669e+00 -1.000005228295131277e+00 -1.000005153882767006e+00 -1.000000599705133553e+00 -9.999955003056808511e-01 -9.999946334501745859e-01 -9.999988401734757337e-01 -1.000004088639646405e+00 -1.000005593670743709e+00 -1.000002244387094841e+00 -9.999968100584690767e-01 -9.999943599343450451e-01 -9.999972393619038780e-01 -1.000002700174408510e+00 -1.000005642882210610e+00 -1.000003719982992090e+00 -9.999983506613785433e-01 -9.999943787217002722e-01 -9.999958495178359330e-01 -1.000001096954051372e+00 -1.000005496312835085e+00 -1.000004875478968547e+00 -1.000000033287579804e+00 -9.999951168764663612e-01 -9.999946331056811522e-01 -9.999994003330860970e-01 -1.000004499543848624e+00 -1.000005512885039138e+00 -1.000001709477368106e+00 -9.999963473939638137e-01 -9.999943780502227320e-01 -9.999977557667091288e-01 -1.000003189830621864e+00 -1.000005637416242932e+00 -1.000003254456625301e+00 -9.999978144411570957e-01 -9.999943589123083676e-01 -9.999962802371848980e-01 -1.000001649278303040e+00 -1.000005621498109232e+00 -1.000004538678776589e+00 -9.999994658622803900e-01 -9.999947722049712384e-01 -9.999948463156687062e-01 -9.999999666883998550e-01 -1.000004883059735450e+00 -1.000005366258192918e+00 -1.000001159913265214e+00 -9.999959109662573953e-01 -9.999944021225303237e-01 -9.999982906507698388e-01 -1.000003652592456493e+00 -1.000005624334296517e+00 -1.000002760804144186e+00 -9.999972997290592636e-01 -9.999943563340063513e-01 -9.999962802322818201e-01 -1.000002185330710391e+00 -1.000005640836852283e+00 -1.000004150751921728e+00 -9.999989029865307133e-01 -9.999945034238885144e-01 -9.999951246482331468e-01 -1.000000534120877971e+00 -1.000005228311358962e+00 -1.000005153196704244e+00 -1.000000599705133553e+00 -9.999955002942768623e-01 -9.999944865621374079e-01 -9.999988401662210924e-01 -1.000004088660306545e+00 -1.000005597738118590e+00 -1.000002244387627748e+00 -9.999968100301691587e-01 -9.999943593569411426e-01 -9.999967457255370951e-01 -1.000002700093578278e+00 -1.000005643771175512e+00 -1.000003720001498175e+00 -9.999983506613785433e-01 -9.999943800857316045e-01 -9.999954615287282023e-01 -1.000001096930779321e+00 -1.000005496538477034e+00 -1.000004875481131261e+00 -1.000000033332056004e+00 -9.999951162199782750e-01 -9.999946336329671448e-01 -9.999988401593464804e-01 -1.000004499339977482e+00 -1.000005513648303479e+00 -1.000001709484434675e+00 -9.999963473939638137e-01 -9.999943649475041552e-01 -9.999972393553445693e-01 -1.000003189733285724e+00 -1.000005644760634915e+00 -1.000003254444560064e+00 -9.999978145439466504e-01 -9.999943571569811507e-01 -9.999958494474100457e-01 -1.000001649208487331e+00 -1.000005620628910519e+00 -1.000004538640674401e+00 -9.999994658770350320e-01 -9.999947722049712384e-01 -9.999948468252387590e-01 -9.999994003217230754e-01 -1.000004883005678025e+00 -1.000005366956320030e+00 -1.000001159915184790e+00 -9.999959114697550833e-01 -9.999944071037120130e-01 -9.999977557691764884e-01 -1.000003652558603573e+00 -1.000005626612752385e+00 -1.000002760792013445e+00 -9.999972997261444840e-01 -9.999943563340063513e-01 -9.999962801845050375e-01 -1.000001649218661415e+00 -1.000005641543503909e+00 -1.000004150665656066e+00 -9.999989029996332324e-01 -9.999945047613947313e-01 -9.999951247846718960e-01 -9.999999667155246019e-01 -1.000005228450156824e+00 -1.000005153536246860e+00 -1.000000599682605351e+00 -9.999955003056808511e-01 -9.999944865621374079e-01 -9.999982906611800670e-01 -1.000004088755765075e+00 -1.000005598271136220e+00 -1.000002244348032532e+00 -9.999968100916505342e-01 -9.999943640835056158e-01 -9.999967457235865442e-01 -1.000002185347521610e+00 -1.000005644115180781e+00 -1.000003719963573623e+00 -9.999983506322962512e-01 -9.999943787217002722e-01 -9.999954615287282023e-01 -1.000000534153414167e+00 -1.000005496702192076e+00 -1.000004875448733621e+00 -1.000000033272859801e+00 -9.999951163138114385e-01 -9.999946326145957576e-01 -9.999988401736024102e-01 -1.000004088469371721e+00 -1.000005513960985581e+00 -1.000001709420340612e+00 -9.999963472312672907e-01 -9.999943780502227320e-01 -9.999972393553445693e-01 -1.000002700212754503e+00 -1.000005645330762860e+00 -1.000003254434076005e+00 -9.999978145630771253e-01 -9.999943576993874927e-01 -9.999958495086429533e-01 -1.000001096954989510e+00 -1.000005620483547686e+00 -1.000004538623388450e+00 -9.999994658109249146e-01 -9.999947714133969878e-01 -9.999948463156687062e-01 -9.999994003217230754e-01 -1.000004499638560196e+00 -1.000005366980012411e+00 -1.000001159875671508e+00 -9.999959114989783737e-01 -9.999944078390725632e-01 -9.999977557599983857e-01 -1.000003189869540288e+00 -1.000005628965033422e+00 -1.000002760794253653e+00 -9.999972997383014262e-01 -9.999943567613929218e-01 -9.999962802322818201e-01 -1.000001649218661415e+00 -1.000005620196233735e+00 -1.000004150703476480e+00 -9.999989029721221279e-01 -9.999945044132163563e-01 -9.999951245066918126e-01 -9.999999666814920474e-01 -1.000004883573349268e+00 -1.590367539041457537e-06 -1.850912408548692162e-07 --1.388671263919384155e-06 --1.701402812589428375e-06 --5.275245537920439392e-07 -1.127207486787609383e-06 -1.725513118501249725e-06 -6.926330706003840947e-07 --9.843926778009791954e-07 --1.741847553483955041e-06 --1.004286466214067983e-06 -6.744469082427657453e-07 -1.741808948583900944e-06 -1.148018083313236715e-06 --5.089801355624572556e-07 --1.734300937657732912e-06 --1.400619844916229366e-06 -1.648243680517588008e-07 -1.613372567611652862e-06 -1.504581194016223573e-06 -1.027057519218336824e-08 --1.506965207472381200e-06 --1.656413222052862091e-06 --3.579387071900327507e-07 -1.261832909686356920e-06 -1.726530510927856447e-06 -5.275505036001899248e-07 --1.127283476069760551e-06 --1.738991200931393030e-06 --8.519558241342447753e-07 -8.332683681776776070e-07 -1.741543189546330788e-06 -1.004348184969015373e-06 --6.744859534995672576e-07 --1.741128142201565000e-06 --1.280871119453824414e-06 -3.385081253452617528e-07 -1.695890104207147870e-06 -1.400661979984600656e-06 --1.648645316816077902e-07 --1.613580376379654210e-06 --1.590413693388966491e-06 --1.850598198757230344e-07 -1.388569531686582505e-06 -1.701594467880254092e-06 -3.579597281925138896e-07 --1.262011462870778688e-06 --1.727044134608643941e-06 --6.925960486877090200e-07 -9.843413329179845028e-07 -1.741951384656060736e-06 -8.520003192028401959e-07 --8.334344866947607621e-07 --1.741882352976100373e-06 --1.147954552841384033e-06 -5.089651065550712443e-07 -1.734499890489319603e-06 -1.400657029138366716e-06 --3.385102729909338517e-07 --1.696068296688998337e-06 --1.504578035365303085e-06 --1.027792459486570175e-08 -1.506889434950186658e-06 -1.656390619447936956e-06 -1.850579395021460447e-07 --1.388808107645545854e-06 --1.701449497581291826e-06 --5.275302993430160891e-07 -1.127129333673650356e-06 -1.739139556716176440e-06 -6.926275968399265429e-07 --9.843286574677046199e-07 --1.738833342797969636e-06 --1.004290676010142285e-06 -6.744290190047559574e-07 -1.741608192516847482e-06 -1.280938630120717270e-06 --5.089962051112259242e-07 --1.734595962635921047e-06 --1.400613937252086356e-06 -1.648187793660271902e-07 -1.613277429219103755e-06 -1.590372361279220448e-06 -1.026314296979694244e-08 --1.507199462265586231e-06 --1.656498548720226997e-06 --3.579462891905561392e-07 -1.261786933656091673e-06 -1.727036888705413617e-06 -6.926335290490646140e-07 --1.127305826552776190e-06 --1.734666481337623469e-06 --8.519554876257128211e-07 -8.332817916134800692e-07 -1.741570517560293097e-06 -1.148022411434485064e-06 --6.745069916869999551e-07 --1.740893134080086125e-06 --1.280863526478345686e-06 -3.385007361086511826e-07 -1.695925012644290348e-06 -1.504607413513912528e-06 --1.648357971565737352e-07 --1.613293906584121117e-06 --1.590371049557741721e-06 --1.850698468576710981e-07 -1.388584273480631005e-06 -1.701619351222137502e-06 -5.275664409881972963e-07 --1.261890835872668019e-06 --1.726952582184757882e-06 --6.925939866257344096e-07 -9.843387710206114620e-07 -1.741827775922222212e-06 -1.004355416021173411e-06 --8.332887313101556406e-07 --1.741493618313671237e-06 --1.147954127011784552e-06 -5.089536504049958006e-07 -1.734405951374010952e-06 -1.400665060755398470e-06 --1.648147270693007677e-07 --1.696026167294465809e-06 --1.504516816255540277e-06 --1.028570061164219897e-08 -1.506966518059593660e-06 -1.656403618202785754e-06 -3.579714705349063427e-07 --1.388536818637949710e-06 --1.701761632405453606e-06 --5.275227261590045916e-07 -1.127176760524699177e-06 -1.738899137111849145e-06 -8.520110766617004472e-07 --9.843379307777278582e-07 --1.741806689421038490e-06 --1.004295364866518706e-06 -6.743993238623917024e-07 -1.741251734869187301e-06 -1.280939977094745456e-06 --3.385363576962681561e-07 --1.734477658715999417e-06 --1.400598327554048858e-06 -1.648342583341889763e-07 -1.613392098859980280e-06 -1.590351309578219981e-06 -1.850793076461093879e-07 --1.506990158082079201e-06 --1.656348914111078322e-06 --3.579349890952416767e-07 -1.261748446896396022e-06 -1.726954622768805722e-06 -6.926407633502231383e-07 --9.843909964545581157e-07 --1.737771171945459064e-06 --8.519585978329506073e-07 -8.332596957649066877e-07 -1.741883632174271865e-06 -1.148021789733716254e-06 --5.089646463921921279e-07 --1.741108470592745993e-06 --1.280864836161371862e-06 -3.385379663369058124e-07 -1.696309625747959570e-06 -1.504609226014816693e-06 -1.029970426659359921e-08 --1.613350527324997949e-06 --1.590302724258083714e-06 --1.850619383300565058e-07 -1.388450251314567264e-06 -1.701199448021600421e-06 -5.275580778973880034e-07 --1.127288025163849908e-06 --1.726563348516592840e-06 --6.925950592092194851e-07 -9.843972034917504489e-07 -1.741986631626964369e-06 -1.004347296229844827e-06 --6.744476113853419399e-07 --1.741626188449242454e-06 --1.147951644189643252e-06 -5.089802152540511556e-07 -1.734464186140258252e-06 -1.400680181652775881e-06 --1.648386383732958587e-07 --1.613458033031303169e-06 --1.504577710435353542e-06 --1.028608283527124951e-08 -1.506928969757272522e-06 -1.656256499901239233e-06 -3.579525644479939614e-07 --1.261872342220463811e-06 --1.701610355631484363e-06 --5.275271344634275999e-07 -1.127198902199723192e-06 -1.738062302039671397e-06 -8.520017892937290931e-07 --8.333539228821495243e-07 --1.739065206108945372e-06 --1.004291260290182602e-06 -6.744101958019806226e-07 -1.740803442299043410e-06 -1.280937036840117323e-06 --3.385636630176459017e-07 --1.696207774224466475e-06 --1.400602580196387577e-06 -1.648167040171728496e-07 -1.613295641558825584e-06 -1.590472857050408798e-06 -1.850645034035481419e-07 --1.388546378061128693e-06 --1.656265722646537589e-06 --3.579423946587949524e-07 -1.261757825222560384e-06 -1.727582033192779542e-06 -6.926310400115253552e-07 --9.844404762875896280e-07 --1.740739744440883911e-06 --8.519585516470582605e-07 -8.332593986167771238e-07 -1.742379863835736858e-06 -1.148012931887122658e-06 --5.090402292842649397e-07 --1.734407858537690999e-06 --1.280868972595230153e-06 -3.384999746174194464e-07 -1.696185029077676697e-06 -1.504623543518964511e-06 -1.028223359323060603e-08 --1.507075600978145329e-06 --1.590475652061203466e-06 --1.850639222710864662e-07 -1.388557421002558836e-06 -1.701684251364760647e-06 -5.275512265607479172e-07 --1.127286411336689808e-06 --1.737395596505640334e-06 --6.925970737149970458e-07 -9.843206542173732238e-07 -1.741832499020683313e-06 -1.004348077440545941e-06 --6.744556533661682400e-07 --1.740774003594915025e-06 --1.147947685019042703e-06 -5.089704719231065102e-07 -1.734460067992656717e-06 -1.400663872155841370e-06 --1.648378112527584822e-07 --1.613128151027974754e-06 --1.590478454220823225e-06 --1.027735954261626783e-08 -1.506985638456615903e-06 -1.656094988251150500e-06 -3.579615998315412844e-07 --1.261784008739208135e-06 --1.726486447474969463e-06 --5.275304255328125951e-07 -1.127163313841367346e-06 -1.738319230226277259e-06 -8.520030490968967593e-07 --8.333047458639765553e-07 --1.741716031385066482e-06 --1.004287515401397678e-06 -6.744269820492903251e-07 -1.741063303667368066e-06 -1.280944197266247243e-06 --3.385375740156370755e-07 --1.696114681665394089e-06 --1.504607320341262842e-06 -1.648247549435342491e-07 -1.613279411811395556e-06 -1.590272262240389021e-06 -1.850763445089354181e-07 --1.388543432206800262e-06 --1.701579535502788515e-06 --3.579417799535259882e-07 -1.261825112859412937e-06 -1.726522186015714916e-06 -6.926298797425628815e-07 --9.844451534597710166e-07 --1.739154462062785886e-06 --1.004287950956870439e-06 -8.332800073558788083e-07 -1.741705419103144528e-06 -1.148018190478056903e-06 --5.089998106922283231e-07 --1.734196579128229177e-06 --1.400607469785205336e-06 -3.385147902400103756e-07 -1.696188697615942383e-06 -1.504598469109849392e-06 -1.028602692912153095e-08 --1.507149469115720502e-06 --1.656302740691107855e-06 --1.850615676620913606e-07 -1.388597063993844476e-06 -1.701598914005255437e-06 -5.275488683394952047e-07 --1.127246230087635691e-06 --1.735630160281970369e-06 --8.519545802429056960e-07 -9.843770273985061775e-07 -1.738994245936507767e-06 -1.004344838744613408e-06 --6.744224451044774395e-07 --1.741648171681971972e-06 --1.280885326922608207e-06 -5.089825455281064240e-07 -1.734625715760534243e-06 -1.400662582495071480e-06 --1.648427259867010545e-07 --1.613604219551383351e-06 --1.590309690913316708e-06 --1.850585260142685255e-07 -1.507093017892425353e-06 -1.656433997566742238e-06 -3.579501898796839463e-07 --1.261749084508135586e-06 --1.724793762976749918e-06 --6.925917484604561872e-07 -1.127192347382897711e-06 -1.734482175110829926e-06 -8.520028454191943851e-07 --8.332712925714974420e-07 --1.741297315315238620e-06 --1.147961367548260921e-06 -6.744510731501280489e-07 -1.741062871064385555e-06 -1.280935962263143594e-06 --3.385459381808079238e-07 --1.695877980024644523e-06 --1.504538951390613589e-06 --1.027454453753881087e-08 -1.613311494828473454e-06 -1.590490523396869758e-06 -1.850695136122860865e-07 --1.388564292089949174e-06 --1.701054595687863222e-06 --5.275258794933226751e-07 -1.261802895159962122e-06 -1.725896452052280683e-06 -6.926385417940661657e-07 --9.843876064109259510e-07 --1.742343816791561549e-06 --1.004292673205916525e-06 -6.744367176308645790e-07 -1.741774388432576809e-06 -1.148021657984709214e-06 --5.090048672716291640e-07 --1.734424760616408426e-06 --1.400604165194647486e-06 -1.648261974188203075e-07 -1.696104137809677338e-06 -1.504611812673658253e-06 -1.028752509823473208e-08 --1.506835049173077311e-06 --1.656367124297569766e-06 --3.579465736156574359e-07 -1.388594163091088900e-06 -1.701600197062117496e-06 -5.275662193295627331e-07 --1.127189509723428754e-06 --1.739140251596924240e-06 --8.519555146344949567e-07 -8.332803391730492215e-07 -1.739083119561703616e-06 -1.004355888144604765e-06 --6.744274295560516419e-07 --1.741157556063786170e-06 --1.280874370092632454e-06 -3.384979849638089985e-07 -1.734512488043795267e-06 -1.400653876346155224e-06 --1.648191437038570666e-07 --1.613357878702286843e-06 --1.590415680388212322e-06 --1.850664008729975969e-07 -1.388616654257874299e-06 -1.656410100155959239e-06 -3.579718078982790880e-07 --1.261742194831940182e-06 --1.727519293799483035e-06 --6.925931453648272117e-07 -9.842945020186080504e-07 -1.734544076054722863e-06 -8.520144031590257785e-07 --8.332964747435799942e-07 --1.742269274102159542e-06 --1.147947761448823896e-06 -5.089688992861910061e-07 -1.741034279003410450e-06 -1.280937689492520769e-06 --3.385423636830984105e-07 --1.696046231791591712e-06 --1.504618301607008335e-06 --1.027719594533354700e-08 -1.506805351404585447e-06 -1.590436401080494455e-06 -1.851002823797434524e-07 --1.388659116484511387e-06 --1.701540621950765053e-06 --5.275256108826528761e-07 -1.127166171203310547e-06 -1.725110778275020941e-06 -6.926393770930866481e-07 --9.844323146218771484e-07 --1.741784308782007532e-06 --1.004286311019523964e-06 -6.744500208297343382e-07 -1.741938329515638949e-06 -1.148014876225550924e-06 --5.089785309167015760e-07 --1.734497322243950889e-06 --1.400614927829939636e-06 -1.648214631468875859e-07 -1.613184576308203552e-06 -1.504606788480569668e-06 -1.027859207800901093e-08 --1.507276578279591926e-06 --1.656275217839760236e-06 --3.579405943675265619e-07 -1.261816236083089866e-06 -1.700849706196831590e-06 -5.275493876965313214e-07 --1.127198411639394720e-06 --1.734249179605773024e-06 --8.519561298198576426e-07 -8.332521341154560291e-07 -1.741764638932950718e-06 -1.004347431623458245e-06 --6.744596310186115941e-07 --1.741748314809797923e-06 --1.280869111820689546e-06 -3.385171037921494635e-07 -1.695901592563524869e-06 -1.400678617835233387e-06 --1.648565488028164046e-07 --1.613561807138776761e-06 --1.590478919667941100e-06 --1.850638870732223280e-07 -1.388590675464703539e-06 -1.701433691138633940e-06 -3.579618944853186432e-07 --1.261779512233267517e-06 --1.726504418213852149e-06 --6.925968883290548120e-07 -9.843697821249760282e-07 -1.741881945693986421e-06 -8.520030971635724024e-07 --8.333062008962849171e-07 --1.742331654135019675e-06 --1.147952122313779147e-06 -5.089734434596512497e-07 -1.734662045847583257e-06 -1.280942301929910373e-06 --3.384895853315081937e-07 --1.696161302241622052e-06 --1.504564104073877981e-06 --1.027659593009093002e-08 -1.507009727967658865e-06 -1.656445198085565982e-06 -1.850641972503618116e-07 --1.388693950052137218e-06 --1.701040840053128679e-06 --5.275290161123662693e-07 -1.127187285536431175e-06 -1.734723813921674311e-06 -6.926307761678843503e-07 --9.843077859588004430e-07 --1.739164557000898997e-06 --1.004292273601492378e-06 -6.744365888437088794e-07 -1.742010130835613369e-06 -1.280938872272856700e-06 --5.090256218436111423e-07 --1.734564342862144349e-06 --1.400616732860756689e-06 -1.648170136371230491e-07 -1.613363813253878648e-06 -1.590438272052297480e-06 -1.025305568602594536e-08 --1.506926596493520195e-06 --1.656431298117337317e-06 --3.579410847930974386e-07 -1.261875034073571594e-06 -1.726397781239574585e-06 -5.275604617026915171e-07 --1.127137339160950026e-06 --1.739180713235708337e-06 --8.519550466243470749e-07 -8.332820516221241366e-07 -1.741974380368915069e-06 -1.148021876526727446e-06 --6.744862491138618203e-07 --1.741113204422817864e-06 --1.280873208255707068e-06 -3.385022989464865713e-07 -1.695972778829141366e-06 -1.504589322247046242e-06 --1.648193240548547792e-07 --1.613117730023353339e-06 --1.590520135504460998e-06 --1.850727975776422814e-07 -1.388654221902107873e-06 -1.701326020156905512e-06 -5.275682709626586915e-07 --1.261947505715531078e-06 --1.726078486393322259e-06 --6.925948186026352393e-07 -9.843279579363112002e-07 -1.741660351150165014e-06 -1.004355483758935434e-06 --8.333099001568666338e-07 --1.742964785873333881e-06 --1.147945751530698009e-06 -5.089534599001360521e-07 -1.734599858759064124e-06 -1.400664681976267818e-06 --3.385016663087549197e-07 --1.695870676087512766e-06 --1.504554066993212823e-06 --1.028948149217973521e-08 -1.507067109023507444e-06 -1.656464995976781995e-06 -3.579755870487922869e-07 --1.388613154987747987e-06 --1.701443479170704020e-06 --5.275255413453798391e-07 -1.127166071302384452e-06 -1.734503200677021406e-06 -8.520142368580808288e-07 --9.844001539690870857e-07 --1.741656127705483338e-06 --1.004294452983678689e-06 -6.744018012792296245e-07 -1.741914613915466185e-06 -1.280937446172547368e-06 --3.385295895269946168e-07 --1.734981101659022615e-06 --1.400588052416376249e-06 -1.648316694002071529e-07 -1.613486565481482708e-06 -1.590483001996038449e-06 -1.850813389808183833e-07 --1.507046180103455067e-06 --1.656478180211654670e-06 --3.579349198709450374e-07 -1.261721756539615536e-06 -1.726441261398884717e-06 -6.926417775169320062e-07 --1.127277146577362529e-06 --1.735176888438899801e-06 --8.519584793928184132e-07 -8.332792681452140573e-07 -1.742157773790548184e-06 -1.148017886747695187e-06 --5.090103794232628682e-07 --1.741570079490782926e-06 --1.280866889089690358e-06 -3.385394777859105416e-07 -1.696098376610258425e-06 -1.504587084259393158e-06 -1.028785662284046328e-08 --1.613721909539123835e-06 --1.590494503503302450e-06 --1.850666926647154778e-07 -1.388542047507202778e-06 -1.701004739452484825e-06 -5.275550978710463273e-07 --1.127242889265855080e-06 --1.724557476696404319e-06 --6.925939616939340237e-07 -9.844457602245243228e-07 -1.739765484603756400e-06 -1.004345694661083607e-06 --6.744448456179310710e-07 --1.741858256020505802e-06 --1.147945817186612501e-06 -5.089811917854148751e-07 -1.734594781773205028e-06 -1.400685564424060619e-06 --1.648459173018137831e-07 --1.696301470419201054e-06 --1.504565791851446476e-06 --1.028660033549784524e-08 -1.507102629175416761e-06 -1.656165813660437616e-06 -3.579516650658177460e-07 --1.261767226410950790e-06 --1.701502509763890978e-06 --5.275255364334288959e-07 -1.127258241735445696e-06 -1.738397037242816289e-06 -8.519992627385563252e-07 --8.333155900301203234e-07 --1.739221311409126995e-06 --1.004289595998266989e-06 -6.743959648938684969e-07 -1.741615939271367849e-06 -1.280948672552883438e-06 --3.385524990879865896e-07 --1.696233800921703517e-06 --1.400615320269782821e-06 -1.648185908126635101e-07 -1.613437983733279720e-06 -1.590376068386415433e-06 -1.850698379245635073e-07 --1.388559203520277344e-06 --1.656422452523023210e-06 --3.579448456761001503e-07 -1.261793790873533917e-06 -1.724840191291488373e-06 -6.926291817612108503e-07 --9.844155398481135369e-07 --1.739034044670750390e-06 --8.519585025669481418e-07 -8.332434238338562824e-07 -1.742439124717288334e-06 -1.148015029905874473e-06 --5.090224365676433484e-07 --1.734886862324567029e-06 --1.280870799559112837e-06 -3.384997738350057584e-07 -1.695894495341171505e-06 -1.504610243716362610e-06 -1.028324945307817011e-08 --1.507023215201327417e-06 --1.590389675798358597e-06 --1.850716221977319632e-07 -1.388596165212837243e-06 -1.701872268799283530e-06 -5.275521592616450156e-07 --1.127213027036639802e-06 --1.738868422425264749e-06 --6.925967090994584708e-07 -9.843423265322836092e-07 -1.739658055559617132e-06 -1.004347820281614206e-06 --6.744326650540783592e-07 --1.741589486302994863e-06 --1.147951091273796405e-06 -5.089665116460267154e-07 -1.734732998372292519e-06 -1.400646831852552351e-06 --1.648303833899749357e-07 --1.613310940965321688e-06 --1.504616294783919238e-06 --1.028165214295071204e-08 -1.506992341721636064e-06 -1.656530616962355248e-06 -3.579615973366801713e-07 --1.261896119614707845e-06 --1.725099656955503595e-06 --5.275287952149862621e-07 -1.127169131548336438e-06 -1.736726332399427330e-06 -8.520031875083780306e-07 --8.332788361584564856e-07 --1.741739780205732736e-06 --1.004287541940401894e-06 -6.744270469032312699e-07 -1.741887255151474092e-06 -1.280932833180469980e-06 --3.385350963948087381e-07 --1.695992647258216778e-06 --1.504617839395090027e-06 -1.648217192121360978e-07 -1.613600533049282307e-06 -1.590344747451266808e-06 -1.850778758784153934e-07 --1.388497459901435881e-06 --1.701694326018869044e-06 --3.579403082721944544e-07 -1.261788553692148542e-06 -1.725249752752758465e-06 -6.926305704182332962e-07 --9.844267509989011877e-07 --1.739205736135453973e-06 --8.519543788221921723e-07 -8.332742280914557694e-07 -1.741817340405137131e-06 -1.148018306641354476e-06 --5.089990489542986116e-07 --1.734404740651791544e-06 --1.400611542120221849e-06 -3.385030299852301277e-07 -1.696213744038989187e-06 -1.504595461529449285e-06 -1.028736312899410621e-08 --1.507070054026904670e-06 --1.656396408560201997e-06 --1.850607566820000722e-07 -1.388605038353589448e-06 -1.701245856308883713e-06 -5.275505345612582854e-07 --1.127252558694745794e-06 --1.738070038224574988e-06 --8.519549505936305591e-07 -9.843313317470496922e-07 -1.739225041207011783e-06 -1.004346551016468279e-06 --6.744387716338965728e-07 --1.740866185716938826e-06 --1.280870248436358718e-06 -5.089666806394039543e-07 -1.734652445953194618e-06 -1.400686053030659741e-06 --1.648523921474952950e-07 --1.613681664011344824e-06 --1.590316014061567975e-06 --1.027950810415140155e-08 -1.506931218489344571e-06 -1.656346474519701670e-06 -3.579517781802106437e-07 --1.261829655859476321e-06 --1.727214126451728340e-06 --6.925930708928146570e-07 -1.127135099675787796e-06 -1.739042613578682417e-06 -8.520049857849844323e-07 --8.333550763716032250e-07 --1.741223931084412381e-06 --1.147960783092397317e-06 -6.744292292009189618e-07 -1.741077513878259454e-06 -1.280943363517365134e-06 --3.385752748583262396e-07 --1.696353572023426503e-06 --1.504547365475786380e-06 --1.027654991288985169e-08 -1.613222830497776154e-06 -1.590460028730889177e-06 -1.850729280492534397e-07 --1.388806688619792916e-06 --1.701267205312941893e-06 --5.275260701054195476e-07 -1.261773888119183541e-06 -1.725800301839937553e-06 -6.926372703249363807e-07 --9.844933858999205134e-07 --1.742084440336276215e-06 --1.004293243730852016e-06 -8.332837782442656153e-07 -1.741681429266680129e-06 -1.148018244108631514e-06 --5.090140825880851954e-07 --1.734648181794856410e-06 --1.400615529873207839e-06 -1.648194043834948003e-07 -1.696085223384804340e-06 -1.504625704785217146e-06 -1.029153370613536142e-08 --1.507118939351625321e-06 --1.655993897924089542e-06 --3.579469875844075512e-07 -1.388652301991538058e-06 -1.701558517642712207e-06 -5.275674631140396064e-07 --1.127395666019526808e-06 --1.734941680691810245e-06 --8.519556369383823392e-07 -8.332644240114245081e-07 -1.739256246116809773e-06 -1.004353996353650715e-06 --6.744530441635463773e-07 --1.742187448380313789e-06 --1.280886367995999871e-06 -3.384972043392220203e-07 -1.734506840627408643e-06 -1.400668240434210029e-06 --1.648250127578891211e-07 --1.613605146426856459e-06 --1.590345985028447486e-06 --1.850701461615776516e-07 -1.507153337590947545e-06 -1.656372251468303937e-06 -3.579702221657228762e-07 --1.261874832669292944e-06 --1.726027473690091609e-06 --6.925920677498453038e-07 -9.843190773326566283e-07 -1.739058466695084452e-06 -8.520157362474218902e-07 --8.333066511850163643e-07 --1.742433685415770509e-06 --1.147955603304021991e-06 -5.089720785437955275e-07 -1.741096844482296675e-06 -1.280939701917045734e-06 --3.385226925826214097e-07 --1.696133847163606545e-06 --1.504574588735058261e-06 --1.027584873402550068e-08 -1.506900793475349959e-06 -1.590413619220728538e-06 -1.850912530880386366e-07 --1.388642336697858351e-06 --1.701249147872202525e-06 --5.275242307718781953e-07 -1.127191767414049902e-06 -1.726028621147986852e-06 -6.926392242527476744e-07 --9.844647089534297558e-07 --1.739939757177752584e-06 --1.004288355418123485e-06 -6.744529207682478307e-07 -1.742380102408182492e-06 -1.148016912806621430e-06 --5.089940474249223597e-07 --1.734603309747968123e-06 --1.400616570006886963e-06 -1.648218615872110605e-07 -1.613295571530132512e-06 -1.504603161806901259e-06 -1.029429429793979145e-08 --1.506989540017813416e-06 --1.656398114490014354e-06 --3.579385752429690782e-07 -1.261835544753514274e-06 +1.000005628528739532e+00 +1.000002760802300550e+00 +9.999972997553338017e-01 +9.999943563572948335e-01 +9.999962801846589144e-01 +1.000001649258276837e+00 +1.000005620820334062e+00 +1.000004150766180988e+00 +9.999989030075261409e-01 +9.999945031723588285e-01 +9.999951247111897867e-01 +9.999999666911228990e-01 +1.000004883382004550e+00 +1.000005153047535122e+00 +1.000000599705294313e+00 +9.999955004486030807e-01 +9.999944881934780261e-01 +9.999982906361591928e-01 +1.000003652479025895e+00 +1.000005628541846825e+00 +1.000002244380341576e+00 +9.999968100808547256e-01 +9.999943599485604517e-01 +9.999967457293845730e-01 +1.000002185340829408e+00 +1.000005642541941020e+00 +1.000003720006343633e+00 +9.999983506438546721e-01 +9.999943794813748177e-01 +9.999954614264627839e-01 +1.000000534103744565e+00 +1.000005228487242270e+00 +1.000004875453037512e+00 +1.000000033313619863e+00 +9.999951164331567499e-01 +9.999946341582036657e-01 +9.999988401547134087e-01 +1.000004088571126992e+00 +1.000005593284785776e+00 +1.000001709470952349e+00 +9.999963473356909827e-01 +9.999943704194970140e-01 +9.999972393583208552e-01 +1.000002700092997632e+00 +1.000005643849237513e+00 +1.000003254450103407e+00 +9.999978145320155276e-01 +9.999943574942783409e-01 +9.999958494267148223e-01 +1.000001096926375732e+00 +1.000005496774726943e+00 +1.000004875452669584e+00 +9.999994658651549795e-01 +9.999947713165886487e-01 +9.999948470910047238e-01 +9.999994003354394367e-01 +1.000004499364685495e+00 +1.000005512001406638e+00 +1.000001159920171689e+00 +9.999959112267413630e-01 +9.999944067110044799e-01 +9.999977557557993002e-01 +1.000003189758528865e+00 +1.000005640313947675e+00 +1.000002760802300550e+00 +9.999972997557676768e-01 +9.999943560688479005e-01 +9.999962801842345872e-01 +1.000001649257038938e+00 +1.000005621004812717e+00 +1.000004538757190087e+00 +9.999989030075642216e-01 +9.999945030243991839e-01 +9.999951247130476339e-01 +9.999999666910829310e-01 +1.000004883539776568e+00 +1.000005365915851652e+00 +1.000000599705294313e+00 +9.999955004500691302e-01 +9.999944880729773056e-01 +9.999982906363659163e-01 +1.000003652484243277e+00 +1.000005627067319880e+00 +1.000002760802310542e+00 +9.999968100813726446e-01 +9.999943587434514392e-01 +9.999967457292208151e-01 +1.000002185339766481e+00 +1.000005642856323318e+00 +1.000004150766037547e+00 +9.999983506438546721e-01 +9.999943793195951169e-01 +9.999954614275690101e-01 +1.000000534102996053e+00 +1.000005228451245953e+00 +1.000005153042230033e+00 +1.000000033313618086e+00 +9.999951164340383780e-01 +9.999946341918117820e-01 +9.999988401550783390e-01 +1.000004088573421823e+00 +1.000005592102742868e+00 +1.000002244380358674e+00 +9.999963473356909827e-01 +9.999943708998354497e-01 +9.999972393583131947e-01 +1.000002700091364494e+00 +1.000005643622340346e+00 +1.000003720006071184e+00 +9.999978145317437450e-01 +9.999943574706082750e-01 +9.999958494265849263e-01 +1.000001096927432220e+00 +1.000005496575158581e+00 +1.000004875448803565e+00 +1.000000033313679593e+00 +9.999947713165886487e-01 +9.999948470919628463e-01 +9.999994003355955341e-01 +1.000004499438863936e+00 +1.000005512169376720e+00 +1.000001709470946354e+00 +9.999959112259187988e-01 +9.999944066527843844e-01 +9.999977557556760654e-01 +1.000003189752930677e+00 +1.000005639422707926e+00 +1.000003254449838064e+00 +9.999972997553338017e-01 +9.999943560688479005e-01 +9.999962801842277038e-01 +1.000001649258156711e+00 +1.000005621386515831e+00 +1.000004538758224371e+00 +9.999994658652552326e-01 +9.999945030200390050e-01 +9.999951247131133591e-01 +9.999999666910801555e-01 +1.000004883363642572e+00 +1.000005366041464505e+00 +1.000001159920191895e+00 +9.999955004486030807e-01 +9.999944880729773056e-01 +9.999982906361969404e-01 +1.000003652475752958e+00 +1.000005625834710976e+00 +1.000002760802211066e+00 +9.999972997562609489e-01 +9.999943586451935928e-01 +9.999967457292189277e-01 +1.000002185340485239e+00 +1.000005642549671059e+00 +1.000004150767109579e+00 +9.999989030078371144e-01 +9.999943794813748177e-01 +9.999954614275690101e-01 +1.000000534103019145e+00 +1.000005228568834337e+00 +1.000005153061212182e+00 +1.000000599705332727e+00 +9.999951164793701164e-01 +9.999946341923420245e-01 +9.999988401550476969e-01 +1.000004088577316930e+00 +1.000005593928879444e+00 +1.000002244380455263e+00 +9.999968100858261932e-01 +9.999943704194970140e-01 +9.999972393583131947e-01 +1.000002700086370488e+00 +1.000005644432087282e+00 +1.000003720006215957e+00 +9.999983506445834225e-01 +9.999943575934433504e-01 +9.999958494266537601e-01 +1.000001096925456467e+00 +1.000005496899583068e+00 +1.000004875454455266e+00 +1.000000033313196868e+00 +9.999951164930553915e-01 +9.999948470910047238e-01 +9.999994003355955341e-01 +1.000004499321753393e+00 +1.000005511920205148e+00 +1.000001709470967448e+00 +9.999963473372371903e-01 +9.999944062123556909e-01 +9.999977557557915286e-01 +1.000003189739357756e+00 +1.000005642795034788e+00 +1.000003254449985057e+00 +9.999978145329047052e-01 +9.999943563572948335e-01 +9.999962801842345872e-01 +1.000001649258156711e+00 +1.000005619599322992e+00 +1.000004538759156958e+00 +9.999994658639600464e-01 +9.999947714438510715e-01 +9.999951247112304209e-01 +9.999999666913291785e-01 +1.000004883399038924e+00 +1.000005365872079333e+00 +1.000001159920033578e+00 +9.999959112445278020e-01 +9.999944881934780261e-01 +9.999982906363659163e-01 +1.000003652475752958e+00 +1.000005632338691219e+00 +1.000002760802288337e+00 +9.999972997472790226e-01 +9.999943567464447680e-01 +9.999967457292198159e-01 +1.000002185344750050e+00 +1.000005642550357843e+00 +1.000004150766443001e+00 +9.999989030047454763e-01 +9.999945035217056022e-01 +9.999954614264627839e-01 +1.000000534102996053e+00 +1.000005228568834337e+00 +1.000005153038430850e+00 +1.000000599705102911e+00 +9.999955003701609391e-01 +9.999946341798189309e-01 +9.999988401550761186e-01 +1.000004088588974716e+00 +1.000005592525647469e+00 +1.000002244380356675e+00 +9.999968100447359509e-01 +9.999943610515239500e-01 +9.999972393583208552e-01 +1.000002700091364494e+00 +1.000005644432087282e+00 +1.000003720005975927e+00 +9.999983506418793633e-01 +9.999943790659656662e-01 +9.999958494266214526e-01 +1.000001096925393851e+00 +1.000005496709364339e+00 +1.000004875462937370e+00 +1.000000033313657388e+00 +9.999951161300691949e-01 +9.999946340991267002e-01 +9.999994003354394367e-01 +1.000004499438863936e+00 +1.000005511920205148e+00 +1.000001709471006528e+00 +9.999963473216946230e-01 +9.999943736582742604e-01 +9.999977557556830599e-01 +1.000003189743372323e+00 +1.000005639596431850e+00 +1.000003254450003931e+00 +9.999978145331541723e-01 +9.999943571553258082e-01 +9.999962801846589144e-01 +1.000001649257038938e+00 +1.000005621386515831e+00 +1.000004538759156958e+00 +9.999994658655363411e-01 +9.999947711702450448e-01 +9.999948470910431375e-01 +9.999999666915803109e-01 +1.000004883258530208e+00 +1.000005365874394370e+00 +1.000001159920109517e+00 +9.999959112661601646e-01 +9.999944082371383658e-01 +9.999982906361591928e-01 +1.000003652484243277e+00 +1.000005625834710976e+00 +1.000002760802288337e+00 +9.999972997569891442e-01 +9.999943557159270968e-01 +9.999962801846287164e-01 +1.000002185340025163e+00 +1.000005642176008847e+00 +1.000004150765567479e+00 +9.999989030076809060e-01 +9.999945034702872881e-01 +9.999951247204843519e-01 +1.000000534103744565e+00 +1.000005228451245953e+00 +1.000005153061212182e+00 +1.000000599705102911e+00 +9.999955004476279719e-01 +9.999944881888750414e-01 +9.999988401548404182e-01 +1.000004088606992303e+00 +1.000005594940963638e+00 +1.000002244380288952e+00 +9.999968100845629815e-01 +9.999943613398070141e-01 +9.999967457294722806e-01 +1.000002700092997632e+00 +1.000005643622340346e+00 +1.000003720006215957e+00 +9.999983506418793633e-01 +9.999943795461668783e-01 +9.999954614258708130e-01 +1.000001096926525168e+00 +1.000005496958468632e+00 +1.000004875463852416e+00 +1.000000033312921977e+00 +9.999951165621137061e-01 +9.999946341059653410e-01 +9.999988401548982608e-01 +1.000004499364685495e+00 +1.000005512169376720e+00 +1.000001709470967448e+00 +9.999963473216946230e-01 +9.999943696170027208e-01 +9.999972393583844710e-01 +1.000003189746767607e+00 +1.000005643480392559e+00 +1.000003254449885803e+00 +9.999978145247830907e-01 +9.999943579815728789e-01 +9.999958494264630238e-01 +1.000001649258276837e+00 +1.000005621004812717e+00 +1.000004538758224371e+00 +9.999994658639600464e-01 +9.999947711702450448e-01 +9.999948470843763593e-01 +9.999994003352858929e-01 +1.000004883338294182e+00 +1.000005365845172189e+00 +1.000001159920220983e+00 +9.999959111446200533e-01 +9.999944054997563780e-01 +9.999977557557988561e-01 +1.000003652479025895e+00 +1.000005627067319880e+00 +1.000002760802211066e+00 +9.999972997472790226e-01 +9.999943557159270968e-01 +9.999962801843798044e-01 +1.000001649258226211e+00 +1.000005642635592551e+00 +1.000004150767031641e+00 +9.999989030075279173e-01 +9.999945026359662315e-01 +9.999951247017295763e-01 +9.999999666912579022e-01 +1.000005228487242270e+00 +1.000005153042230033e+00 +1.000000599705332727e+00 +9.999955003701609391e-01 +9.999944881888750414e-01 +9.999982906362907542e-01 +1.000004088582988171e+00 +1.000005593699111905e+00 +1.000002244380362448e+00 +9.999968100784266678e-01 +9.999943560692980959e-01 +9.999967457293303941e-01 +1.000002185341857697e+00 +1.000005643849237513e+00 +1.000003720006071184e+00 +9.999983506445834225e-01 +9.999943790659656662e-01 +9.999954614258708130e-01 +1.000000534103909544e+00 +1.000005496884300182e+00 +1.000004875456988129e+00 +1.000000033313726000e+00 +9.999951165602208869e-01 +9.999946344156538380e-01 +9.999988401548888239e-01 +1.000004088580648709e+00 +1.000005512001406638e+00 +1.000001709470946354e+00 +9.999963473372371903e-01 +9.999943736582742604e-01 +9.999972393583844710e-01 +1.000002700094186014e+00 +1.000005641490590458e+00 +1.000003254450039458e+00 +9.999978145329936341e-01 +9.999943578841843372e-01 +9.999958494268899045e-01 +1.000001096926845356e+00 +1.000005620820334062e+00 +1.000004538757190087e+00 +9.999994658652552326e-01 +9.999947714438510715e-01 +9.999948470910431375e-01 +9.999994003352858929e-01 +1.000004499353685850e+00 +1.000005365897656429e+00 +1.000001159920152149e+00 +9.999959112420302443e-01 +9.999944051090710051e-01 +9.999977557558098473e-01 +1.000003189750150678e+00 +1.000005628541846825e+00 +1.000002760802310542e+00 +9.999972997562609489e-01 +9.999943567464447680e-01 +9.999962801846287164e-01 +1.000001649258226211e+00 +1.000005620814725660e+00 +1.000004150766166111e+00 +9.999989030075592256e-01 +9.999945030900928566e-01 +9.999951247068651350e-01 +9.999999666912370300e-01 +1.000004883374563613e+00 +1.590264765490082914e-06 +1.850729140869637458e-07 +-1.388588244203136346e-06 +-1.701031557416438364e-06 +-5.275213099630358417e-07 +1.127180602577723188e-06 +1.726204084242227366e-06 +6.926305762819332230e-07 +-9.844306660256146090e-07 +-1.740803289592741682e-06 +-1.004289949222725119e-06 +6.744112212491573858e-07 +1.741335549237821071e-06 +1.148018656192231561e-06 +-5.090023917688826266e-07 +-1.734498598948816102e-06 +-1.400634720834447876e-06 +1.648279703523221325e-07 +1.613568967277801687e-06 +1.504597324170937172e-06 +1.028067228938113768e-08 +-1.507104483490528637e-06 +-1.655936937116649721e-06 +-3.579361456535955894e-07 +1.261764197295514305e-06 +1.726212177919020907e-06 +5.275539698013462470e-07 +-1.127236664219711872e-06 +-1.737220343331189526e-06 +-8.519527408269153213e-07 +8.332674221312309354e-07 +1.741805190543981896e-06 +1.004344870327094468e-06 +-6.744500693762746864e-07 +-1.741403751630510061e-06 +-1.280895003494581976e-06 +3.385191388017711458e-07 +1.696356387088678474e-06 +1.400690768157706481e-06 +-1.648377153886270362e-07 +-1.613615946761236905e-06 +-1.590224364818632895e-06 +-1.850607130217392161e-07 +1.388532104941291530e-06 +1.701031341485648219e-06 +3.579589418115815814e-07 +-1.261822845087495224e-06 +-1.726119637576285491e-06 +-6.925890190663733147e-07 +9.843772412154688514e-07 +1.740927294808470333e-06 +8.520018471357216599e-07 +-8.333136191524715298e-07 +-1.741691672644508485e-06 +-1.147960314753250192e-06 +5.089729776927090865e-07 +1.734534116829732256e-06 +1.400690801915521886e-06 +-3.385393394898985487e-07 +-1.696405267145462151e-06 +-1.504546415202605068e-06 +-1.027946375866068890e-08 +1.507023368261896227e-06 +1.655957371204284394e-06 +1.850729093002186981e-07 +-1.388594751825199030e-06 +-1.701013619523377908e-06 +-5.275213653444438600e-07 +1.127183222209527258e-06 +1.737421589172538437e-06 +6.926305765555142771e-07 +-9.844309318740227226e-07 +-1.740786773625750572e-06 +-1.004290020861649846e-06 +6.744120618436009555e-07 +1.741390778335149306e-06 +1.280953965234318406e-06 +-5.090024021606439771e-07 +-1.734632918663487114e-06 +-1.400634057052309753e-06 +1.648278595439363850e-07 +1.613607653265826696e-06 +1.590264905666604030e-06 +1.028067065780491688e-08 +-1.507107115435284108e-06 +-1.655926036113621952e-06 +-3.579363188955392669e-07 +1.261781259633358326e-06 +1.726655678971084598e-06 +6.926305783295573401e-07 +-1.127236716565594789e-06 +-1.737006477177174457e-06 +-8.519527617554620274e-07 +8.332646747978481785e-07 +1.742258799404387898e-06 +1.148018555346056507e-06 +-6.744500485209134269e-07 +-1.741347098123176928e-06 +-1.280894987313796288e-06 +3.385181697038487692e-07 +1.696401660585019038e-06 +1.504598040628004677e-06 +-1.648377187923204881e-07 +-1.613613792776174776e-06 +-1.590222706573371879e-06 +-1.850608810711246522e-07 +1.388522568641007264e-06 +1.701136590258371164e-06 +5.275539753600890077e-07 +-1.261822820560824661e-06 +-1.726131226545413262e-06 +-6.925890031772266819e-07 +9.843718222060919152e-07 +1.742000704344227011e-06 +1.004344794715776859e-06 +-8.333134573449257216e-07 +-1.741714163750060391e-06 +-1.147960327360615846e-06 +5.089712561595381667e-07 +1.734574036784703316e-06 +1.400691744043100323e-06 +-1.648377415733707867e-07 +-1.696400115162551198e-06 +-1.504545903645650570e-06 +-1.027953056309141547e-08 +1.506989973039731223e-06 +1.655920479308038736e-06 +3.579589522959742434e-07 +-1.388593351470622148e-06 +-1.701011952376674966e-06 +-5.275212863162724223e-07 +1.127177964477139891e-06 +1.738250354748017606e-06 +8.520018101900386781e-07 +-9.844303504170458106e-07 +-1.740859182696326817e-06 +-1.004290027059698876e-06 +6.744107908519097616e-07 +1.741116874674736869e-06 +1.280954091395851010e-06 +-3.385393066482944901e-07 +-1.734642112838170612e-06 +-1.400633959351386143e-06 +1.648282474766000761e-07 +1.613508647759532765e-06 +1.590249907748959577e-06 +1.850729168912970547e-07 +-1.507102555053953027e-06 +-1.655924432546027512e-06 +-3.579362132718278165e-07 +1.261761708176442196e-06 +1.726532829553030439e-06 +6.926305492548642814e-07 +-9.844306970560387685e-07 +-1.736893099432485814e-06 +-8.519527397257628548e-07 +8.332692941469899978e-07 +1.741728089231568348e-06 +1.148018495461474057e-06 +-5.090023511046549143e-07 +-1.741303242388386698e-06 +-1.280894957643578081e-06 +3.385193196125991168e-07 +1.696379544777888894e-06 +1.504602180952277671e-06 +1.028068958602692250e-08 +-1.613561232713966268e-06 +-1.590222696773235619e-06 +-1.850608111244003354e-07 +1.388538546951377015e-06 +1.701034391126270827e-06 +5.275539233213765546e-07 +-1.127234671664729506e-06 +-1.726011152294675224e-06 +-6.925890098825439449e-07 +9.843883284494780983e-07 +1.740038832783223951e-06 +1.004344836786030559e-06 +-6.744498740321052226e-07 +-1.741504245649460200e-06 +-1.147960309000444030e-06 +5.089722069015032994e-07 +1.734508523287152402e-06 +1.400692320193904770e-06 +-1.648377033204455520e-07 +-1.613557636144620663e-06 +-1.504546181484033033e-06 +-1.027957913124318570e-08 +1.507101195155202836e-06 +1.655965490179735808e-06 +3.579588997065866131e-07 +-1.261816372878705018e-06 +-1.701023201711399771e-06 +-5.275213654853882366e-07 +1.127182558734247784e-06 +1.736287626053159374e-06 +8.520018575416590598e-07 +-8.333115896018436259e-07 +-1.741233791562141908e-06 +-1.004290013300689863e-06 +6.744104854365436031e-07 +1.741474501100723134e-06 +1.280954225901281985e-06 +-3.385392123944796676e-07 +-1.696339783109555203e-06 +-1.400633853529916587e-06 +1.648276988163869395e-07 +1.613616280226383397e-06 +1.590272583922566486e-06 +1.850728452661747030e-07 +-1.388571029411387782e-06 +-1.655924530264996947e-06 +-3.579363204340156911e-07 +1.261765294842253382e-06 +1.725704550970076383e-06 +6.926305822189542276e-07 +-9.844239440449483160e-07 +-1.741860529457180813e-06 +-8.519527518524718275e-07 +8.332648269389474782e-07 +1.741686763624658990e-06 +1.148018541475697390e-06 +-5.090025861066646113e-07 +-1.734476959485240437e-06 +-1.280894948269444336e-06 +3.385183143243838288e-07 +1.696254254232860618e-06 +1.504595116280361796e-06 +1.028056233335913099e-08 +-1.507077654368541262e-06 +-1.590216314667034771e-06 +-1.850608713169052016e-07 +1.388520441361015031e-06 +1.700980338400680608e-06 +5.275539633215826346e-07 +-1.127236063755064691e-06 +-1.738352324809754403e-06 +-6.925889977216639691e-07 +9.843722963441212532e-07 +1.741591220560113558e-06 +1.004344803979023826e-06 +-6.744506833673830678e-07 +-1.741389301872752054e-06 +-1.147960219981160685e-06 +5.089721876573454540e-07 +1.734476542971107082e-06 +1.400690338484537455e-06 +-1.648377427507179431e-07 +-1.613657006753198217e-06 +-1.590211924678924318e-06 +-1.027959209146975030e-08 +1.506988131456349131e-06 +1.655938485993354342e-06 +3.579589168854693923e-07 +-1.261829484093831079e-06 +-1.726715825596879808e-06 +-5.275212437362224136e-07 +1.127178057903389972e-06 +1.737521352184773082e-06 +8.520018229417028529e-07 +-8.333125651304750253e-07 +-1.742161269370573554e-06 +-1.004289947210272245e-06 +6.744127985546446917e-07 +1.741051388884109057e-06 +1.280953755630713605e-06 +-3.385392033861634126e-07 +-1.696253277177149094e-06 +-1.504548069805109415e-06 +1.648278354643091128e-07 +1.613475959251400039e-06 +1.590262408484331375e-06 +1.850728479543541457e-07 +-1.388583254850119703e-06 +-1.700955423263663647e-06 +-3.579361129172779379e-07 +1.261779325749936502e-06 +1.726525155565902702e-06 +6.926305780720819822e-07 +-9.844266832580904818e-07 +-1.741654060643989825e-06 +-1.004289939756303958e-06 +8.332680579400941169e-07 +1.742243013133727748e-06 +1.148018595987038247e-06 +-5.090027725329849093e-07 +-1.734378166739236002e-06 +-1.400636100968319663e-06 +3.385190319124972339e-07 +1.696506281846732884e-06 +1.504604861744389618e-06 +1.028059905200941356e-08 +-1.507069193346817950e-06 +-1.655914345597965604e-06 +-1.850606863265083224e-07 +1.388552092878703080e-06 +1.701122628313673062e-06 +5.275539809636132356e-07 +-1.127232993435450235e-06 +-1.737436630235110488e-06 +-8.519527397658789705e-07 +9.843782249210293688e-07 +1.740739431917946449e-06 +1.004344838814747736e-06 +-6.744508427269661772e-07 +-1.741353822898104329e-06 +-1.280895085241792278e-06 +5.089722879147188954e-07 +1.734705775677896747e-06 +1.400691819755092668e-06 +-1.648375501755424355e-07 +-1.613494986889711667e-06 +-1.590222334497007968e-06 +-1.850607192132729485e-07 +1.507043730069361151e-06 +1.655922976586692305e-06 +3.579589254593023206e-07 +-1.261827036489203838e-06 +-1.725958591374725790e-06 +-6.925890294860836820e-07 +1.127180733253413700e-06 +1.736649648758746197e-06 +8.520018083487564456e-07 +-8.333123625034528594e-07 +-1.741452707647580027e-06 +-1.147960193312130135e-06 +6.744112495709264575e-07 +1.741096945602424631e-06 +1.280954164947170303e-06 +-3.385392326444060757e-07 +-1.696434418638351865e-06 +-1.504549858019050800e-06 +-1.027942930088404444e-08 +1.613534489317199388e-06 +1.590260040480478103e-06 +1.850729169299731349e-07 +-1.388605987413341007e-06 +-1.701035332109263384e-06 +-5.275213130458057946e-07 +1.261763984985514936e-06 +1.726241266265620074e-06 +6.926305555714041155e-07 +-9.844252420403123427e-07 +-1.740161427431340968e-06 +-1.004289978381531725e-06 +6.744112099647509990e-07 +1.741731817663470630e-06 +1.148018570729173269e-06 +-5.090026755620329171e-07 +-1.734742759434928427e-06 +-1.400634515905571959e-06 +1.648279538339424900e-07 +1.696376704814610811e-06 +1.504598494872169923e-06 +1.028066870892576501e-08 +-1.507091911883608387e-06 +-1.655893623464248965e-06 +-3.579361675225453786e-07 +1.388533938734241403e-06 +1.701038926954952633e-06 +5.275539739055161733e-07 +-1.127234130548387463e-06 +-1.736386854650884862e-06 +-8.519527442219273472e-07 +8.332673401775128358e-07 +1.740817689762474523e-06 +1.004344825011346362e-06 +-6.744500422891672025e-07 +-1.741096689210724751e-06 +-1.280894868778158905e-06 +3.385189366979776427e-07 +1.734652198908528086e-06 +1.400690791354813143e-06 +-1.648377541514747078e-07 +-1.613519101369212377e-06 +-1.590222433229536816e-06 +-1.850607103140266643e-07 +1.388533408879053315e-06 +1.655954223837759175e-06 +3.579589533039035342e-07 +-1.261819035994373110e-06 +-1.726388645621787296e-06 +-6.925890301642044953e-07 +9.843785017159068400e-07 +1.737000350907603905e-06 +8.520018475476419348e-07 +-8.333136848140522488e-07 +-1.742061194791934156e-06 +-1.147960231265457910e-06 +5.089721727050693534e-07 +1.741297441231007580e-06 +1.280954014745376018e-06 +-3.385392960721465077e-07 +-1.696451581426947372e-06 +-1.504550365415478776e-06 +-1.027943118621443695e-08 +1.507040538203202165e-06 +1.590264633512082422e-06 +1.850729074580114792e-07 +-1.388589550375052561e-06 +-1.701090859141943968e-06 +-5.275213157429978796e-07 +1.127180362195730999e-06 +1.726192361805550371e-06 +6.926305758712251581e-07 +-9.844312394114428420e-07 +-1.740848301116477094e-06 +-1.004289947260643176e-06 +6.744111886836564748e-07 +1.741318183988452249e-06 +1.148018657348503531e-06 +-5.090024160993471123e-07 +-1.734644591159978853e-06 +-1.400635158561785535e-06 +1.648279631807470339e-07 +1.613526411383233149e-06 +1.504597547354309073e-06 +1.028065787466899798e-08 +-1.507094808177593535e-06 +-1.655919501024793512e-06 +-3.579361336895907404e-07 +1.261761598316162847e-06 +1.701043142061134715e-06 +5.275539686005711651e-07 +-1.127236636967320203e-06 +-1.736909142709718707e-06 +-8.519527328932085376e-07 +8.332673448540016429e-07 +1.741725489135484205e-06 +1.004344870763382504e-06 +-6.744501337659342148e-07 +-1.741340970541436932e-06 +-1.280894975300672104e-06 +3.385190317652273211e-07 +1.696338176289258743e-06 +1.400690744447755463e-06 +-1.648377118329533743e-07 +-1.613597865805794811e-06 +-1.590226031244483694e-06 +-1.850606733397233657e-07 +1.388531945827222191e-06 +1.701047207817729136e-06 +3.579589414388314452e-07 +-1.261823264317818449e-06 +-1.726313466700230310e-06 +-6.925889919202242462e-07 +9.843787614439498560e-07 +1.740917213504126696e-06 +8.520018467356391998e-07 +-8.333135005031860428e-07 +-1.741557060862754581e-06 +-1.147960268085925917e-06 +5.089725240092820862e-07 +1.734630835993027996e-06 +1.280954040857855735e-06 +-3.385392950009363383e-07 +-1.696389995147857853e-06 +-1.504546890036757210e-06 +-1.027961188272786209e-08 +1.507036240073446114e-06 +1.655959555826614411e-06 +1.850729076436188443e-07 +-1.388593212752186936e-06 +-1.701006838237213828e-06 +-5.275212852257531840e-07 +1.127178683840543033e-06 +1.737020925846196760e-06 +6.926305748046115189e-07 +-9.844299088774786119e-07 +-1.740925542203291759e-06 +-1.004290007003716702e-06 +6.744107042022599548e-07 +1.741435709825935481e-06 +1.280954053944180827e-06 +-5.090023840791434605e-07 +-1.734595364073576457e-06 +-1.400634092044772874e-06 +1.648279295576969274e-07 +1.613463305719000815e-06 +1.590265258593738983e-06 +1.028065892532635130e-08 +-1.507095744533528857e-06 +-1.655925203232106663e-06 +-3.579362009799558894e-07 +1.261748430319599773e-06 +1.726121581406505635e-06 +5.275539683611384561e-07 +-1.127236383812855309e-06 +-1.737105258668889080e-06 +-8.519527344823373202e-07 +8.332655063772550808e-07 +1.741999493867693744e-06 +1.148018639623488477e-06 +-6.744501555272217753e-07 +-1.741300265041495049e-06 +-1.280894964161216665e-06 +3.385198128477862204e-07 +1.696089828885080503e-06 +1.504598314128847461e-06 +-1.648377264320529201e-07 +-1.613602985855504784e-06 +-1.590222425754394400e-06 +-1.850606615669670153e-07 +1.388531452354886985e-06 +1.701102540831980756e-06 +5.275539716868027299e-07 +-1.261823966723208634e-06 +-1.726190644147426965e-06 +-6.925889942170049046e-07 +9.843857574402787822e-07 +1.741095491941425659e-06 +1.004344842583398701e-06 +-8.333135707087331351e-07 +-1.741663383577325622e-06 +-1.147960329353777756e-06 +5.089728844887976374e-07 +1.734684800346359833e-06 +1.400691590590690259e-06 +-3.385393480201725429e-07 +-1.696399485446525602e-06 +-1.504546510586320973e-06 +-1.027938485238273844e-08 +1.507080305796631483e-06 +1.656015133408582491e-06 +3.579589492293104631e-07 +-1.388594604913892693e-06 +-1.701010685315829370e-06 +-5.275212901818534962e-07 +1.127185078357103859e-06 +1.736616342016953167e-06 +8.520018344173663146e-07 +-9.844304725327911878e-07 +-1.740965161270790161e-06 +-1.004290031213002537e-06 +6.744108472012478456e-07 +1.741581278298808486e-06 +1.280954403015497926e-06 +-3.385393118433523258e-07 +-1.734619107468389150e-06 +-1.400633980049512253e-06 +1.648278906750231311e-07 +1.613570582697605439e-06 +1.590260759974151735e-06 +1.850729094683444679e-07 +-1.507102500025513167e-06 +-1.655921556354297312e-06 +-3.579362475542472533e-07 +1.261756780791450496e-06 +1.725748945055332470e-06 +6.926305705391843071e-07 +-1.127237630278965120e-06 +-1.737040415864784659e-06 +-8.519527374473198244e-07 +8.332656578725061578e-07 +1.742253152560022106e-06 +1.148018563730156966e-06 +-5.090024215591247669e-07 +-1.741243658726735979e-06 +-1.280894950455009569e-06 +3.385185393912210502e-07 +1.696334705621889013e-06 +1.504592795340949584e-06 +1.028061790487706699e-08 +-1.613657169410413575e-06 +-1.590222084155557492e-06 +-1.850608462758699380e-07 +1.388518131384592072e-06 +1.701072734164493894e-06 +5.275539304736860212e-07 +-1.127238008125842839e-06 +-1.726185478023244631e-06 +-6.925889787473879732e-07 +9.843733839301473761e-07 +1.740160141609993975e-06 +1.004344780140668966e-06 +-6.744507542533378564e-07 +-1.741669229525617891e-06 +-1.147960308469618858e-06 +5.089713572115521794e-07 +1.734339372165476306e-06 +1.400691425918160963e-06 +-1.648376386491944467e-07 +-1.696477797889114190e-06 +-1.504546315678766606e-06 +-1.027938717743279726e-08 +1.506990789283571675e-06 +1.655976698270561457e-06 +3.579589494230031813e-07 +-1.261830028874151778e-06 +-1.700983082463960959e-06 +-5.275212621773382855e-07 +1.127173160533150306e-06 +1.737568977892021735e-06 +8.520018339862569317e-07 +-8.333142415685074648e-07 +-1.740939344644816669e-06 +-1.004290007531021605e-06 +6.744115422746657827e-07 +1.740955319811052847e-06 +1.280954237749410292e-06 +-3.385392772277136745e-07 +-1.696408577544060122e-06 +-1.400633975581495195e-06 +1.648283710793455411e-07 +1.613541157771279822e-06 +1.590271223727216485e-06 +1.850729391147819072e-07 +-1.388612279700782862e-06 +-1.655908914243125178e-06 +-3.579361619669748092e-07 +1.261772547886908059e-06 +1.726494664666686955e-06 +6.926306019008986165e-07 +-9.844319084557291042e-07 +-1.737026184834235509e-06 +-8.519527400719455380e-07 +8.332689029542550202e-07 +1.741371226171650574e-06 +1.148018655309984913e-06 +-5.090027485612200303e-07 +-1.734723042830691599e-06 +-1.280894883536558799e-06 +3.385195705406192733e-07 +1.696366486535114283e-06 +1.504602990292107365e-06 +1.028077317576515492e-08 +-1.507133447163947495e-06 +-1.590222092814339725e-06 +-1.850607621872985744e-07 +1.388542698225709939e-06 +1.700956209036821758e-06 +5.275540081326880979e-07 +-1.127234033009331002e-06 +-1.736801148419747147e-06 +-6.925890247408395166e-07 +9.843863585442160342e-07 +1.741897476277560789e-06 +1.004344838463326896e-06 +-6.744498164834365655e-07 +-1.741618202388645123e-06 +-1.147960252130848252e-06 +5.089719849636938765e-07 +1.734786802983807041e-06 +1.400690929348552800e-06 +-1.648375407929886808e-07 +-1.613487824697420219e-06 +-1.504546947025254604e-06 +-1.027949592541522509e-08 +1.507082540295700812e-06 +1.655914888576624143e-06 +3.579589548223013070e-07 +-1.261825295115731484e-06 +-1.725877888125440188e-06 +-5.275212640265594401e-07 +1.127177136628583626e-06 +1.736480201726351806e-06 +8.520018414740653490e-07 +-8.333128431089876549e-07 +-1.741956109823979605e-06 +-1.004289970032103924e-06 +6.744083160638388031e-07 +1.741423660789217258e-06 +1.280953714161992211e-06 +-3.385396475408251925e-07 +-1.696242829560748933e-06 +-1.504544921077952609e-06 +1.648277199650172369e-07 +1.613573796495554423e-06 +1.590262895351839295e-06 +1.850729264289812674e-07 +-1.388617896793517876e-06 +-1.701104335420668386e-06 +-3.579361365442565734e-07 +1.261725574991984579e-06 +1.725927095259592511e-06 +6.926305975980303250e-07 +-9.844332591812374142e-07 +-1.741566735081765478e-06 +-8.519527372834580185e-07 +8.332631232779616813e-07 +1.742211802605906232e-06 +1.148018435661381874e-06 +-5.090026485225517664e-07 +-1.734720142535303327e-06 +-1.400634780055624592e-06 +3.385191237651266991e-07 +1.696085009963342313e-06 +1.504596475680821139e-06 +1.028076077472234919e-08 +-1.507123908665125899e-06 +-1.655938862777205078e-06 +-1.850607029207132426e-07 +1.388509248107039563e-06 +1.701066517026948025e-06 +5.275540059934432856e-07 +-1.127239805770184038e-06 +-1.736092143553970527e-06 +-8.519527361450001035e-07 +9.843754225922577657e-07 +1.739643212083854234e-06 +1.004344843433495366e-06 +-6.744496218819077910e-07 +-1.741333365699922349e-06 +-1.280895317327004634e-06 +5.089723609021159452e-07 +1.734482161150154862e-06 +1.400692954643723281e-06 +-1.648375766769760290e-07 +-1.613699532047422093e-06 +-1.590226314757858155e-06 +-1.027944017076012186e-08 +1.507039298405862370e-06 +1.656006947935212472e-06 +3.579589261123017248e-07 +-1.261825516591687405e-06 +-1.726366632561289537e-06 +-6.925890308139091532e-07 +1.127179188628730195e-06 +1.738538725988569965e-06 +8.520018361738839483e-07 +-8.333130985189662907e-07 +-1.741587194395977850e-06 +-1.147960233455665095e-06 +6.744110676648418459e-07 +1.741333821610902535e-06 +1.280954485934699437e-06 +-3.385393181167661655e-07 +-1.696443634018159185e-06 +-1.504546304205754675e-06 +-1.027943251450810224e-08 +1.613533829205230461e-06 +1.590252344359356449e-06 +1.850729367243258634e-07 +-1.388583961626842159e-06 +-1.700953865505401834e-06 +-5.275213143991535331e-07 +1.261760805805104762e-06 +1.725819267574720927e-06 +6.926305500241761324e-07 +-9.844303732986032653e-07 +-1.740293455998816554e-06 +-1.004289964934459536e-06 +8.332671041652599658e-07 +1.742274051589850055e-06 +1.148018616373521665e-06 +-5.090030226700787340e-07 +-1.734395104781504249e-06 +-1.400634788150126604e-06 +1.648279423971163625e-07 +1.696371138759718979e-06 +1.504596158615509619e-06 +1.028067767113458230e-08 +-1.507099273513728817e-06 +-1.655912739855413748e-06 +-3.579361695146866141e-07 +1.388528122621444856e-06 +1.701050240375294913e-06 +5.275539633029978847e-07 +-1.127239214595986499e-06 +-1.738200214381804411e-06 +-8.519527517336309874e-07 +8.332671037666123795e-07 +1.740751701307312179e-06 +1.004344783992067638e-06 +-6.744508687318256995e-07 +-1.741251296447213591e-06 +-1.280895004223554836e-06 +3.385188939962563274e-07 +1.734572544061627382e-06 +1.400691395214223112e-06 +-1.648377751771490938e-07 +-1.613654069756421429e-06 +-1.590215504365368672e-06 +-1.850607111848939276e-07 +1.507037537425401796e-06 +1.655958339317997208e-06 +3.579589518641616114e-07 +-1.261822172097506300e-06 +-1.725797783200190056e-06 +-6.925890294727918293e-07 +9.843778963328664999e-07 +1.737178431780542629e-06 +8.520018413846791268e-07 +-8.333139473196456831e-07 +-1.742245021774007872e-06 +-1.147960148770921662e-06 +5.089721369342301751e-07 +1.741310603667982871e-06 +1.280953932279122835e-06 +-3.385393170266433916e-07 +-1.696296137648279874e-06 +-1.504548338331404144e-06 +-1.027942829324316174e-08 +1.507035621867332381e-06 +1.590264345849646242e-06 +1.850729035477137626e-07 +-1.388595280322372227e-06 +-1.701050399214992493e-06 +-5.275213135840591278e-07 +1.127179510866506852e-06 +1.726137462826043499e-06 +6.926305803623038528e-07 +-9.844296559440050383e-07 +-1.739797690169686256e-06 +-1.004289941200076911e-06 +6.744111433083192837e-07 +1.741811333519652724e-06 +1.148018651135363127e-06 +-5.090023546146911557e-07 +-1.734527413667989552e-06 +-1.400635785566348553e-06 +1.648279431176801968e-07 +1.613534133241710520e-06 +1.504597441069569458e-06 +1.028065202879326984e-08 +-1.507090455995950591e-06 +-1.655940491521968514e-06 +-3.579361683397125794e-07 +1.261762431799203453e-06 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 diff --git a/tst/regression/linear_modes.py b/tst/regression/linear_modes.py index 99a821b0f..e9a76c1cb 100755 --- a/tst/regression/linear_modes.py +++ b/tst/regression/linear_modes.py @@ -40,18 +40,24 @@ modified_inputs["parthenon/mesh/nx2"] = 64 modified_inputs["parthenon/meshblock/nx1"] = 64 modified_inputs["parthenon/meshblock/nx2"] = 64 +modified_inputs["fluid/mhd"] = "true" + + +cmake_extra_args = ["-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off", "-DPHOEBUS_DO_NU_HEAVY=Off", "-DCMAKE_CXX_RELEASE_FLAGS='-O1'"] code = rt.gold_comparison( - variables=["p.density", "p.velocity"], + variables=["p.density", "p.velocity"], input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args=cmake_extra_args, geometry="Minkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, build_type=args.build_type, upgold=args.upgold, compression_factor=10, + tolerance=1.0e-3 ) sys.exit(code) diff --git a/tst/regression/mocmc_diffusion.gold b/tst/regression/mocmc_diffusion.gold index 8ac0793ae..f6c33b283 100644 --- a/tst/regression/mocmc_diffusion.gold +++ b/tst/regression/mocmc_diffusion.gold @@ -1,25 +1,25 @@ -2.578814481287529143e-07 -8.539726882839995124e-11 -9.881445129638462621e-11 -2.880460357023293774e-08 -2.198153555894489294e-03 -6.512936231879704607e-01 -1.589897079949703271e-04 -2.487714603862657345e-02 -2.197674042014613471e-02 -1.033579527379025267e-02 -2.531893588544336000e-03 -1.082843017130321120e-02 -4.744884338522541156e-02 -7.923109834475633752e-20 -2.826042354446315629e-19 --5.282457299947488459e-18 --3.598817619128907654e-19 -2.373795279650825908e-18 -1.611427945521856623e-19 --4.579214891635570918e-18 -4.115761985687671323e-18 --2.074785247678172627e-19 -2.203076930037841375e-18 -2.798240882969251344e-18 --8.643975586171317102e-19 +2.578814482748386221e-07 +8.539726882830980148e-11 +9.881492890533372952e-11 +2.878000679792628713e-08 +2.198127933800962577e-03 +6.512936284317732971e-01 +1.589897082372890248e-04 +2.487714603868329197e-02 +2.197674234395428569e-02 +1.035157783173230352e-02 +2.531339100900188883e-03 +1.082842734053424301e-02 +4.744884335341727905e-02 +2.535851403270078324e-18 +4.297042292744506166e-19 +7.780364944017298687e-19 +-2.523409998529510235e-18 +-3.779919609858391220e-18 +-8.950313327148915038e-20 +2.582566784304384226e-18 +1.451093382697625446e-18 +-4.241573517575109588e-19 +-2.809441171738037641e-18 +1.644841674053745085e-19 +6.040096338555097036e-19 diff --git a/tst/regression/mocmc_diffusion.py b/tst/regression/mocmc_diffusion.py index f77bd09a8..b14b12385 100755 --- a/tst/regression/mocmc_diffusion.py +++ b/tst/regression/mocmc_diffusion.py @@ -56,11 +56,14 @@ modified_inputs["parthenon/mesh/nx1"] = 64 modified_inputs["parthenon/meshblock/nx1"] = 64 +cmake_extra_args = ["-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off", "-DPHOEBUS_DO_NU_HEAVY=Off"] + code = rt.gold_comparison( variables=["r.p.J", "r.p.H"], input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args=cmake_extra_args, geometry="Minkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/mocmc_equilibration.gold b/tst/regression/mocmc_equilibration.gold index 50c369f26..83414a878 100644 --- a/tst/regression/mocmc_equilibration.gold +++ b/tst/regression/mocmc_equilibration.gold @@ -1,16 +1,16 @@ -7.782239480291789357e-01 -7.782239480291789357e-01 -7.782239480291788247e-01 -7.782239480291789357e-01 --3.319442647366402016e-17 --2.641239402025653325e-17 -1.803654437194548050e-17 -5.094346950324803810e-17 --1.430155146108605356e-18 --6.894023125021501989e-18 --5.658156072167891793e-18 -4.106285280551914311e-18 -1.742030982911749756e-19 -1.424440983663824178e-18 -5.480844936769992495e-18 -1.804352161149941056e-18 +7.698791973673844158e-01 +7.698791973673844158e-01 +7.698791973673843048e-01 +7.698791973673843048e-01 +6.913800778225646729e-17 +-2.572591733296812912e-17 +-7.173655366961485181e-17 +2.832446322032650131e-17 +3.478217022700787467e-18 +-1.527472444189700360e-18 +-2.790333412930537917e-18 +8.395888344194466692e-19 +3.637066182233955877e-18 +-2.981293358266952529e-18 +-4.067698972485125619e-18 +3.411926148518122272e-18 diff --git a/tst/regression/mocmc_equilibration.py b/tst/regression/mocmc_equilibration.py index ab4c23680..6a50a1574 100755 --- a/tst/regression/mocmc_equilibration.py +++ b/tst/regression/mocmc_equilibration.py @@ -47,11 +47,14 @@ modified_inputs["radiation/mocmc/nsamp_per_zone"] = 32 modified_inputs["radiation_equilibration/J"] = 0.2 +cmake_extra_args = ["-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off", "-DPHOEBUS_DO_NU_HEAVY=Off"] + code = rt.gold_comparison( variables=["r.p.J", "r.p.H"], input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args=cmake_extra_args, geometry="Minkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/radiation_diffusion.gold b/tst/regression/radiation_diffusion.gold index 52117688e..3444878e3 100644 --- a/tst/regression/radiation_diffusion.gold +++ b/tst/regression/radiation_diffusion.gold @@ -1,204 +1,204 @@ -5.727756994965728402e-07 -2.390134319486193814e-08 -8.552041533534621507e-10 -1.078296430112438385e-10 -9.711876459274814704e-11 -9.713910069953824051e-11 -9.714019477587794905e-11 -9.714018556719315477e-11 -9.714013583064870344e-11 -9.714004143906456544e-11 -9.714136057593570395e-11 -9.715605647034073990e-11 -9.723061089040996236e-11 -9.746785021414422651e-11 -9.798169248803654994e-11 -9.873601435495395353e-11 -9.944825354323743467e-11 -9.985170078244693900e-11 -9.997856789721988854e-11 -9.999862401572621337e-11 -9.999997429126063500e-11 -9.999999997608914604e-11 -1.000000000145484692e-10 -1.000000045740661299e-10 -1.000006404787215883e-10 -1.000486007678750447e-10 -1.023092093125954138e-10 -1.757621936255253417e-10 -1.941590168349628520e-09 -3.504491520959144366e-08 -5.225935904712619433e-07 -6.176559144957398269e-06 -5.779425074201662320e-05 -4.282440483189261051e-04 -2.512772859434293186e-03 -1.167086067769811455e-02 -4.287634506986499200e-02 -1.244570709266687308e-01 -2.850180079099510766e-01 -5.140050326773021316e-01 -7.282509198683035923e-01 -8.067801476054499155e-01 -6.998498162483933749e-01 -4.717382893901956309e-01 -2.462135553878651295e-01 -9.900483254205026673e-02 -3.049183983645771534e-02 -7.143371984330692398e-03 -1.262702652840829451e-03 -1.668041847985367678e-04 -1.627808859937456717e-05 -1.123635279827187067e-06 -5.016698617450419950e-02 -5.311995564210896664e-02 -1.680911903892158554e-02 -2.685308764008071427e-05 --2.355360567551283064e-06 --2.278357889466065552e-09 -2.666990845618345271e-09 -1.268488393554597766e-08 -1.269934976676971702e-08 --4.400954823655228269e-07 --4.079477796975521067e-06 --1.864875973519957813e-05 --5.415651297246726640e-05 --1.063016992923665233e-04 --1.377446224133029192e-04 --1.102671082456988558e-04 --5.036791500535575921e-05 --1.194434774010715570e-05 --1.280766957156953484e-06 --4.683094159786734856e-08 --1.484523354899965649e-10 --4.421560534975262109e-12 --1.451084030538389196e-09 --2.046534387257914043e-07 --1.533153330586361159e-05 --7.046323563657893750e-04 --1.645589022743774651e-02 --4.965832613189426203e-02 --5.020480634544625259e-02 --4.626722341124880900e-02 --4.213012095570772136e-02 --3.798606919926616338e-02 --3.384163722145849124e-02 --2.969559488533482852e-02 --2.554636732861065643e-02 --2.139230530721064616e-02 --1.723174961639098446e-02 --1.306297343227211250e-02 --8.883500610028981226e-03 --4.689001102128260642e-03 --2.397992361798591298e-04 -3.754639271729428174e-03 -8.014743424947094394e-03 -1.230509638607621588e-02 -1.663267006808121992e-02 -2.100717937714585704e-02 -2.543980051006110993e-02 -2.994404537918140055e-02 -3.453717733249578314e-02 -3.924181203534791995e-02 -4.407161256738489907e-02 --1.868767268482240491e-22 --7.641427680194446907e-23 --6.804516732338074833e-23 -7.696573611664989199e-23 --3.029789149918279561e-22 -1.325849849415484424e-22 -1.827833703677350937e-22 --1.798204906089769979e-22 --1.994375227157259929e-22 --1.128720256415789531e-22 -2.806257048517271053e-22 -4.353674803135171031e-23 --9.760997454810156110e-23 -1.896638842126431961e-22 -1.936603512183230315e-23 -1.580851173246986268e-22 --2.920286453783151726e-22 --6.458585200373196152e-23 -3.261438212187573023e-22 --1.635206802771003537e-22 -5.196807219326522329e-23 --1.940874631207208132e-22 -3.202876348596336658e-22 --9.205330069826698777e-23 -6.685343097314401535e-23 -1.876387094514924327e-22 --3.346456175067505297e-22 --8.661806913602193686e-23 -2.714175968898874512e-22 --1.375071315110659645e-22 --3.821544454524002346e-23 -3.363643437678766407e-23 --7.050516093164550709e-23 -1.685568510204387630e-22 --4.654457833402880344e-23 -1.290084028486700240e-22 -1.457948221271299510e-22 -9.072037070181711738e-23 -6.031616359304777192e-23 --3.219915272597726373e-22 --3.661177225926254289e-22 --1.292076190041262082e-22 --1.848766504132850588e-22 -3.576826733278346032e-22 -4.004773267704565626e-22 --1.536059366362080154e-22 --2.384660447196411676e-22 -3.717300217412897763e-22 --1.101313977316121747e-22 --3.319711620057026527e-22 -9.779126994823706531e-23 -2.006372983480654967e-22 --2.627955561810508638e-22 -2.168296403126974973e-23 -3.038608284578627255e-22 -4.545338251117119525e-23 --7.387304247504195904e-23 --5.245124621623921559e-23 -1.887761815603797742e-22 -5.071921698641457001e-24 -8.435045479648305304e-23 --1.516127675197357563e-22 --4.575003927093133333e-23 --1.963465725402569970e-22 -3.705826620196841251e-23 --1.750363343885057716e-22 -1.154917064906796530e-22 -4.486429391980784950e-22 --1.753935150281962650e-22 -3.120498712890031069e-22 -2.060536443397245257e-22 -1.772067652250144103e-22 --8.640881062529312116e-23 --5.903633035570862533e-23 --1.756470543022796535e-22 -7.092859059373325788e-24 --6.643883450454763159e-23 --1.713818450873749800e-22 --6.995206398125411142e-24 --7.727514724108877460e-23 --2.056816450131976979e-22 --2.122987666405352667e-22 --1.616826368358732784e-22 --1.555098971713018895e-22 --3.008697083894963898e-22 --1.637921629454198903e-22 --1.746625146671844762e-22 -2.731092170496370455e-22 --2.751892007889170968e-22 -3.410244855996552411e-23 -1.289209871199362097e-23 --2.333180364727235204e-22 --4.187978621485973221e-23 --1.825614611388922589e-22 -2.755308002943173615e-22 --2.711917168254534971e-22 -8.927134484809464633e-23 -1.693058093012223153e-22 --1.072491368960832741e-22 --5.021986005922945311e-22 -4.021009130812441423e-22 +6.148519464058904541e-07 +2.623892799365704451e-08 +9.539299957556873008e-10 +1.097925175521065226e-10 +9.698685107840106121e-11 +9.700735207148735836e-11 +9.700855170309239901e-11 +9.700854131505957682e-11 +9.700848604280721838e-11 +9.700839486333799533e-11 +9.700998705309238909e-11 +9.702660891623257019e-11 +9.710828702343892631e-11 +9.736260875897138338e-11 +9.790545571475139446e-11 +9.869430905366765583e-11 +9.943312020840976194e-11 +9.984853597427683277e-11 +9.997824932337328797e-11 +9.999861352928875433e-11 +9.999997436561353504e-11 +9.999999997660431154e-11 +1.000000000140515146e-10 +1.000000043840857233e-10 +1.000006084550747897e-10 +1.000456769896741220e-10 +1.021446086957923230e-10 +1.701063277402130924e-10 +1.827686282796141058e-09 +3.335576536443429658e-08 +5.034117650423832286e-07 +6.011002868924303051e-06 +5.672185718319043676e-05 +4.231424312440042060e-04 +2.495696167647177763e-03 +1.163478979356291458e-02 +4.284849082069772108e-02 +1.245451911643267323e-01 +2.853591843479902357e-01 +5.145531879348246296e-01 +7.286884038685831344e-01 +8.068838598063016176e-01 +6.998431817787323928e-01 +4.720041922922508237e-01 +2.467486611988026213e-01 +9.951367284863543716e-02 +3.079012872976234425e-02 +7.260759428897966931e-03 +1.294831044491474934e-03 +1.730107873849609415e-04 +1.712756748856066036e-05 +1.202183306907686552e-06 +5.074225279675498995e-02 +5.402810637784936226e-02 +1.888716606287351232e-02 +3.788337365701142240e-05 +-2.616499114530969193e-06 +-2.286177320353717435e-09 +3.000371620316649112e-09 +1.405787203100672773e-08 +6.773859766534667883e-09 +-5.232686354011619752e-07 +-4.589144658609571317e-06 +-2.034565222209474867e-05 +-5.787117026449888600e-05 +-1.120306814485762340e-04 +-1.437216322540358032e-04 +-1.140893004446133188e-04 +-5.171257580973905743e-05 +-1.217308947984350423e-05 +-1.295554805768857514e-06 +-4.694238308857297540e-08 +-1.452650655100214653e-10 +-4.272717129183675924e-12 +-1.391212583164108648e-09 +-1.944289561274645613e-07 +-1.440455946451621791e-05 +-6.541320671425316377e-04 +-1.543791468150581755e-02 +-4.872072052042272089e-02 +-4.953862261182483129e-02 +-4.569416026514530604e-02 +-4.164081235621738819e-02 +-3.757548493151072894e-02 +-3.350430264466900537e-02 +-2.942556656250817568e-02 +-2.533722626053625665e-02 +-2.123714005338387875e-02 +-1.712313852154304789e-02 +-1.299297077336182896e-02 +-8.843623314542374894e-03 +-4.670205125795492268e-03 +-2.305441085521117239e-04 +3.760799292794245446e-03 +8.030257233360404631e-03 +1.234052436289972098e-02 +1.669840962643823895e-02 +2.111341029293156007e-02 +2.559647842759555161e-02 +3.016088480912522407e-02 +3.482362737397468044e-02 +3.960701533656195428e-02 +4.454103932968823260e-02 +1.331543215889358418e-22 +1.449048012375765925e-22 +-5.840344864391299591e-23 +-4.142925430177124312e-22 +-6.625058095387177299e-23 +-7.140446846678581789e-23 +1.771553100880480093e-22 +9.781944953591423673e-23 +9.162720621882662440e-23 +1.204254755894580195e-22 +1.981917164133671776e-22 +1.837239786602573270e-22 +-6.532090072774898377e-23 +3.388608665940369599e-22 +-2.637135005108609832e-23 +-5.667218291935442503e-23 +3.824257239590099752e-22 +1.642723364845939941e-22 +-1.191764057220626943e-22 +2.032010375432194001e-22 +1.468539127018607578e-22 +1.362816788494320745e-22 +-1.490398551282741487e-22 +2.014876104064194665e-22 +2.945654730970317451e-22 +-2.566953706825307772e-22 +-2.568607864953637649e-22 +-2.248216315953183658e-22 +-8.535929972452356132e-23 +-1.226486645198053177e-22 +3.303108876537525858e-22 +-3.154040624455984280e-22 +1.115516992741287495e-22 +-2.722502965051908163e-22 +-2.214823134498584583e-22 +1.739254321183681078e-22 +2.880609836370706566e-22 +-2.605351091226014014e-22 +2.352664962607755773e-22 +-1.228618762985802809e-22 +7.917302483171992569e-23 +-2.088467446501521971e-22 +8.708499363504617154e-23 +-3.215095351150821572e-22 +1.716397533209515926e-22 +2.865097620584068819e-22 +-2.801131585172059201e-22 +7.248544286539459298e-23 +-1.118118003285572442e-22 +-4.246754181384880607e-23 +1.616548269686875165e-22 +3.536602752843460642e-22 +-2.562182240611372686e-22 +1.420944762750985085e-22 +3.327098699446866535e-23 +4.045582076110651761e-23 +-1.815692516630341649e-22 +-4.888613501607002443e-23 +-2.386824240657490642e-22 +5.610593882581087645e-23 +1.497790119297060339e-22 +1.384318036790286671e-22 +2.322793653141452714e-22 +6.097019390604360411e-23 +-1.271637813864604155e-22 +-9.103367589897254445e-23 +2.176824137475862983e-23 +-1.368971579691912148e-22 +7.639490373611592537e-23 +3.281970443953975512e-24 +2.233467307263979615e-22 +6.278623074712339672e-23 +5.929016282816159961e-23 +2.458367288273487457e-22 +-5.631456179917167376e-23 +-1.858499646409524352e-22 +1.187050096287243954e-22 +-8.394083332511742891e-23 +5.489616617044200368e-22 +4.282216076559168001e-22 +3.921078498099024952e-22 +1.170384473448311927e-22 +1.604361978512895507e-23 +-2.207706446395669817e-22 +-2.537958809470855371e-22 +3.549470522578481163e-22 +-3.072257625811678830e-22 +-2.462839338467965054e-22 +8.965576263635079820e-25 +2.278226835262596707e-22 +-3.860083743465736477e-22 +-2.637660224911399509e-22 +2.931952295852275688e-23 +2.356276635677745275e-22 +1.161521464425352583e-22 +-1.052166011900077566e-23 +1.913216458015005852e-22 +-1.241810591828786866e-23 +-4.486848283273262171e-22 +8.900011991657956885e-23 +-8.485179905361157198e-23 diff --git a/tst/regression/radiation_diffusion.py b/tst/regression/radiation_diffusion.py index 1e36fc001..ce995b35a 100755 --- a/tst/regression/radiation_diffusion.py +++ b/tst/regression/radiation_diffusion.py @@ -50,11 +50,14 @@ modified_inputs["radiation_advection/width"] = 0.0333 modified_inputs["radiation_advection/kappas_init"] = 1.0e3 +cmake_extra_args = ["-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off", "-DPHOEBUS_DO_NU_HEAVY=Off"] + code = rt.gold_comparison( variables=["r.p.J", "r.p.H"], input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args=cmake_extra_args, geometry="Minkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/radiation_equilibration.gold b/tst/regression/radiation_equilibration.gold index c760e2a3b..d18cd1de5 100644 --- a/tst/regression/radiation_equilibration.gold +++ b/tst/regression/radiation_equilibration.gold @@ -1,7 +1,7 @@ -7.590477907760553311e-01 -7.590477907760553311e-01 -7.590477907760553311e-01 -7.590477907760553311e-01 +7.463101650882452143e-01 +7.463101650882452143e-01 +7.463101650882452143e-01 +7.463101650882452143e-01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 diff --git a/tst/regression/radiation_equilibration.py b/tst/regression/radiation_equilibration.py index dfd11e2fc..4b2582791 100755 --- a/tst/regression/radiation_equilibration.py +++ b/tst/regression/radiation_equilibration.py @@ -43,11 +43,14 @@ modified_inputs["opacity/gray_kappa"] = 0.1 modified_inputs["radiation_equilibration/J"] = 0.2 +cmake_extra_args = ["-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off", "-DPHOEBUS_DO_NU_HEAVY=Off"] + code = rt.gold_comparison( variables=["r.p.J", "r.p.H"], input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args=cmake_extra_args, geometry="Minkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/regression_test.py b/tst/regression/regression_test.py index 6f1450e3b..f04f2577e 100644 --- a/tst/regression/regression_test.py +++ b/tst/regression/regression_test.py @@ -29,7 +29,7 @@ BUILD_DIR = "build" RUN_DIR = "run" SOURCE_DIR = "../../../" -NUM_PROCS = 4 # Default values for cmake --build --parallel can overwhelm CI systems +NUM_PROCS = 8 # Default values for cmake --build --parallel can overwhelm CI systems TEMPORARY_INPUT_FILE = "test_input.pin" SCRIPT_NAME = os.path.basename(__main__.__file__).split(".py")[0] @@ -148,7 +148,7 @@ def modify_input(dict_key, value, input_file): # -- Configure and build phoebus with problem-specific options -def build_code(geometry, use_gpu=False, build_type="Release"): +def build_code(geometry, use_gpu=False, build_type="Release", cmake_extra_args=[]): if os.path.isdir(BUILD_DIR): print( f'BUILD_DIR "{BUILD_DIR}" already exists! Clean up before calling a regression test script!' @@ -166,6 +166,8 @@ def build_code(geometry, use_gpu=False, build_type="Release"): else: print(f'Build type "{build_type}" not known!') sys.exit(os.EX_SOFTWARE) + + configure_options += cmake_extra_args configure_options.append("-DPHOEBUS_ENABLE_UNIT_TESTS=OFF") configure_options.append("-DMAX_NUMBER_CONSERVED_VARS=10") configure_options.append("-DPHOEBUS_CACHE_GEOMETRY=ON") @@ -187,6 +189,7 @@ def build_code(geometry, use_gpu=False, build_type="Release"): for option in configure_options: cmake_call.append(option) cmake_call.append(SOURCE_DIR) + print(cmake_call) # Configure call(cmake_call) @@ -238,6 +241,7 @@ def gold_comparison( input_file, modified_inputs={}, executable=None, + cmake_extra_args=[], geometry="Minkowski", use_gpu=False, use_mpiexec=False, @@ -260,7 +264,7 @@ def gold_comparison( if executable is None: executable = os.path.join(BUILD_DIR, "src", "phoebus") - build_code(geometry, use_gpu, build_type) + build_code(geometry, use_gpu, build_type, cmake_extra_args) if os.path.isdir(RUN_DIR): print( @@ -298,7 +302,9 @@ def gold_comparison( if len(variable.shape) > 1: dim = variable.shape[0] for d in range(dim): - variables_data = np.concatenate((variables_data, variable[d, :])) + variables_data = np.concatenate( + (variables_data, variable[d, :].flatten()) + ) else: variables_data = np.concatenate((variables_data, variable)) diff --git a/tst/regression/run/test_input.pin b/tst/regression/run/test_input.pin deleted file mode 100644 index 208b2e594..000000000 --- a/tst/regression/run/test_input.pin +++ /dev/null @@ -1,82 +0,0 @@ - -problem = linear_modes - -problem_id = hydro_modes # problem ID: basename of output filenames - -variables = p.density, & - c.density, & - p.velocity, & - p.energy, & - p.bfield, & - pressure, & - cs -file_type = hdf5 # Tabular data dump -dt = 0.5 # time increment between outputs - -nlim = -1 # cycle limit -tlim = 0.4 # time limit -integrator = rk2 # time integration algorithm -ncycle_out = 10 # interval for stdout summary info -dt_init_fact = 0.1 - -nghost = 4 -nx1 = 64 -x1min = 0 # minimum value of X1 -x1max = 1 # maximum value of X1 -ix1_bc = periodic # Inner-X1 boundary condition flag -ox1_bc = periodic # Outer-X1 boundary condition flag -nx2 = 64 -x2min = 0 # minimum value of X2 -x2max = 1 # maximum value of X2 -ix2_bc = periodic # Inner-X2 boundary condition flag -ox2_bc = periodic # Outer-X2 boundary condition flag -nx3 = 1 # Number of zones in X3-direction -x3min = -1 # minimum value of X3 -x3max = 1 # maximum value of X3 -ix3_bc = periodic # Inner-X3 boundary condition flag -ox3_bc = periodic # Outer-X3 boundary condition flfgag -num_threads = 1 # maximum number of OMP threads - - -field = c.c.bulk.rho -method = derivative_order_1 -max_level = 3 - -vx = 0. # 0.436436 -vy = 0. # 0.436436 - -kmult = 4 # Multiply k by this factor -a = 0.3 # amplitude of wiggle -alpha = 1. # 0.85 # lapse -vy = 0. # 0.524631 -do_fd_on_grid = false - -type = IdealGas -Gamma = 1.333333333333333333 -Cv = 1.0 - -hydro = true -he = false -3t = false -rad = false - -mhd = true -xorder = 2 -cfl = 0.4 -riemann = hll -c2p_max_iter = 100 -recon = linear -c2p_tol = 1.e-12 -c2p_method = robust - -physics = hydro -mode = sound -amplitude = 1.e-5 - -nx1 = 64 - -nx2 = 64 - -nx1 = 64 - -nx2 = 64 diff --git a/tst/regression/thincooling.py b/tst/regression/thincooling.py index 39789b360..1379723bb 100755 --- a/tst/regression/thincooling.py +++ b/tst/regression/thincooling.py @@ -38,11 +38,14 @@ modified_inputs = {} modified_inputs["radiation/method"] = "monte_carlo" +cmake_extra_args = ["-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off", "-DPHOEBUS_DO_NU_HEAVY=Off"] + code = rt.gold_comparison( variables=["p.density", "p.energy"], input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args=cmake_extra_args, geometry="Minkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/thincooling_coolingfunction.py b/tst/regression/thincooling_coolingfunction.py index e97d477ef..f9d7195f5 100755 --- a/tst/regression/thincooling_coolingfunction.py +++ b/tst/regression/thincooling_coolingfunction.py @@ -38,11 +38,14 @@ modified_inputs = {} modified_inputs["radiation/method"] = "cooling_function" +cmake_extra_args = ["-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off", "-DPHOEBUS_DO_NU_HEAVY=Off"] + code = rt.gold_comparison( variables=["p.density", "p.energy"], input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args=cmake_extra_args, geometry="Minkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec,