From 00205d11762bfd3ebae6533f6bd78e3d1d3f175a Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Thu, 17 Oct 2024 20:06:28 +0000 Subject: [PATCH 01/48] cleanup pgens, fix tracer init in advection problem and move to postinit --- src/pgen/advection.cpp | 136 ++++++++++++++++++++++++++--------------- src/pgen/pgen.hpp | 1 + src/pgen/torus.cpp | 3 +- 3 files changed, 88 insertions(+), 52 deletions(-) diff --git a/src/pgen/advection.cpp b/src/pgen/advection.cpp index bc1eae066..12e6df710 100644 --- a/src/pgen/advection.cpp +++ b/src/pgen/advection.cpp @@ -104,58 +104,94 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { }); fluid::PrimitiveToConserved(rc.get()); +} - /* tracer init section */ - if (do_tracers) { - auto &sc = pmb->swarm_data.Get(); - auto &swarm = pmb->swarm_data.Get()->Get("tracers"); - auto rng_pool = tracer_pkg->Param("rng_pool"); - - const Real &x_min = pmb->coords.Xf<1>(ib.s); - const Real &y_min = pmb->coords.Xf<2>(jb.s); - const Real &z_min = pmb->coords.Xf<3>(kb.s); - const Real &x_max = pmb->coords.Xf<1>(ib.e + 1); - const Real &y_max = pmb->coords.Xf<2>(jb.e + 1); - const Real &z_max = pmb->coords.Xf<3>(kb.e + 1); - - // as for num_tracers on each block... will get too many on multiple blocks - // TODO: distribute amongst blocks. - const auto num_tracers_total = tracer_pkg->Param("num_tracers"); - const int number_block = num_tracers_total; - - auto new_particles_context = swarm->AddEmptyParticles(number_block); - - auto &x = swarm->Get("x").Get(); - auto &y = swarm->Get("y").Get(); - auto &z = swarm->Get("z").Get(); - auto &id = swarm->Get("id").Get(); - - auto swarm_d = swarm->GetDeviceContext(); - - const int gid = pmb->gid; - const int max_active_index = new_particles_context.GetNewParticlesMaxIndex(); - pmb->par_for( - "ProblemGenerator::Torus::DistributeTracers", 0, max_active_index, - KOKKOS_LAMBDA(const int n) { - if (swarm_d.IsActive(n)) { - auto rng_gen = rng_pool.get_state(); - - // sample in ye ball - Real r2 = 1.0 + rin * rin; // init > rin^2 - while (r2 > rin * rin) { - x(n) = x_min + rng_gen.drand() * (x_max - x_min); - y(n) = y_min + rng_gen.drand() * (y_max - y_min); - z(n) = z_min + rng_gen.drand() * (z_max - z_min); - r2 = x(n) * x(n) + y(n) * y(n) + z(n) * z(n); +void PostInitializationModifier(ParameterInput *pin, Mesh *pmesh) { + + const int ndim = pmesh->ndim; + for (auto &pmb : pmesh->block_list) { + auto &rc = pmb->meshblock_data.Get(); + auto tracer_pkg = pmb->packages.Get("tracers"); + bool do_tracers = tracer_pkg->Param("active"); + const Real rin = pin->GetOrAddReal("advection", "rin", 0.1); + const Real v_inner = ndim == 3 ? (4. / 3.) * M_PI * std::pow(rin, 3.) + : ndim == 2 ? M_PI * rin * rin + : rin; + + + auto geom = Geometry::GetCoordinateSystem(rc.get()); + auto coords = pmb->coords; + if (do_tracers) { + const auto num_tracers_total = tracer_pkg->Param("num_tracers"); + auto rng_pool = tracer_pkg->Param("rng_pool"); + auto &swarm = rc->GetSwarmData()->Get("tracers"); + + auto ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + auto jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + auto kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + Real number_block = 0.0; + + // Get fraction of block containing ye sphere + pmb->par_reduce( + "Phoebus::ProblemGenerator::Torus::BlockTracerNumber", kb.s, kb.e, jb.s, jb.e, + ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i, + Real &number_block_reduce) { + const Real dx1 = coords.Dxc<1>(k, j, i); + const Real dx2 = coords.Dxc<2>(k, j, i); + const Real dx3 = coords.Dxc<3>(k, j, i); + const Real x1 = coords.Xc<1>(k, j, i); + const Real x2 = coords.Xc<2>(k, j, i); + const Real x3 = coords.Xc<3>(k, j, i); + + if (x1 * x1 + x2 * x2 + x3 * x3 < rin * rin) { + Real vol_block = dx1 * dx2 * dx3; + number_block_reduce += vol_block; } - id(n) = num_tracers_total * gid + n; - - bool on_current_mesh_block = true; - swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); - rng_pool.free_state(rng_gen); - } - }); + }, + Kokkos::Sum(number_block)); + number_block /= v_inner; + number_block = number_block * num_tracers_total; + const int num_tracers_block = (int)number_block; + + // distribute + auto new_particles_context = swarm->AddEmptyParticles(number_block); + + auto &x = swarm->Get(swarm_position::x::name()).Get(); + auto &y = swarm->Get(swarm_position::y::name()).Get(); + auto &z = swarm->Get(swarm_position::z::name()).Get(); + auto &id = swarm->Get("id").Get(); + + auto swarm_d = swarm->GetDeviceContext(); + + const int gid = pmb->gid; + const int max_active_index = new_particles_context.GetNewParticlesMaxIndex(); + pmb->par_for( + "ProblemGenerator::Advection::DistributeTracers", 0, max_active_index, + KOKKOS_LAMBDA(const int new_n) { +// if (swarm_d.IsActive(n)) { + const int n = new_particles_context.GetNewParticleIndex(new_n); + auto rng_gen = rng_pool.get_state(); + + // sample in ye ball + Real r2 = 1.0 + rin * rin; // init > rin^2 + while (r2 > rin * rin) { + x(n) = -rin + + rng_gen.drand() * 2.0 * rin; // just sampling x \in [-rin, +rin] + y(n) = -rin + rng_gen.drand() * 2.0 * rin; + z(n) = -rin + rng_gen.drand() * 2.0 * rin; + r2 = x(n) * x(n) + y(n) * y(n) + z(n) * z(n); + } + id(n) = num_tracers_total * gid + n; + + bool on_current_mesh_block = true; + swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); + rng_pool.free_state(rng_gen); + // } + }); + } } -} + +} // PostInitializationModifier } // namespace advection diff --git a/src/pgen/pgen.hpp b/src/pgen/pgen.hpp index 1491f3fdd..d3ee017a3 100644 --- a/src/pgen/pgen.hpp +++ b/src/pgen/pgen.hpp @@ -60,6 +60,7 @@ using namespace parthenon::package::prelude; // the name here #define FOREACH_POSTINIT_MODIFIER \ POSTINIT_MODIFIER(phoebus) \ + POSTINIT_MODIFIER(advection) \ POSTINIT_MODIFIER(torus) /* diff --git a/src/pgen/torus.cpp b/src/pgen/torus.cpp index 6db236c9f..b2d7211ba 100644 --- a/src/pgen/torus.cpp +++ b/src/pgen/torus.cpp @@ -630,8 +630,7 @@ void PostInitializationModifier(ParameterInput *pin, Mesh *pmesh) { // tracer initialization. if (do_tracers) { - auto &sc = pmb->swarm_data.Get(); - auto &swarm = pmb->swarm_data.Get()->Get("tracers"); + auto &swarm = rc->GetSwarmData()->Get("tracers"); auto rng_pool = tracer_pkg->Param("rng_pool"); const auto num_tracers_total = tracer_pkg->Param("num_tracers"); From e33d1c7a532230261987abb202ba7dbd5eb83042 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Thu, 17 Oct 2024 20:07:03 +0000 Subject: [PATCH 02/48] boundary condition work --- src/phoebus_boundaries/phoebus_boundaries.cpp | 43 ++++++++++++------- src/phoebus_boundaries/phoebus_boundaries.hpp | 39 ++++++++--------- 2 files changed, 44 insertions(+), 38 deletions(-) diff --git a/src/phoebus_boundaries/phoebus_boundaries.cpp b/src/phoebus_boundaries/phoebus_boundaries.cpp index e440ddaa9..4e9d5ca07 100644 --- a/src/phoebus_boundaries/phoebus_boundaries.cpp +++ b/src/phoebus_boundaries/phoebus_boundaries.cpp @@ -33,6 +33,8 @@ using namespace parthenon::package::prelude; namespace Boundaries { +void SwarmNoWorkBC(std::shared_ptr &swarm) {} + parthenon::TopologicalElement CC = parthenon::TopologicalElement::CC; // Copied out of Parthenon, with slight modification @@ -84,8 +86,9 @@ void GenericBC(std::shared_ptr> &rc, bool coarse) { const bool rescale = fluid->Param("bc_vars") == "conserved"; // Do the thing + const bool fine = false; // NOTE(BLB): will need changing for fine fields pmb->par_for_bndry( - label, nb, domain, CC, coarse, + label, nb, domain, CC, coarse, fine, KOKKOS_LAMBDA(const int &l, const int &k, const int &j, const int &i) { int kref, jref, iref, sgn; if (TYPE == BCType::Reflect) { @@ -128,9 +131,10 @@ void OutflowInnerX1(std::shared_ptr> &rc, bool coarse) { auto &fluid = rc->GetMeshPointer()->packages.Get("fluid"); std::string bc_vars = fluid->Param("bc_vars"); + const bool fine = false; if (bc_vars == "conserved") { pmb->par_for_bndry( - "OutflowInnerX1Cons", nb, domain, CC, coarse, + "OutflowInnerX1Cons", nb, domain, CC, coarse, fine, KOKKOS_LAMBDA(const int &l, const int &k, const int &j, const int &i) { Real detg_ref = geom.DetGamma(CellLocation::Cent, k, j, ref); Real detg = geom.DetGamma(CellLocation::Cent, k, j, i); @@ -139,7 +143,7 @@ void OutflowInnerX1(std::shared_ptr> &rc, bool coarse) { }); } else if (bc_vars == "primitive") { pmb->par_for_bndry( - "OutflowInnerX1Prim", nb, domain, CC, coarse, + "OutflowInnerX1Prim", nb, domain, CC, coarse, fine, KOKKOS_LAMBDA(const int &l, const int &k, const int &j, const int &i) { q(l, k, j, i) = q(l, k, j, ref); }); @@ -165,8 +169,9 @@ void PolarInnerX2(std::shared_ptr> &rc, bool coarse) { const auto idx_pvel = imap.GetFlatIdx(fluid_prim::velocity::name(), false); const auto idx_pb = imap.GetFlatIdx(fluid_prim::bfield::name(), false); + const bool fine = false; pmb->par_for_bndry( - "PolarInnerX2Prim", nb, domain, CC, coarse, + "PolarInnerX2Prim", nb, domain, CC, coarse, fine, KOKKOS_LAMBDA(const int &l, const int &k, const int &j, const int &i) { const int jref = -j + 2 * j0 - 1; if (l == idx_pvel(1)) { @@ -199,8 +204,9 @@ void PolarOuterX2(std::shared_ptr> &rc, bool coarse) { const auto idx_pb = imap.GetFlatIdx(fluid_prim::bfield::name(), false); const std::string label = "PolarOuterX2Prim"; + const bool fine = false; pmb->par_for_bndry( - label, nb, domain, CC, coarse, + label, nb, domain, CC, coarse, fine, KOKKOS_LAMBDA(const int &l, const int &k, const int &j, const int &i) { const int jref = -j + 2 * (j0 + 1) - 1; if (l == idx_pvel(1)) { @@ -235,9 +241,10 @@ void OutflowOuterX1(std::shared_ptr> &rc, bool coarse) { std::string bc_vars = fluid->Param("bc_vars"); const int num_species = rad->Param("active") ? rad->Param("num_species") : 0; + const bool fine = false; if (bc_vars == "conserved") { pmb->par_for_bndry( - "OutflowOuterX1Cons", nb, IndexDomain::outer_x1, CC, coarse, + "OutflowOuterX1Cons", nb, IndexDomain::outer_x1, CC, coarse, fine, KOKKOS_LAMBDA(const int &l, const int &k, const int &j, const int &i) { Real detg_ref = geom.DetGamma(CellLocation::Cent, k, j, ref); Real detg = geom.DetGamma(CellLocation::Cent, k, j, i); @@ -246,7 +253,7 @@ void OutflowOuterX1(std::shared_ptr> &rc, bool coarse) { }); } else if (bc_vars == "primitive") { pmb->par_for_bndry( - "OutflowOuterX1Prim", nb, domain, CC, coarse, + "OutflowOuterX1Prim", nb, domain, CC, coarse, fine, KOKKOS_LAMBDA(const int &l, const int &k, const int &j, const int &i) { q(l, k, j, i) = q(l, k, j, ref); }); @@ -335,6 +342,9 @@ TaskStatus ConvertBoundaryConditions(std::shared_ptr> &rc) { // TODO(BRR) Is this always true? const bool coarse = false; + // NOTE(BLB): setting fine to false. Will require refactoring + // if we use fine fields in the future, ala parthenon/pull/991 + const bool fine = false; PackIndexMap imap; std::vector vars{fluid_prim::velocity::name(), @@ -348,7 +358,7 @@ TaskStatus ConvertBoundaryConditions(std::shared_ptr> &rc) { const int num_species = pkg_rad->Param("active") ? pkg_rad->Param("num_species") : 0; pmb->par_for_bndry( - "OutflowOuterX1PrimFixup", nb1, domain, CC, coarse, + "OutflowOuterX1PrimFixup", nb1, domain, CC, coarse, fine, KOKKOS_LAMBDA(const int &dummy, const int &k, const int &j, const int &i) { // Enforce u^1 >= 0 Real vcon[3] = {q(pv_lo, k, j, i), q(pv_lo + 1, k, j, i), @@ -498,21 +508,22 @@ void ProcessBoundaryConditions(parthenon::ParthenonManager &pman) { if (rad_method == "mocmc") { pman.app_input ->swarm_boundary_conditions[parthenon::BoundaryFace::inner_x1] = - Boundaries::SetSwarmNoWorkBC; + Boundaries::SwarmNoWorkBC; } else { pman.app_input ->swarm_boundary_conditions[parthenon::BoundaryFace::inner_x1] = - Boundaries::SetSwarmIX1Outflow; + // Boundaries::SetSwarmIX1Outflow; + parthenon::BoundaryFunction::SwarmOutflowInnerX1; } } else if (outer == 1) { if (rad_method == "mocmc") { pman.app_input ->swarm_boundary_conditions[parthenon::BoundaryFace::outer_x1] = - Boundaries::SetSwarmNoWorkBC; + Boundaries::SwarmNoWorkBC; } else { pman.app_input ->swarm_boundary_conditions[parthenon::BoundaryFace::outer_x1] = - Boundaries::SetSwarmOX1Outflow; + parthenon::BoundaryFunction::SwarmOutflowOuterX1; } } } @@ -521,21 +532,21 @@ void ProcessBoundaryConditions(parthenon::ParthenonManager &pman) { if (rad_method == "mocmc") { pman.app_input ->swarm_boundary_conditions[parthenon::BoundaryFace::inner_x2] = - Boundaries::SetSwarmNoWorkBC; + Boundaries::SwarmNoWorkBC; } else { pman.app_input ->swarm_boundary_conditions[parthenon::BoundaryFace::inner_x2] = - Boundaries::SetSwarmIX2Outflow; + parthenon::BoundaryFunction::SwarmOutflowInnerX2; } } else if (outer == 1) { if (rad_method == "mocmc") { pman.app_input ->swarm_boundary_conditions[parthenon::BoundaryFace::outer_x2] = - Boundaries::SetSwarmNoWorkBC; + Boundaries::SwarmNoWorkBC; } else { pman.app_input ->swarm_boundary_conditions[parthenon::BoundaryFace::outer_x2] = - Boundaries::SetSwarmOX2Outflow; + parthenon::BoundaryFunction::SwarmOutflowOuterX2; } } } diff --git a/src/phoebus_boundaries/phoebus_boundaries.hpp b/src/phoebus_boundaries/phoebus_boundaries.hpp index f6d69c961..dbcf72e77 100644 --- a/src/phoebus_boundaries/phoebus_boundaries.hpp +++ b/src/phoebus_boundaries/phoebus_boundaries.hpp @@ -18,11 +18,14 @@ #include +#include +#include #include #include #include #include using namespace parthenon::package::prelude; +using namespace parthenon::BoundaryFunction; namespace Boundaries { @@ -46,28 +49,20 @@ void OutflowOuterX3(std::shared_ptr> &rc, bool coarse); void ReflectInnerX3(std::shared_ptr> &rc, bool coarse); void ReflectOuterX3(std::shared_ptr> &rc, bool coarse); -class ParticleBoundNoWork : public parthenon::ParticleBound { - public: - KOKKOS_INLINE_FUNCTION void - Apply(const int n, double &x, double &y, double &z, - const parthenon::SwarmDeviceContext &swarm_d) const override {} -}; - -inline auto SetSwarmIX1Outflow() { - return parthenon::DeviceAllocate(); -} -inline auto SetSwarmOX1Outflow() { - return parthenon::DeviceAllocate(); -} -inline auto SetSwarmIX2Outflow() { - return parthenon::DeviceAllocate(); -} -inline auto SetSwarmOX2Outflow() { - return parthenon::DeviceAllocate(); -} -inline auto SetSwarmNoWorkBC() { - return parthenon::DeviceAllocate(); -} +void SwarmNoWorkBC(std::shared_ptr &swarm); + +// inline auto SetSwarmIX1Outflow() { +// return parthenon::BoundaryFunction::OutflowInnerX1; +// } +// inline auto SetSwarmOX1Outflow() { +// return parthenon::BoundaryFunction::OutflowOuterX1; +// } +// inline auto SetSwarmIX2Outflow() { +// return parthenon::BoundaryFunction::OutflowInnerX2; +// } +// inline auto SetSwarmOX2Outflow() { +// return parthenon::BoundaryFunction::OutflowOuterX2; +// } TaskStatus ConvertBoundaryConditions(std::shared_ptr> &rc); From 59b10ab5b7912baea557ac8bc1c19cca87e034cc Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Thu, 17 Oct 2024 20:08:19 +0000 Subject: [PATCH 03/48] refactoring, misc, lots of changes, bad commit practice... --- src/fixup/fixup.cpp | 1 - src/fixup/fixup_c2p.cpp | 1 - src/fixup/fixup_netfield.cpp | 1 - src/fixup/fixup_particles.cpp | 3 +- src/fixup/fixup_radc2p.cpp | 1 - src/fixup/fixup_src.cpp | 1 - src/fluid/fluid.cpp | 65 ++++++---- src/fluid/riemann.hpp | 24 ++-- src/geometry/geometry.cpp | 3 +- src/phoebus_driver.cpp | 13 +- src/phoebus_driver.hpp | 3 +- src/radiation/mocmc.cpp | 40 +++--- src/radiation/monte_carlo.cpp | 71 ++++++----- src/radiation/radiation.hpp | 4 +- src/tracers/tracers.cpp | 222 ++++++++++++++++++++-------------- 15 files changed, 264 insertions(+), 189 deletions(-) diff --git a/src/fixup/fixup.cpp b/src/fixup/fixup.cpp index f2dbc0c4a..879b79afa 100644 --- a/src/fixup/fixup.cpp +++ b/src/fixup/fixup.cpp @@ -15,7 +15,6 @@ #include "fixup.hpp" -#include #include #include "analysis/history.hpp" diff --git a/src/fixup/fixup_c2p.cpp b/src/fixup/fixup_c2p.cpp index 5e7147bc9..23ae5208b 100644 --- a/src/fixup/fixup_c2p.cpp +++ b/src/fixup/fixup_c2p.cpp @@ -15,7 +15,6 @@ #include "fixup.hpp" -#include #include #include "fluid/con2prim_robust.hpp" diff --git a/src/fixup/fixup_netfield.cpp b/src/fixup/fixup_netfield.cpp index 3195b0a75..ca4787564 100644 --- a/src/fixup/fixup_netfield.cpp +++ b/src/fixup/fixup_netfield.cpp @@ -15,7 +15,6 @@ #include "fixup.hpp" -#include #include #include diff --git a/src/fixup/fixup_particles.cpp b/src/fixup/fixup_particles.cpp index 341fd2551..b5762863f 100644 --- a/src/fixup/fixup_particles.cpp +++ b/src/fixup/fixup_particles.cpp @@ -38,7 +38,8 @@ TaskStatus PurgeParticles(MeshBlockData *rc, const std::string swarmName) /* only do this when FMKS is used. */ if constexpr (std::is_same::value) { auto *pmb = rc->GetParentPointer(); - auto &swarm = pmb->swarm_data.Get()->Get(swarmName); + // auto &swarm = rc->swarm_data.Get()->Get(swarmName); + auto &swarm = rc->GetSwarmData()->Get(swarmName); auto &x = swarm->Get("x").Get(); auto &y = swarm->Get("y").Get(); diff --git a/src/fixup/fixup_radc2p.cpp b/src/fixup/fixup_radc2p.cpp index 3ec3b3d75..f593e1f0c 100644 --- a/src/fixup/fixup_radc2p.cpp +++ b/src/fixup/fixup_radc2p.cpp @@ -15,7 +15,6 @@ #include "fixup.hpp" -#include #include #include "fluid/con2prim_robust.hpp" diff --git a/src/fixup/fixup_src.cpp b/src/fixup/fixup_src.cpp index 54f7feac9..73ca6951c 100644 --- a/src/fixup/fixup_src.cpp +++ b/src/fixup/fixup_src.cpp @@ -15,7 +15,6 @@ #include "fixup.hpp" -#include #include #include "fluid/con2prim_robust.hpp" diff --git a/src/fluid/fluid.cpp b/src/fluid/fluid.cpp index 6e8be2b36..be743716c 100644 --- a/src/fluid/fluid.cpp +++ b/src/fluid/fluid.cpp @@ -768,6 +768,7 @@ TaskStatus CalculateFluxes(MeshBlockData *rc) { } TaskStatus FluxCT(MeshBlockData *rc) { + using parthenon::MakePackDescriptor; Mesh *pmesh = rc->GetMeshPointer(); auto &fluid = pmesh->packages.Get("fluid"); if (!fluid->Param("mhd") || !fluid->Param("active") || @@ -780,49 +781,65 @@ TaskStatus FluxCT(MeshBlockData *rc) { IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); IndexRange kb = rc->GetBoundsK(IndexDomain::interior); - auto f1 = rc->Get(fluid_cons::bfield::name()).flux[X1DIR]; - auto f2 = rc->Get(fluid_cons::bfield::name()).flux[X2DIR]; - auto f3 = rc->Get(fluid_cons::bfield::name()).flux[X3DIR]; + auto &resolved_pkgs = pmesh->resolved_packages; + //static auto desc = MakePackDescriptor(resolved_pkgs.get(), {parthenon::Metadata::WithFluxes}, {parthenon::PDOpt::WithFluxes}); + static auto desc = MakePackDescriptor(resolved_pkgs.get(), {}, {parthenon::PDOpt::WithFluxes}); + //const auto v = rc->PackVariablesAndFluxes(std::vector{fluid_cons::bfield::name()}); + //auto f1 = rc->Get("bnd_flux::" + fluid_cons::bfield::name()).data.Get(0,0,0); + //auto f2 = rc->Get("bnd_flux::" + fluid_cons::bfield::name()).data.Get(1,0,0); + //auto f3 = rc->Get("bnd_flux::" + fluid_cons::bfield::name()).data.Get(2,0,0); + auto v = desc.GetPack(rc); + // auto f1 = v.flux(0, X1DIR, fluid_cons::bfield::name(), k, j, i); + // auto f2 = v.flux(0, X2DIR, fluid_cons::bfield::name(), k, j, i); + // auto f3 = v.flux(0, X3DIR, fluid_cons::bfield::name(), k, j, i); + // auto f1 = rc->Get(fluid_cons::bfield::name()).flux[X1DIR]; + // auto f2 = rc->Get(fluid_cons::bfield::name()).flux[X2DIR]; + // auto f3 = rc->Get(fluid_cons::bfield::name()).flux[X3DIR]; auto emf = rc->Get(internal_variables::emf::name()).data; + //print if (ndim == 2) { parthenon::par_for( DEFAULT_LOOP_PATTERN, "FluxCT::EMF::2D", DevExecSpace(), kb.s, kb.e, jb.s, jb.e + 1, ib.s, ib.e + 1, KOKKOS_LAMBDA(const int k, const int j, const int i) { - emf(k, j, i) = 0.25 * (f1(1, k, j, i) + f1(1, k, j - 1, i) - f2(0, k, j, i) - - f2(0, k, j, i - 1)); + emf(k, j, i) = + 0.25 * (v.flux(0, X1DIR, 1, k, j, i) + v.flux(0, X1DIR, 1, k, j - 1, i) - + v.flux(0, X2DIR, 0, k, j, i) - v.flux(0, X2DIR, 0, k, j, i - 1)); }); parthenon::par_for( DEFAULT_LOOP_PATTERN, "FluxCT::Flux::2D", DevExecSpace(), kb.s, kb.e, jb.s, jb.e + 1, ib.s, ib.e + 1, KOKKOS_LAMBDA(const int k, const int j, const int i) { - f1(0, k, j, i) = 0.0; - f1(1, k, j, i) = 0.5 * (emf(k, j, i) + emf(k, j + 1, i)); - f2(0, k, j, i) = -0.5 * (emf(k, j, i) + emf(k, j, i + 1)); - f2(1, k, j, i) = 0.0; + v.flux(0, X1DIR, 0, k, j, i) = 0.0; + v.flux(0, X1DIR, 1, k, j, i) = 0.5 * (emf(k, j, i) + emf(k, j + 1, i)); + v.flux(0, X2DIR, 0, k, j, i) = -0.5 * (emf(k, j, i) + emf(k, j, i + 1)); + v.flux(0, X2DIR, 1, k, j, i) = 0.0; }); } else if (ndim == 3) { parthenon::par_for( DEFAULT_LOOP_PATTERN, "FluxCT::EMF::3D", DevExecSpace(), kb.s, kb.e + 1, jb.s, jb.e + 1, ib.s, ib.e + 1, KOKKOS_LAMBDA(const int k, const int j, const int i) { - emf(0, k, j, i) = 0.25 * (f2(2, k, j, i) + f2(2, k - 1, j, i) - f3(1, k, j, i) - - f3(1, k, j - 1, i)); - emf(1, k, j, i) = -0.25 * (f1(2, k, j, i) + f1(2, k - 1, j, i) - - f3(0, k, j, i) - f3(0, k, j, i - 1)); - emf(2, k, j, i) = 0.25 * (f1(1, k, j, i) + f1(1, k, j - 1, i) - f2(0, k, j, i) - - f2(0, k, j, i - 1)); + emf(0, k, j, i) = + 0.25 * (v.flux(0, X2DIR, 2, k, j, i) + v.flux(0, X2DIR, 2, k - 1, j, i) - + v.flux(0, X3DIR, 1, k, j, i) - v.flux(0, X3DIR, 1, k, j - 1, i)); + emf(1, k, j, i) = + -0.25 * (v.flux(0, X1DIR, 2, k, j, i) + v.flux(0, X1DIR, 2, k - 1, j, i) - + v.flux(0, X3DIR, 0, k, j, i) - v.flux(0, X3DIR, 0, k, j, i - 1)); + emf(2, k, j, i) = + 0.25 * (v.flux(0, X1DIR, 1, k, j, i) + v.flux(0, X1DIR, 1, k, j - 1, i) - + v.flux(0, X2DIR, 0, k, j, i) - v.flux(0, X2DIR, 0, k, j, i - 1)); }); parthenon::par_for( DEFAULT_LOOP_PATTERN, "FluxCT::Flux::3D", DevExecSpace(), kb.s, kb.e + 1, jb.s, jb.e + 1, ib.s, ib.e + 1, KOKKOS_LAMBDA(const int k, const int j, const int i) { - f1(0, k, j, i) = 0.0; - f1(1, k, j, i) = 0.5 * (emf(2, k, j, i) + emf(2, k, j + 1, i)); - f1(2, k, j, i) = -0.5 * (emf(1, k, j, i) + emf(1, k + 1, j, i)); - f2(0, k, j, i) = -0.5 * (emf(2, k, j, i) + emf(2, k, j, i + 1)); - f2(1, k, j, i) = 0.0; - f2(2, k, j, i) = 0.5 * (emf(0, k, j, i) + emf(0, k + 1, j, i)); - f3(0, k, j, i) = 0.5 * (emf(1, k, j, i) + emf(1, k, j, i + 1)); - f3(1, k, j, i) = -0.5 * (emf(0, k, j, i) + emf(0, k, j + 1, i)); - f3(2, k, j, i) = 0.0; + v.flux(0, X1DIR, 0, k, j, i) = 0.0; + v.flux(0, X1DIR, 1, k, j, i) = 0.5 * (emf(2, k, j, i) + emf(2, k, j + 1, i)); + v.flux(0, X1DIR, 2, k, j, i) = -0.5 * (emf(1, k, j, i) + emf(1, k + 1, j, i)); + v.flux(0, X2DIR, 0, k, j, i) = -0.5 * (emf(2, k, j, i) + emf(2, k, j, i + 1)); + v.flux(0, X2DIR, 1, k, j, i) = 0.0; + v.flux(0, X2DIR, 2, k, j, i) = 0.5 * (emf(0, k, j, i) + emf(0, k + 1, j, i)); + v.flux(0, X3DIR, 0, k, j, i) = 0.5 * (emf(1, k, j, i) + emf(1, k, j, i + 1)); + v.flux(0, X3DIR, 1, k, j, i) = -0.5 * (emf(0, k, j, i) + emf(0, k, j + 1, i)); + v.flux(0, X3DIR, 2, k, j, i) = 0.0; }); } diff --git a/src/fluid/riemann.hpp b/src/fluid/riemann.hpp index 02500d4ee..bb0cdefe9 100644 --- a/src/fluid/riemann.hpp +++ b/src/fluid/riemann.hpp @@ -70,8 +70,16 @@ class FluxState { } static void FluxVars(const std::string &var) { flux_vars.push_back(var); } - static std::vector ReconVars() { return recon_vars; } - static std::vector FluxVars() { return flux_vars; } + static std::vector ReconVars() { + for ( auto & v : recon_vars ) { + } + return recon_vars; + } + static std::vector FluxVars() { + for ( auto & v : flux_vars ) { + } + return flux_vars; + } KOKKOS_FORCEINLINE_FUNCTION int NumConserved() const { return ncons; } @@ -229,12 +237,12 @@ class FluxState { pye(imap[fluid_prim::ye::name()].second), prs(imap[fluid_prim::pressure::name()].first), gm1(imap[fluid_prim::gamma1::name()].first), - crho(imap[fluid_cons::density::name()].first), - cmom_lo(imap[fluid_cons::momentum::name()].first), - ceng(imap[fluid_cons::energy::name()].first), - cb_lo(imap[fluid_cons::bfield::name()].first), - cb_hi(imap[fluid_cons::bfield::name()].second), - cye(imap[fluid_cons::ye::name()].first), + crho(imap[fluid_prim::density::name()].first), + cmom_lo(imap[fluid_prim::velocity::name()].first), + ceng(imap[fluid_prim::energy::name()].first), + cb_lo(imap[fluid_prim::bfield::name()].first), + cb_hi(imap[fluid_prim::bfield::name()].second), + cye(imap[fluid_prim::ye::name()].first), ncons(5 + (pb_hi - pb_lo + 1) + (cye > 0)) { PARTHENON_REQUIRE_THROWS( ncons <= NCONS_MAX, diff --git a/src/geometry/geometry.cpp b/src/geometry/geometry.cpp index 2ceff06d0..c38afa93b 100644 --- a/src/geometry/geometry.cpp +++ b/src/geometry/geometry.cpp @@ -54,8 +54,9 @@ std::shared_ptr Initialize(ParameterInput *pin) { Metadata gcoord_cell = Metadata({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, cell_shape); // TODO(JMM): Make this actual node-centered data when available + // TODO: Warning: g.n.coord output broken in Parthenon currently std::vector node_shape = {dims.nx1 + 1, dims.nx2 + 1, dims.nx3 + 1, 4}; - Metadata gcoord_node = Metadata({Metadata::Derived, Metadata::OneCopy}, node_shape); + Metadata gcoord_node = Metadata({Metadata::Derived, Metadata::OneCopy, Metadata::None}, node_shape); geometry->AddField(geometric_variables::cell_coords::name(), gcoord_cell); geometry->AddField(geometric_variables::node_coords::name(), gcoord_node); diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 998c9679f..23df07a04 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -324,7 +324,7 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { if (rad_mocmc_active) { using MDT = std::remove_pointer::type; // TODO(BRR) stage_name[stage - 1]? - auto &sd0 = pmb->swarm_data.Get(stage_name[integrator->nstages]); + auto &sd0 = pmb->meshblock_data.Get()->GetSwarmData(); auto samples_transport = tl.AddTask(none, radiation::MOCMCTransport, sc0.get(), dt); auto send = tl.AddTask(samples_transport, &SwarmContainer::Send, sd0.get(), @@ -580,7 +580,7 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { for (int i = 0; i < blocks.size(); i++) { auto &tl = sync_region_tr[0]; auto &pmb = blocks[i]; - auto &sc = pmb->swarm_data.Get(); + auto &sc = pmb->meshblock_data.Get()->GetSwarmData(); auto reset_comms = tl.AddTask(none, &SwarmContainer::ResetCommunication, sc.get()); } @@ -590,8 +590,9 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { for (int n = 0; n < blocks.size(); n++) { auto &tl = async_region_tr[n]; auto &pmb = blocks[n]; - auto &sc = pmb->swarm_data.Get(); + auto &sc = pmb->meshblock_data.Get()->GetSwarmData(); auto &mbd0 = pmb->meshblock_data.Get(stage_name[stage]); + auto tracerAdvect = tl.AddTask(none, tracers::AdvectTracers, mbd0.get(), dt); auto tracerPurge = tl.AddTask(tracerAdvect, fixup::PurgeParticles, mbd0.get(), swarmName); @@ -601,6 +602,7 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { auto receive = tl.AddTask(send, &SwarmContainer::Receive, sc.get(), BoundaryCommSubset::all); + auto defrag = tl.AddTask(receive, &SwarmContainer::Defrag, sc.get(), 0.9); } } @@ -932,7 +934,7 @@ TaskListStatus PhoebusDriver::MonteCarloStep() { auto &pmb = blocks[i]; auto &tl = async_region0[i]; auto &mbd0 = pmb->meshblock_data.Get(stage_name[integrator->nstages]); - auto &sc0 = pmb->swarm_data.Get(stage_name[integrator->nstages]); + auto &sc0 = pmb->meshblock_data.Get()->GetSwarmData(); auto sample_particles = tl.AddTask(none, radiation::MonteCarloSourceParticles, pmb.get(), mbd0.get(), sc0.get(), t0, dt); auto transport_particles = @@ -1023,7 +1025,8 @@ TaskListStatus PhoebusDriver::MonteCarloStep() { * Fills Tracers * Computes entropy for output **/ -void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { +void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin, + const parthenon::SimTime &time) { auto tracer_pkg = pmb->packages.Get("tracers"); bool do_tracers = tracer_pkg->Param("active"); diff --git a/src/phoebus_driver.hpp b/src/phoebus_driver.hpp index 921e06905..65a358556 100644 --- a/src/phoebus_driver.hpp +++ b/src/phoebus_driver.hpp @@ -44,7 +44,8 @@ class PhoebusDriver : public EvolutionDriver { }; parthenon::Packages_t ProcessPackages(std::unique_ptr &pin); -void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin); +void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin, + const parthenon::SimTime &time); } // namespace phoebus diff --git a/src/radiation/mocmc.cpp b/src/radiation/mocmc.cpp index 509a24151..d84d40144 100644 --- a/src/radiation/mocmc.cpp +++ b/src/radiation/mocmc.cpp @@ -55,7 +55,7 @@ template void MOCMCInitSamples(T *rc) { auto *pmb = rc->GetParentPointer(); - auto &sc = pmb->swarm_data.Get(); + auto &sc = rc->GetSwarmData(); auto &swarm = sc->Get("mocmc"); StateDescriptor *rad = pmb->packages.Get("radiation").get(); auto rng_pool = rad->Param("rng_pool"); @@ -160,9 +160,9 @@ void MOCMCInitSamples(T *rc) { // }, // result); - const auto &x = swarm->template Get("x").Get(); - const auto &y = swarm->template Get("y").Get(); - const auto &z = swarm->template Get("z").Get(); + const auto &x = swarm->template Get(swarm_position::x::name()).Get(); + const auto &y = swarm->template Get(swarm_position::y::name()).Get(); + const auto &z = swarm->template Get(swarm_position::z::name()).Get(); const auto &ncov = swarm->template Get("ncov").Get(); const auto &Inuinv = swarm->template Get("Inuinv").Get(); @@ -233,7 +233,7 @@ void MOCMCInitSamples(T *rc) { template TaskStatus MOCMCSampleBoundaries(T *rc) { auto *pmb = rc->GetParentPointer(); - auto &sc = pmb->swarm_data.Get(); + auto &sc = rc->GetSwarmData(); auto &swarm = sc->Get("mocmc"); StateDescriptor *rad = pmb->packages.Get("radiation").get(); @@ -251,9 +251,9 @@ TaskStatus MOCMCSampleBoundaries(T *rc) { auto opac_pkg = pmb->packages.Get("opacity"); const auto opac = opac_pkg->template Param("opacities"); - const auto &x = swarm->template Get("x").Get(); - const auto &y = swarm->template Get("y").Get(); - const auto &z = swarm->template Get("z").Get(); + const auto &x = swarm->template Get(swarm_position::x::name()).Get(); + const auto &y = swarm->template Get(swarm_position::y::name()).Get(); + const auto &z = swarm->template Get(swarm_position::z::name()).Get(); const auto &ncov = swarm->template Get("ncov").Get(); const auto &Inuinv = swarm->template Get("Inuinv").Get(); const auto &mu_lo = swarm->template Get("mu_lo").Get(); @@ -349,7 +349,7 @@ TaskStatus MOCMCSampleBoundaries(T *rc) { template TaskStatus MOCMCReconstruction(T *rc) { auto *pmb = rc->GetParentPointer(); - auto &sc = pmb->swarm_data.Get(); + auto &sc = rc->GetSwarmData(); auto &swarm = sc->Get("mocmc"); StateDescriptor *rad = pmb->packages.Get("radiation").get(); @@ -362,9 +362,9 @@ TaskStatus MOCMCReconstruction(T *rc) { PackIndexMap imap; auto v = rc->PackVariables(variables, imap); - const auto &x = swarm->template Get("x").Get(); - const auto &y = swarm->template Get("y").Get(); - const auto &z = swarm->template Get("z").Get(); + const auto &x = swarm->template Get(swarm_position::x::name()).Get(); + const auto &y = swarm->template Get(swarm_position::y::name()).Get(); + const auto &z = swarm->template Get(swarm_position::z::name()).Get(); const auto &ncov = swarm->template Get("ncov").Get(); const auto &Inuinv = swarm->template Get("Inuinv").Get(); const auto &mu_lo = swarm->template Get("mu_lo").Get(); @@ -454,8 +454,8 @@ TaskStatus MOCMCReconstruction(T *rc) { } break; } // if inside - } // m = 0..n - } // n = 0..nsamp + } // m = 0..n + } // n = 0..nsamp }); return TaskStatus::complete; @@ -464,14 +464,14 @@ TaskStatus MOCMCReconstruction(T *rc) { template TaskStatus MOCMCTransport(T *rc, const Real dt) { auto *pmb = rc->GetParentPointer(); - auto &sc = pmb->swarm_data.Get(); + auto &sc = rc->GetSwarmData(); auto &swarm = sc->Get("mocmc"); auto geom = Geometry::GetCoordinateSystem(rc); auto &t = swarm->template Get("t").Get(); - auto &x = swarm->template Get("x").Get(); - auto &y = swarm->template Get("y").Get(); - auto &z = swarm->template Get("z").Get(); + auto &x = swarm->template Get(swarm_position::x::name()).Get(); + auto &y = swarm->template Get(swarm_position::y::name()).Get(); + auto &z = swarm->template Get(swarm_position::z::name()).Get(); auto &ncov = swarm->template Get("ncov").Get(); auto swarm_d = swarm->GetDeviceContext(); @@ -505,7 +505,7 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { // Assume particles are already sorted from MOCMCReconstruction call! auto *pmb = rc->GetParentPointer(); - auto &sc = pmb->swarm_data.Get(); + auto &sc = rc->GetSwarmData(); auto &swarm = sc->Get("mocmc"); StateDescriptor *rad = pmb->packages.Get("radiation").get(); @@ -788,7 +788,7 @@ TaskStatus MOCMCEddington(T *rc) { namespace ir = radmoment_internal; auto *pmb = rc->GetParentPointer(); - auto &sc = pmb->swarm_data.Get(); + auto &sc = rc->GetSwarmData(); auto &swarm = sc->Get("mocmc"); StateDescriptor *rad = pmb->packages.Get("radiation").get(); diff --git a/src/radiation/monte_carlo.cpp b/src/radiation/monte_carlo.cpp index 85c66f8b1..c9bd47b9f 100644 --- a/src/radiation/monte_carlo.cpp +++ b/src/radiation/monte_carlo.cpp @@ -285,9 +285,9 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, const auto new_particles_context = swarm->AddEmptyParticles(Nstot); auto &t = swarm->Get("t").Get(); - auto &x = swarm->Get("x").Get(); - auto &y = swarm->Get("y").Get(); - auto &z = swarm->Get("z").Get(); + auto &x = swarm->Get(swarm_position::x::name()).Get(); + auto &y = swarm->Get(swarm_position::y::name()).Get(); + auto &z = swarm->Get(swarm_position::z::name()).Get(); auto &k0 = swarm->Get("k0").Get(); auto &k1 = swarm->Get("k1").Get(); auto &k2 = swarm->Get("k2").Get(); @@ -438,9 +438,9 @@ TaskStatus MonteCarloTransport(MeshBlock *pmb, MeshBlockData *rc, const Real d4x = dx_i * dx_j * dx_k * dt; auto geom = Geometry::GetCoordinateSystem(rc); auto &t = swarm->Get("t").Get(); - auto &x = swarm->Get("x").Get(); - auto &y = swarm->Get("y").Get(); - auto &z = swarm->Get("z").Get(); + auto &x = swarm->Get(swarm_position::x::name()).Get(); + auto &y = swarm->Get(swarm_position::y::name()).Get(); + auto &z = swarm->Get(swarm_position::z::name()).Get(); auto &k0 = swarm->Get("k0").Get(); auto &k1 = swarm->Get("k1").Get(); auto &k2 = swarm->Get("k2").Get(); @@ -661,33 +661,42 @@ TaskStatus MonteCarloUpdateTuning(Mesh *pmesh, std::vector *resolution, return TaskStatus::complete; } -TaskStatus MonteCarloCountCommunicatedParticles(MeshBlock *pmb, - int *particles_outstanding) { - auto &swarm = pmb->swarm_data.Get()->Get("monte_carlo"); +// TODO(BLB) make sure this works if/when monte carlo solver in driver is updated... +TaskStatus MonteCarloCountCommunicatedParticles(const BlockList_t &blocks, + const double tf_, bool *done) { + int num_unfinished = 0; + for (auto &block : blocks) { + auto sc = block->meshblock_data.Get()->GetSwarmData(); + auto swarm = sc->Get("my_particles"); + int max_active_index = swarm->GetMaxActiveIndex(); - *particles_outstanding += swarm->num_particles_sent_; + auto &t = swarm->Get("t").Get(); - // Reset communication flags - for (int n = 0; n < pmb->pbval->nneighbor; n++) { - auto &nb = pmb->pbval->neighbor[n]; - swarm->vbswarm->bd_var_.flag[nb.bufid] = BoundaryStatus::waiting; -#ifdef MPI_PARALLEL - swarm->vbswarm->bd_var_.req_send[nb.bufid] = MPI_REQUEST_NULL; -#endif // MPI_PARALLEL + auto swarm_d = swarm->GetDeviceContext(); + + const auto &tf = tf_; + + parthenon::par_reduce( + PARTHENON_AUTO_LABEL, 0, max_active_index, + KOKKOS_LAMBDA(const int n, int &num_unfinished) { + if (swarm_d.IsActive(n)) { + if (t(n) < tf) { + num_unfinished++; + } + } + }, + Kokkos::Sum(num_unfinished)); } #ifdef MPI_PARALLEL - pmb->exec_space.fence(); - for (int n = 0; n < pmb->pbval->nneighbor; n++) { - auto &nb = pmb->pbval->neighbor[n]; - if (nb.snb.rank != Globals::my_rank) { - PARTHENON_MPI_CHECK( - MPI_Wait(&(swarm->vbswarm->bd_var_.req_send[nb.bufid]), MPI_STATUS_IGNORE)); - } - swarm->vbswarm->bd_var_.req_send[nb.bufid] = MPI_REQUEST_NULL; - } + MPI_Allreduce(MPI_IN_PLACE, &num_unfinished, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); +#endif // MPI_PARALLEL -#endif + if (num_unfinished > 0) { + *done = false; + } else { + *done = true; + } return TaskStatus::complete; } @@ -697,9 +706,9 @@ TaskStatus InitializeCommunicationMesh(const std::string swarmName, // Boundary transfers on same MPI proc are blocking #ifdef MPI_PARALLEL for (auto &block : blocks) { - auto swarm = block->swarm_data.Get()->Get(swarmName); - for (int n = 0; n < block->pbval->nneighbor; n++) { - auto &nb = block->pbval->neighbor[n]; + auto swarm = block->meshblock_data.Get()->GetSwarmData()->Get(swarmName); + for (int n = 0; n < block->neighbors.size(); n++) { + auto &nb = block->neighbors[n]; #ifdef MPI_PARALLEL swarm->vbswarm->bd_var_.req_send[nb.bufid] = MPI_REQUEST_NULL; #endif @@ -710,7 +719,7 @@ TaskStatus InitializeCommunicationMesh(const std::string swarmName, // Reset boundary statuses for (auto &block : blocks) { auto &pmb = block; - auto sc = pmb->swarm_data.Get(); + auto sc = pmb->meshblock_data.Get()->GetSwarmData(); auto swarm = sc->Get(swarmName); swarm->ResetCommunication(); } diff --git a/src/radiation/radiation.hpp b/src/radiation/radiation.hpp index b1935d810..924e3199f 100644 --- a/src/radiation/radiation.hpp +++ b/src/radiation/radiation.hpp @@ -116,8 +116,8 @@ TaskStatus MonteCarloEstimateParticles(MeshBlock *pmb, MeshBlockData *rc, SwarmContainer *sc, const Real t0, const Real dt, Real *dNtot); -TaskStatus MonteCarloCountCommunicatedParticles(MeshBlock *pmb, - int *particles_outstanding); +TaskStatus MonteCarloCountCommunicatedParticles(const BlockList_t &blocks, + const double tf_, bool *done); TaskStatus InitializeCommunicationMesh(const std::string swarmName, const BlockList_t &blocks); diff --git a/src/tracers/tracers.cpp b/src/tracers/tracers.cpp index 4d314c351..e926bd96b 100644 --- a/src/tracers/tracers.cpp +++ b/src/tracers/tracers.cpp @@ -20,6 +20,7 @@ namespace tracers { using namespace parthenon::package::prelude; +using parthenon::MakePackDescriptor; std::shared_ptr Initialize(ParameterInput *pin) { auto physics = std::make_shared("tracers"); @@ -79,18 +80,16 @@ TaskStatus AdvectTracers(MeshBlockData *rc, const Real dt) { namespace p = fluid_prim; auto *pmb = rc->GetParentPointer(); - auto &sc = pmb->swarm_data.Get(); - auto &swarm = sc->Get("tracers"); + Mesh *pmesh = rc->GetMeshPointer(); const auto ndim = pmb->pmy_mesh->ndim; - auto &x = swarm->Get("x").Get(); - auto &y = swarm->Get("y").Get(); - auto &z = swarm->Get("z").Get(); - - auto swarm_d = swarm->GetDeviceContext(); + // tracer position swarm pack + const auto swarm_name = "tracers"; + static auto desc_tracer = MakeSwarmPackDescriptor(swarm_name); + auto pack_tracers = desc_tracer.GetPack(rc); - const std::vector vars = {p::velocity::name()}; + static const auto vars = {p::velocity::name()}; PackIndexMap imap; auto pack = rc->PackVariables(vars, imap); @@ -98,34 +97,38 @@ TaskStatus AdvectTracers(MeshBlockData *rc, const Real dt) { const int pvel_lo = imap[p::velocity::name()].first; const int pvel_hi = imap[p::velocity::name()].second; - auto geom = Geometry::GetCoordinateSystem(rc); + const auto geom = Geometry::GetCoordinateSystem(rc); // update loop. RK2 - const int max_active_index = swarm->GetMaxActiveIndex(); - pmb->par_for( - "Advect Tracers", 0, max_active_index, KOKKOS_LAMBDA(const int n) { + parthenon::par_for( + "Advect Tracers", 0, pack_tracers.GetMaxFlatIndex(), KOKKOS_LAMBDA(const int idx) { + const auto [b, n] = pack_tracers.GetBlockParticleIndices(idx); + const auto swarm_d = pack_tracers.GetContext(b); if (swarm_d.IsActive(n)) { - int k, j, i; - swarm_d.Xtoijk(x(n), y(n), z(n), i, j, k); - Real rhs1, rhs2, rhs3; + const Real x = pack_tracers(b, swarm_position::x(), n); + const Real y = pack_tracers(b, swarm_position::y(), n); + const Real z = pack_tracers(b, swarm_position::z(), n); + // predictor - tracers_rhs(pack, geom, pvel_lo, pvel_hi, ndim, dt, x(n), y(n), z(n), rhs1, + tracers_rhs(pack, geom, pvel_lo, pvel_hi, ndim, dt, x, y, z, rhs1, rhs2, rhs3); - const Real kx = x(n) + 0.5 * dt * rhs1; - const Real ky = y(n) + 0.5 * dt * rhs2; - const Real kz = z(n) + 0.5 * dt * rhs3; + const Real kx = x + 0.5 * dt * rhs1; + const Real ky = y + 0.5 * dt * rhs2; + const Real kz = z + 0.5 * dt * rhs3; // corrector tracers_rhs(pack, geom, pvel_lo, pvel_hi, ndim, dt, kx, ky, kz, rhs1, rhs2, rhs3); - x(n) += rhs1 * dt; - y(n) += rhs2 * dt; - z(n) += rhs3 * dt; + + // update positions + pack_tracers(b, swarm_position::x(), n) += rhs1 * dt; + pack_tracers(b, swarm_position::y(), n) += rhs2 * dt; + pack_tracers(b, swarm_position::z(), n) += rhs3 * dt; bool on_current_mesh_block = true; - swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); + swarm_d.GetNeighborBlockIndex(n, x, y, z, on_current_mesh_block); } }); @@ -144,43 +147,55 @@ void FillTracers(MeshBlockData *rc) { auto *pmb = rc->GetParentPointer(); auto fluid = pmb->packages.Get("fluid"); - auto &sc = pmb->swarm_data.Get(); + auto &sc = rc->GetSwarmData(); auto &swarm = sc->Get("tracers"); auto eos = pmb->packages.Get("eos")->Param("d.EOS"); const auto mhd = fluid->Param("mhd"); - // pull swarm vars - auto &x = swarm->Get("x").Get(); - auto &y = swarm->Get("y").Get(); - auto &z = swarm->Get("z").Get(); - auto &v1 = swarm->Get("vel_x").Get(); - auto &v2 = swarm->Get("vel_y").Get(); - auto &v3 = swarm->Get("vel_z").Get(); - auto B1 = v1.Get(); - auto B2 = v1.Get(); - auto B3 = v1.Get(); + // tracer swarm pack + const auto swarm_name = "tracers"; + static auto desc_tracer_pos = MakeSwarmPackDescriptor(swarm_name); + auto pack_tracers_pos = desc_tracer_pos.GetPack(rc); + + // tracer vars pack + std::vector swarm_vars = {"vel_x", "vel_y", "vel_z", "rho", "temperature", + "ye", "entropy", "energy", "lorentz", "lapse", "shift_x", "shift_y", + "shift_z", "detgamma", "pressure", "bernoulli"}; + if (mhd) { - B1 = swarm->Get("B_x").Get(); - B2 = swarm->Get("B_y").Get(); - B3 = swarm->Get("B_z").Get(); + swarm_vars.push_back("B_x"); + swarm_vars.push_back("B_y"); + swarm_vars.push_back("B_z"); } - auto &s_rho = swarm->Get("rho").Get(); - auto &s_temperature = swarm->Get("temperature").Get(); - auto &s_ye = swarm->Get("ye").Get(); - auto &s_entropy = swarm->Get("entropy").Get(); - auto &s_energy = swarm->Get("energy").Get(); - auto &s_lorentz = swarm->Get("lorentz").Get(); - auto &s_lapse = swarm->Get("lapse").Get(); - auto &s_shift_x = swarm->Get("shift_x").Get(); - auto &s_shift_y = swarm->Get("shift_y").Get(); - auto &s_shift_z = swarm->Get("shift_z").Get(); - auto &s_detgamma = swarm->Get("detgamma").Get(); - auto &s_pressure = swarm->Get("pressure").Get(); - auto &s_bernoulli = swarm->Get("bernoulli").Get(); - - auto swarm_d = swarm->GetDeviceContext(); + // make pack and get index map + static auto desc_tracer_vars = MakeSwarmPackDescriptor(swarm_name, swarm_vars); + auto pack_tracers_vars = desc_tracer_vars.GetPack(rc); + auto pack_tracers_vars_map = desc_tracer_vars.GetMap(); + + // TODO(BLB): way to clean this up? + parthenon::PackIdx spi_vel_x(pack_tracers_vars_map["vel_x"]); + parthenon::PackIdx spi_vel_y(pack_tracers_vars_map["vel_y"]); + parthenon::PackIdx spi_vel_z(pack_tracers_vars_map["vel_z"]); + parthenon::PackIdx spi_B_x(pack_tracers_vars_map["B_x"]); + parthenon::PackIdx spi_B_y(pack_tracers_vars_map["B_y"]); + parthenon::PackIdx spi_B_z(pack_tracers_vars_map["B_z"]); + parthenon::PackIdx spi_rho(pack_tracers_vars_map["rho"]); + parthenon::PackIdx spi_temperature(pack_tracers_vars_map["temperature"]); + parthenon::PackIdx spi_ye(pack_tracers_vars_map["ye"]); + parthenon::PackIdx spi_entropy(pack_tracers_vars_map["entropy"]); + parthenon::PackIdx spi_energy(pack_tracers_vars_map["energy"]); + parthenon::PackIdx spi_lorentz(pack_tracers_vars_map["lorentz"]); + parthenon::PackIdx spi_lapse(pack_tracers_vars_map["lapse"]); + parthenon::PackIdx spi_shift_x(pack_tracers_vars_map["shift_x"]); + parthenon::PackIdx spi_shift_y(pack_tracers_vars_map["shift_y"]); + parthenon::PackIdx spi_shift_z(pack_tracers_vars_map["shift_z"]); + parthenon::PackIdx spi_detgamma(pack_tracers_vars_map["detgamma"]); + parthenon::PackIdx spi_pressure(pack_tracers_vars_map["pressure"]); + parthenon::PackIdx spi_bernoulli(pack_tracers_vars_map["bernoulli"]); + + // hydro vars pack std::vector vars = {p::density::name(), p::temperature::name(), p::velocity::name(), p::energy::name(), p::pressure::name()}; @@ -205,35 +220,60 @@ void FillTracers(MeshBlockData *rc) { // update loop. const int max_active_index = swarm->GetMaxActiveIndex(); pmb->par_for( - "Fill Tracers", 0, max_active_index, KOKKOS_LAMBDA(const int n) { + "Fill Tracers", 0, pack_tracers_pos.GetMaxFlatIndex(), KOKKOS_LAMBDA(const int idx) { + const auto [b, n] = pack_tracers_pos.GetBlockParticleIndices(idx); + const auto swarm_d = pack_tracers_pos.GetContext(b); + + //TODO(BLB): clean up + const int ivel_x = pack_tracers_vars.GetLowerBound(b, spi_vel_x); + const int ivel_y = pack_tracers_vars.GetLowerBound(b, spi_vel_y); + const int ivel_z = pack_tracers_vars.GetLowerBound(b, spi_vel_z); + const int iB_x = pack_tracers_vars.GetLowerBound(b, spi_B_x); + const int iB_y = pack_tracers_vars.GetLowerBound(b, spi_B_y); + const int iB_z = pack_tracers_vars.GetLowerBound(b, spi_B_z); + const int irho = pack_tracers_vars.GetLowerBound(b, spi_rho); + const int itemperature = pack_tracers_vars.GetLowerBound(b, spi_temperature); + const int iye = pack_tracers_vars.GetLowerBound(b, spi_ye); + const int ientropy = pack_tracers_vars.GetLowerBound(b, spi_entropy); + const int ienergy = pack_tracers_vars.GetLowerBound(b, spi_energy); + const int ilorentz = pack_tracers_vars.GetLowerBound(b, spi_lorentz); + const int ilapse = pack_tracers_vars.GetLowerBound(b, spi_lapse); + const int ishift_x = pack_tracers_vars.GetLowerBound(b, spi_shift_x); + const int ishift_y = pack_tracers_vars.GetLowerBound(b, spi_shift_y); + const int ishift_z = pack_tracers_vars.GetLowerBound(b, spi_shift_z); + const int idetgamma = pack_tracers_vars.GetLowerBound(b, spi_detgamma); + const int ipressure = pack_tracers_vars.GetLowerBound(b, spi_pressure); + const int ibernoulli = pack_tracers_vars.GetLowerBound(b, spi_bernoulli); if (swarm_d.IsActive(n)) { - int k, j, i; - swarm_d.Xtoijk(x(n), y(n), z(n), i, j, k); + + const Real x = pack_tracers_pos(b, swarm_position::x(), n); + const Real y = pack_tracers_pos(b, swarm_position::y(), n); + const Real z = pack_tracers_pos(b, swarm_position::z(), n); // geom quantities Real gcov4[4][4]; - geom.SpacetimeMetric(0.0, x(n), y(n), z(n), gcov4); - Real lapse = geom.Lapse(0.0, x(n), y(n), z(n)); + geom.SpacetimeMetric(0.0, x, y, z, gcov4); + Real lapse = geom.Lapse(0.0, x, y, z); Real shift[3]; - geom.ContravariantShift(0.0, x(n), y(n), z(n), shift); - const Real gdet = geom.DetGamma(0.0, x(n), y(n), z(n)); + geom.ContravariantShift(0.0, x, y, z, shift); + const Real gdet = geom.DetGamma(0.0, x, y, z); // Interpolate - const Real Wvel_X1 = LCInterp::Do(0, x(n), y(n), z(n), pack, pvel_lo); - const Real Wvel_X2 = LCInterp::Do(0, x(n), y(n), z(n), pack, pvel_lo + 1); - const Real Wvel_X3 = LCInterp::Do(0, x(n), y(n), z(n), pack, pvel_hi); + const Real Wvel_X1 = LCInterp::Do(0, x, y, z, pack, pvel_lo); + const Real Wvel_X2 = LCInterp::Do(0, x, y, z, pack, pvel_lo + 1); + const Real Wvel_X3 = LCInterp::Do(0, x, y, z, pack, pvel_hi); Real B_X1 = 0.0; Real B_X2 = 0.0; Real B_X3 = 0.0; if (mhd) { - B_X1 = LCInterp::Do(0, x(n), y(n), z(n), pack, pB_lo); - B_X2 = LCInterp::Do(0, x(n), y(n), z(n), pack, pB_lo + 1); - B_X3 = LCInterp::Do(0, x(n), y(n), z(n), pack, pB_hi); + B_X1 = LCInterp::Do(0, x, y, z, pack, pB_lo); + B_X2 = LCInterp::Do(0, x, y, z, pack, pB_lo + 1); + B_X3 = LCInterp::Do(0, x, y, z, pack, pB_hi); } - const Real rho = LCInterp::Do(0, x(n), y(n), z(n), pack, prho); - const Real temperature = LCInterp::Do(0, x(n), y(n), z(n), pack, ptemp); - const Real energy = LCInterp::Do(0, x(n), y(n), z(n), pack, penergy); - const Real pressure = LCInterp::Do(0, x(n), y(n), z(n), pack, ppres); + const Real rho = LCInterp::Do(0, x, y, z, pack, prho); + const Real temperature = LCInterp::Do(0, x, y, z, pack, ptemp); + const Real energy = LCInterp::Do(0, x, y, z, pack, penergy); + const Real pressure = LCInterp::Do(0, x, y, z, pack, ppres); const Real Wvel[] = {Wvel_X1, Wvel_X2, Wvel_X3}; const Real W = phoebus::GetLorentzFactor(Wvel, gcov4); const Real vel_X1 = Wvel_X1 / W; @@ -242,7 +282,7 @@ void FillTracers(MeshBlockData *rc) { Real ye; Real lambda[2] = {0.0, 0.0}; if (pye > 0) { - ye = LCInterp::Do(0, x(n), y(n), z(n), pack, pye); + ye = LCInterp::Do(0, x, y, z, pack, pye); lambda[1] = ye; } else { ye = 0.0; @@ -255,30 +295,30 @@ void FillTracers(MeshBlockData *rc) { const Real bernoulli = -(W / lapse) * h - 1.0; // store - s_rho(n) = rho; - s_temperature(n) = temperature; - s_ye(n) = ye; - s_energy(n) = energy; - s_entropy(n) = entropy; - v1(n) = vel_X1; - v2(n) = vel_X3; - v3(n) = vel_X2; - s_shift_x(n) = shift[0]; - s_shift_y(n) = shift[1]; - s_shift_z(n) = shift[2]; - s_lapse(n) = lapse; - s_lorentz(n) = W; - s_detgamma(n) = gdet; - s_pressure(n) = pressure; - s_bernoulli(n) = bernoulli; + pack_tracers_vars(b, irho, n) = rho; + pack_tracers_vars(b, itemperature, n) = temperature; + pack_tracers_vars(b, iye, n) = ye; + pack_tracers_vars(b, ienergy, n) = energy; + pack_tracers_vars(b, ientropy, n) = entropy; + pack_tracers_vars(b, ivel_x, n) = vel_X1; + pack_tracers_vars(b, ivel_y, n) = vel_X2; + pack_tracers_vars(b, ivel_z, n) = vel_X3; + pack_tracers_vars(b, ishift_x, n) = shift[0]; + pack_tracers_vars(b, ishift_y, n) = shift[1]; + pack_tracers_vars(b, ishift_z, n) = shift[2]; + pack_tracers_vars(b, ilapse, n) = lapse; + pack_tracers_vars(b, ilorentz, n) = W; + pack_tracers_vars(b, idetgamma, n) = gdet; + pack_tracers_vars(b, ipressure, n) = pressure; + pack_tracers_vars(b, ibernoulli, n) = bernoulli; if (mhd) { - B1(n) = B_X1; - B2(n) = B_X2; - B3(n) = B_X3; + pack_tracers_vars(b, iB_x, n) = B_X1; + pack_tracers_vars(b, iB_y, n) = B_X2; + pack_tracers_vars(b, iB_z, n) = B_X3; } - bool on_current_mesh_block = true; - swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); + //bool on_current_mesh_block = true; + //swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); } }); From 498bc46f15649b059ad513e7d742bf702bd61d80 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Thu, 17 Oct 2024 20:08:26 +0000 Subject: [PATCH 04/48] update parthenon --- external/parthenon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/parthenon b/external/parthenon index 3225612e2..1e77c13e5 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit 3225612e267456e5731f80c594d45e6457274c3f +Subproject commit 1e77c13e563849cf3bdc3c1edac0af6d02cee9be From f28bcddaf38a440d601b737d2e4685377942a954 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Thu, 14 Nov 2024 19:21:48 -0500 Subject: [PATCH 05/48] fix: swarm positon names --- src/fixup/fixup_particles.cpp | 6 +++--- src/pgen/torus.cpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/fixup/fixup_particles.cpp b/src/fixup/fixup_particles.cpp index b5762863f..79f1cfe26 100644 --- a/src/fixup/fixup_particles.cpp +++ b/src/fixup/fixup_particles.cpp @@ -41,9 +41,9 @@ TaskStatus PurgeParticles(MeshBlockData *rc, const std::string swarmName) // auto &swarm = rc->swarm_data.Get()->Get(swarmName); auto &swarm = rc->GetSwarmData()->Get(swarmName); - auto &x = swarm->Get("x").Get(); - auto &y = swarm->Get("y").Get(); - auto &z = swarm->Get("z").Get(); + auto &x = swarm->Get(swarm_position::x::name()).Get(); + auto &y = swarm->Get(swarm_position::y::name()).Get(); + auto &z = swarm->Get(swarm_position::z::name()).Get(); auto swarm_d = swarm->GetDeviceContext(); diff --git a/src/pgen/torus.cpp b/src/pgen/torus.cpp index b2d7211ba..f481a3a7a 100644 --- a/src/pgen/torus.cpp +++ b/src/pgen/torus.cpp @@ -668,9 +668,9 @@ void PostInitializationModifier(ParameterInput *pin, Mesh *pmesh) { auto new_particles_context = swarm->AddEmptyParticles(num_tracers); - auto &x = swarm->Get("x").Get(); - auto &y = swarm->Get("y").Get(); - auto &z = swarm->Get("z").Get(); + auto &x = swarm->Get(swarm_position::x::name()).Get(); + auto &y = swarm->Get(swarm_position::y::name()).Get(); + auto &z = swarm->Get(swarm_position::z::name()).Get(); auto &mass = swarm->Get("mass").Get(); auto &id = swarm->Get("id").Get(); From 8c30be3864fd21db6245c9fea0860f0918799218 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Thu, 14 Nov 2024 19:24:50 -0500 Subject: [PATCH 06/48] BC --- src/fixup/fixup.cpp | 10 ++-- src/fluid/fluid.cpp | 4 ++ src/phoebus_boundaries/phoebus_boundaries.cpp | 50 +++++++------------ 3 files changed, 28 insertions(+), 36 deletions(-) diff --git a/src/fixup/fixup.cpp b/src/fixup/fixup.cpp index 879b79afa..6787e7cdf 100644 --- a/src/fixup/fixup.cpp +++ b/src/fixup/fixup.cpp @@ -730,6 +730,8 @@ TaskStatus FixFluxes(MeshBlockData *rc) { const std::string ox1_bc = fluid->Param("ox1_bc"); const std::string ix2_bc = fluid->Param("ix2_bc"); const std::string ox2_bc = fluid->Param("ox2_bc"); + const std::string ix3_bc = fluid->Param("ix3_bc"); + const std::string ox3_bc = fluid->Param("ox3_bc"); auto rad = pmb->packages.Get("radiation"); int num_species = 0; @@ -992,7 +994,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { if (ndim == 2) return TaskStatus::complete; // x3-direction - if (pmb->boundary_flag[BoundaryFace::inner_x3] == BoundaryFlag::outflow) { + if (ix3_bc == "outflow") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes(std::vector({c::density::name()}), @@ -1003,7 +1005,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { v.flux(X3DIR, crho, k, j, i) = std::min(v.flux(X3DIR, crho, k, j, i), 0.0); }); - } else if (pmb->boundary_flag[BoundaryFace::inner_x3] == BoundaryFlag::reflect) { + } else if (ix3_bc == "reflect") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes( std::vector({c::density::name(), c::energy::name(), cr::E::name()}), @@ -1024,7 +1026,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { } }); } - if (pmb->boundary_flag[BoundaryFace::outer_x3] == BoundaryFlag::outflow) { + if (ox3_bc == "outflow") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes(std::vector({c::density::name()}), @@ -1035,7 +1037,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { v.flux(X3DIR, crho, k, j, i) = std::max(v.flux(X3DIR, crho, k, j, i), 0.0); }); - } else if (pmb->boundary_flag[BoundaryFace::outer_x3] == BoundaryFlag::reflect) { + } else if (ox3_bc == "reflect") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes( std::vector({c::density::name(), c::energy::name(), cr::E::name()}), diff --git a/src/fluid/fluid.cpp b/src/fluid/fluid.cpp index be743716c..9b0fba860 100644 --- a/src/fluid/fluid.cpp +++ b/src/fluid/fluid.cpp @@ -183,6 +183,10 @@ std::shared_ptr Initialize(ParameterInput *pin) { params.Add("ix2_bc", ix2_bc); const std::string ox2_bc = pin->GetString("phoebus", "ox2_bc"); params.Add("ox2_bc", ox2_bc); + const std::string ix3_bc = pin->GetString("phoebus", "ix3_bc"); + params.Add("ix3_bc", ix3_bc); + const std::string ox3_bc = pin->GetString("phoebus", "ox3_bc"); + params.Add("ox3_bc", ox3_bc); int ndim = 1; if (pin->GetInteger("parthenon/mesh", "nx3") > 1) diff --git a/src/phoebus_boundaries/phoebus_boundaries.cpp b/src/phoebus_boundaries/phoebus_boundaries.cpp index 4e9d5ca07..96c643892 100644 --- a/src/phoebus_boundaries/phoebus_boundaries.cpp +++ b/src/phoebus_boundaries/phoebus_boundaries.cpp @@ -470,6 +470,7 @@ void ProcessBoundaryConditions(parthenon::ParthenonManager &pman) { const std::string rad_method = pman.pinput->GetOrAddString("radiation", "method", "None"); + // TODO: logic for gr outflow et/ if (typeid(PHOEBUS_GEOMETRY) == typeid(Geometry::FMKS)) { bool derefine_poles = pman.pinput->GetOrAddBoolean("coordinates", "derefine_poles", true); @@ -484,6 +485,7 @@ void ProcessBoundaryConditions(parthenon::ParthenonManager &pman) { "Polar and Reflecting X2 BCs not supported for \"derefine_poles = true\"!"); } } + pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::inner_x3, "swarm_no_bc", &Boundaries::SwarmNoWorkBC); for (int d = 1; d <= 3; ++d) { // outer = 0 for inner face, outer = 1 for outer face @@ -491,62 +493,46 @@ void ProcessBoundaryConditions(parthenon::ParthenonManager &pman) { auto &face = inner_outer[outer]; const std::string name = face + "x" + std::to_string(d) + "_bc"; const std::string parth_bc = pman.pinput->GetString("parthenon/mesh", name); - PARTHENON_REQUIRE(parth_bc == "user" || parth_bc == "periodic", - "Only \"user\" and \"periodic\" allowed for parthenon/mesh/" + - name); + // TODO: proper check here?? +// PARTHENON_REQUIRE(parth_bc == "user" || parth_bc == "periodic", +// "Only \"user\" and \"periodic\" allowed for parthenon/mesh/" + +// name); const std::string bc = pman.pinput->GetOrAddString("phoebus", name, "outflow"); if (bc == "reflect") { - pman.app_input->boundary_conditions[loc[d - 1][outer]] = reflect[d - 1][outer]; + pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "reflect", reflect[d - 1][outer]); } else if (bc == "polar") { PARTHENON_REQUIRE(d == 2, "Polar boundary conditions only supported in X2!"); - pman.app_input->boundary_conditions[loc[d - 1][outer]] = polar[d - 1][outer]; - } else if (bc == "outflow") { - pman.app_input->boundary_conditions[loc[d - 1][outer]] = outflow[d - 1][outer]; + pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "polar", polar[d - 1][outer]); + } else if (bc == "gr_outflow") { + pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "gr_outflow", outflow[d - 1][outer]); if (d == 1) { if (outer == 0) { if (rad_method == "mocmc") { - pman.app_input - ->swarm_boundary_conditions[parthenon::BoundaryFace::inner_x1] = - Boundaries::SwarmNoWorkBC; + pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::inner_x1, "swarm_no_bc", &Boundaries::SwarmNoWorkBC); } else { - pman.app_input - ->swarm_boundary_conditions[parthenon::BoundaryFace::inner_x1] = - // Boundaries::SetSwarmIX1Outflow; - parthenon::BoundaryFunction::SwarmOutflowInnerX1; + pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::inner_x1, "swarm_outflow", &parthenon::BoundaryFunction::SwarmOutflowInnerX1); } } else if (outer == 1) { if (rad_method == "mocmc") { - pman.app_input - ->swarm_boundary_conditions[parthenon::BoundaryFace::outer_x1] = - Boundaries::SwarmNoWorkBC; + pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::outer_x1, "swarm_no_bc", &Boundaries::SwarmNoWorkBC); } else { - pman.app_input - ->swarm_boundary_conditions[parthenon::BoundaryFace::outer_x1] = - parthenon::BoundaryFunction::SwarmOutflowOuterX1; + pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::outer_x1, "swarm_outflow", &parthenon::BoundaryFunction::SwarmOutflowOuterX1); } } } if (d == 2) { if (outer == 0) { if (rad_method == "mocmc") { - pman.app_input - ->swarm_boundary_conditions[parthenon::BoundaryFace::inner_x2] = - Boundaries::SwarmNoWorkBC; + pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::inner_x2, "swarm_no_bc", &Boundaries::SwarmNoWorkBC); } else { - pman.app_input - ->swarm_boundary_conditions[parthenon::BoundaryFace::inner_x2] = - parthenon::BoundaryFunction::SwarmOutflowInnerX2; + pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::inner_x2, "swarm_outflow", &parthenon::BoundaryFunction::SwarmOutflowInnerX1); } } else if (outer == 1) { if (rad_method == "mocmc") { - pman.app_input - ->swarm_boundary_conditions[parthenon::BoundaryFace::outer_x2] = - Boundaries::SwarmNoWorkBC; + pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::outer_x2, "swarm_no_bc", &Boundaries::SwarmNoWorkBC); } else { - pman.app_input - ->swarm_boundary_conditions[parthenon::BoundaryFace::outer_x2] = - parthenon::BoundaryFunction::SwarmOutflowOuterX2; + pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::outer_x2, "swarm_outflow", &parthenon::BoundaryFunction::SwarmOutflowOuterX2); } } } From f687fa28e522372e85f6d829e76603e1829962d5 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Thu, 14 Nov 2024 19:25:08 -0500 Subject: [PATCH 07/48] bump parthenon --- external/parthenon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/parthenon b/external/parthenon index 1e77c13e5..cad5ccdc8 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit 1e77c13e563849cf3bdc3c1edac0af6d02cee9be +Subproject commit cad5ccdc89c360d8cdcb851a2024572c4047b8a0 From ffc7a5b0f66c69709ff3567c5bb2f877e3a9335d Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Mon, 6 Jan 2025 12:55:33 -0500 Subject: [PATCH 08/48] dump submodules --- external/parthenon | 2 +- external/singularity-eos | 2 +- external/singularity-opac | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/external/parthenon b/external/parthenon index cad5ccdc8..d65f3e6b7 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit cad5ccdc89c360d8cdcb851a2024572c4047b8a0 +Subproject commit d65f3e6b7f2f22150e18726be99c3422abcfcb43 diff --git a/external/singularity-eos b/external/singularity-eos index 1ff875514..b6bf01a01 160000 --- a/external/singularity-eos +++ b/external/singularity-eos @@ -1 +1 @@ -Subproject commit 1ff8755149a6ea6e1af44983f9af2b149391a718 +Subproject commit b6bf01a012a9ece27f15e49d33d060d2c678be1a diff --git a/external/singularity-opac b/external/singularity-opac index 8c1bd1a8e..0e1d38840 160000 --- a/external/singularity-opac +++ b/external/singularity-opac @@ -1 +1 @@ -Subproject commit 8c1bd1a8ee714292fd4016e12835549337b6a0c1 +Subproject commit 0e1d38840ae18b1ebec038c888545edbddc63c13 From 4bc1327dba5a2ad7da627a856f33b03843f7beb2 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Mon, 6 Jan 2025 12:55:52 -0500 Subject: [PATCH 09/48] revert something --- src/tracers/tracers.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tracers/tracers.cpp b/src/tracers/tracers.cpp index e926bd96b..39c99cb8c 100644 --- a/src/tracers/tracers.cpp +++ b/src/tracers/tracers.cpp @@ -317,8 +317,8 @@ void FillTracers(MeshBlockData *rc) { pack_tracers_vars(b, iB_z, n) = B_X3; } - //bool on_current_mesh_block = true; - //swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); + bool on_current_mesh_block = true; + swarm_d.GetNeighborBlockIndex(n, x, y, z, on_current_mesh_block); } }); From b3740607aeced8ea85756f6999d1940d08851f96 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Mon, 6 Jan 2025 12:57:04 -0500 Subject: [PATCH 10/48] updates to mean opacity API --- src/microphysics/opac_phoebus/opac_phoebus.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/microphysics/opac_phoebus/opac_phoebus.cpp b/src/microphysics/opac_phoebus/opac_phoebus.cpp index 62d807fed..928ccdb7e 100644 --- a/src/microphysics/opac_phoebus/opac_phoebus.cpp +++ b/src/microphysics/opac_phoebus/opac_phoebus.cpp @@ -159,9 +159,9 @@ std::shared_ptr Initialize(ParameterInput *pin) { const Real lNuMin = std::log10(pin->GetOrAddReal("mean_opacity", "numin", 0.1)); const Real lNuMax = std::log10(pin->GetOrAddReal("mean_opacity", "numax", 10.)); const int NNu = pin->GetOrAddInteger("mean_opacity", "nnu", 100); - MeanOpacity mean_opac_host = - MeanOpacityScaleFree(opacity_host, lRhoMin, lRhoMax, NRho, lTMin, lTMax, NT, - YeMin, YeMax, NYe, lNuMin, lNuMax, NNu); + auto mean_opac_host = + MeanOpacityBase(opacity_host, lRhoMin, lRhoMax, NRho, lTMin, lTMax, NT, YeMin, + YeMax, NYe, lNuMin, lNuMax, NNu); auto mean_opac_device = mean_opac_host.GetOnDevice(); params.Add("h.mean_opacity", mean_opac_host); params.Add("d.mean_opacity", mean_opac_device); @@ -176,12 +176,12 @@ std::shared_ptr Initialize(ParameterInput *pin) { const Real lNuMin = std::log10(pin->GetOrAddReal("mean_opacity", "numin", 1.e10)); const Real lNuMax = std::log10(pin->GetOrAddReal("mean_opacity", "numax", 1.e24)); const int NNu = pin->GetOrAddInteger("mean_opacity", "nnu", 100); - MeanOpacityCGS cgs_mean_opacity = - MeanOpacityCGS(opacity_host, lRhoMin, lRhoMax, NRho, lTMin, lTMax, NT, YeMin, - YeMax, NYe, lNuMin, lNuMax, NNu); - MeanOpacity mean_opac_host = - MeanNonCGSUnits(std::forward(cgs_mean_opacity), - time_unit, mass_unit, length_unit, temp_unit); + auto cgs_mean_opacity = + MeanOpacityBase(opacity_host, lRhoMin, lRhoMax, NRho, lTMin, lTMax, NT, YeMin, + YeMax, NYe, lNuMin, lNuMax, NNu); + auto mean_opac_host = MeanNonCGSUnits( + std::forward(cgs_mean_opacity), time_unit, mass_unit, + length_unit, temp_unit); auto mean_opac_device = mean_opac_host.GetOnDevice(); params.Add("h.mean_opacity", mean_opac_host); params.Add("d.mean_opacity", mean_opac_device); From 0f679cb06becfcc089f16abd296646824c27c79a Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Mon, 6 Jan 2025 12:59:30 -0500 Subject: [PATCH 11/48] fmt --- src/fluid/fluid.cpp | 17 ++++--- src/fluid/riemann.hpp | 12 ++--- src/geometry/geometry.cpp | 3 +- src/pgen/advection.cpp | 36 ++++++------- src/phoebus_boundaries/phoebus_boundaries.cpp | 51 +++++++++++++------ src/tracers/tracers.cpp | 23 +++++---- 6 files changed, 84 insertions(+), 58 deletions(-) diff --git a/src/fluid/fluid.cpp b/src/fluid/fluid.cpp index 9b0fba860..d7874c2cb 100644 --- a/src/fluid/fluid.cpp +++ b/src/fluid/fluid.cpp @@ -786,12 +786,15 @@ TaskStatus FluxCT(MeshBlockData *rc) { IndexRange kb = rc->GetBoundsK(IndexDomain::interior); auto &resolved_pkgs = pmesh->resolved_packages; - //static auto desc = MakePackDescriptor(resolved_pkgs.get(), {parthenon::Metadata::WithFluxes}, {parthenon::PDOpt::WithFluxes}); - static auto desc = MakePackDescriptor(resolved_pkgs.get(), {}, {parthenon::PDOpt::WithFluxes}); - //const auto v = rc->PackVariablesAndFluxes(std::vector{fluid_cons::bfield::name()}); - //auto f1 = rc->Get("bnd_flux::" + fluid_cons::bfield::name()).data.Get(0,0,0); - //auto f2 = rc->Get("bnd_flux::" + fluid_cons::bfield::name()).data.Get(1,0,0); - //auto f3 = rc->Get("bnd_flux::" + fluid_cons::bfield::name()).data.Get(2,0,0); + // static auto desc = MakePackDescriptor(resolved_pkgs.get(), + // {parthenon::Metadata::WithFluxes}, {parthenon::PDOpt::WithFluxes}); + static auto desc = MakePackDescriptor( + resolved_pkgs.get(), {}, {parthenon::PDOpt::WithFluxes}); + // const auto v = + // rc->PackVariablesAndFluxes(std::vector{fluid_cons::bfield::name()}); + // auto f1 = rc->Get("bnd_flux::" + fluid_cons::bfield::name()).data.Get(0,0,0); + // auto f2 = rc->Get("bnd_flux::" + fluid_cons::bfield::name()).data.Get(1,0,0); + // auto f3 = rc->Get("bnd_flux::" + fluid_cons::bfield::name()).data.Get(2,0,0); auto v = desc.GetPack(rc); // auto f1 = v.flux(0, X1DIR, fluid_cons::bfield::name(), k, j, i); // auto f2 = v.flux(0, X2DIR, fluid_cons::bfield::name(), k, j, i); @@ -801,7 +804,7 @@ TaskStatus FluxCT(MeshBlockData *rc) { // auto f3 = rc->Get(fluid_cons::bfield::name()).flux[X3DIR]; auto emf = rc->Get(internal_variables::emf::name()).data; - //print + // print if (ndim == 2) { parthenon::par_for( DEFAULT_LOOP_PATTERN, "FluxCT::EMF::2D", DevExecSpace(), kb.s, kb.e, jb.s, diff --git a/src/fluid/riemann.hpp b/src/fluid/riemann.hpp index bb0cdefe9..87267df88 100644 --- a/src/fluid/riemann.hpp +++ b/src/fluid/riemann.hpp @@ -70,15 +70,15 @@ class FluxState { } static void FluxVars(const std::string &var) { flux_vars.push_back(var); } - static std::vector ReconVars() { - for ( auto & v : recon_vars ) { + static std::vector ReconVars() { + for (auto &v : recon_vars) { } - return recon_vars; + return recon_vars; } - static std::vector FluxVars() { - for ( auto & v : flux_vars ) { + static std::vector FluxVars() { + for (auto &v : flux_vars) { } - return flux_vars; + return flux_vars; } KOKKOS_FORCEINLINE_FUNCTION diff --git a/src/geometry/geometry.cpp b/src/geometry/geometry.cpp index c38afa93b..9d79ef428 100644 --- a/src/geometry/geometry.cpp +++ b/src/geometry/geometry.cpp @@ -56,7 +56,8 @@ std::shared_ptr Initialize(ParameterInput *pin) { // TODO(JMM): Make this actual node-centered data when available // TODO: Warning: g.n.coord output broken in Parthenon currently std::vector node_shape = {dims.nx1 + 1, dims.nx2 + 1, dims.nx3 + 1, 4}; - Metadata gcoord_node = Metadata({Metadata::Derived, Metadata::OneCopy, Metadata::None}, node_shape); + Metadata gcoord_node = + Metadata({Metadata::Derived, Metadata::OneCopy, Metadata::None}, node_shape); geometry->AddField(geometric_variables::cell_coords::name(), gcoord_cell); geometry->AddField(geometric_variables::node_coords::name(), gcoord_node); diff --git a/src/pgen/advection.cpp b/src/pgen/advection.cpp index 12e6df710..ca6929058 100644 --- a/src/pgen/advection.cpp +++ b/src/pgen/advection.cpp @@ -118,7 +118,6 @@ void PostInitializationModifier(ParameterInput *pin, Mesh *pmesh) { : ndim == 2 ? M_PI * rin * rin : rin; - auto geom = Geometry::GetCoordinateSystem(rc.get()); auto coords = pmb->coords; if (do_tracers) { @@ -169,25 +168,22 @@ void PostInitializationModifier(ParameterInput *pin, Mesh *pmesh) { pmb->par_for( "ProblemGenerator::Advection::DistributeTracers", 0, max_active_index, KOKKOS_LAMBDA(const int new_n) { -// if (swarm_d.IsActive(n)) { - const int n = new_particles_context.GetNewParticleIndex(new_n); - auto rng_gen = rng_pool.get_state(); - - // sample in ye ball - Real r2 = 1.0 + rin * rin; // init > rin^2 - while (r2 > rin * rin) { - x(n) = -rin + - rng_gen.drand() * 2.0 * rin; // just sampling x \in [-rin, +rin] - y(n) = -rin + rng_gen.drand() * 2.0 * rin; - z(n) = -rin + rng_gen.drand() * 2.0 * rin; - r2 = x(n) * x(n) + y(n) * y(n) + z(n) * z(n); - } - id(n) = num_tracers_total * gid + n; - - bool on_current_mesh_block = true; - swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); - rng_pool.free_state(rng_gen); - // } + const int n = new_particles_context.GetNewParticleIndex(new_n); + auto rng_gen = rng_pool.get_state(); + + // sample in ye ball + Real r2 = 1.0 + rin * rin; // init > rin^2 + while (r2 > rin * rin) { + x(n) = -rin + rng_gen.drand() * 2.0 * rin; // x \in [-rin, +rin] + y(n) = -rin + rng_gen.drand() * 2.0 * rin; + z(n) = -rin + rng_gen.drand() * 2.0 * rin; + r2 = x(n) * x(n) + y(n) * y(n) + z(n) * z(n); + } + id(n) = num_tracers_total * gid + n; + + bool on_current_mesh_block = true; + swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); + rng_pool.free_state(rng_gen); }); } } diff --git a/src/phoebus_boundaries/phoebus_boundaries.cpp b/src/phoebus_boundaries/phoebus_boundaries.cpp index 96c643892..82ec620ce 100644 --- a/src/phoebus_boundaries/phoebus_boundaries.cpp +++ b/src/phoebus_boundaries/phoebus_boundaries.cpp @@ -485,7 +485,8 @@ void ProcessBoundaryConditions(parthenon::ParthenonManager &pman) { "Polar and Reflecting X2 BCs not supported for \"derefine_poles = true\"!"); } } - pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::inner_x3, "swarm_no_bc", &Boundaries::SwarmNoWorkBC); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::inner_x3, "swarm_no_bc", &Boundaries::SwarmNoWorkBC); for (int d = 1; d <= 3; ++d) { // outer = 0 for inner face, outer = 1 for outer face @@ -494,45 +495,65 @@ void ProcessBoundaryConditions(parthenon::ParthenonManager &pman) { const std::string name = face + "x" + std::to_string(d) + "_bc"; const std::string parth_bc = pman.pinput->GetString("parthenon/mesh", name); // TODO: proper check here?? -// PARTHENON_REQUIRE(parth_bc == "user" || parth_bc == "periodic", -// "Only \"user\" and \"periodic\" allowed for parthenon/mesh/" + -// name); + // PARTHENON_REQUIRE(parth_bc == "user" || parth_bc == "periodic", + // "Only \"user\" and \"periodic\" allowed for + // parthenon/mesh/" + + // name); const std::string bc = pman.pinput->GetOrAddString("phoebus", name, "outflow"); if (bc == "reflect") { - pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "reflect", reflect[d - 1][outer]); + pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "reflect", + reflect[d - 1][outer]); } else if (bc == "polar") { PARTHENON_REQUIRE(d == 2, "Polar boundary conditions only supported in X2!"); - pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "polar", polar[d - 1][outer]); + pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "polar", + polar[d - 1][outer]); } else if (bc == "gr_outflow") { - pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "gr_outflow", outflow[d - 1][outer]); + pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "gr_outflow", + outflow[d - 1][outer]); if (d == 1) { if (outer == 0) { if (rad_method == "mocmc") { - pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::inner_x1, "swarm_no_bc", &Boundaries::SwarmNoWorkBC); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::inner_x1, "swarm_no_bc", + &Boundaries::SwarmNoWorkBC); } else { - pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::inner_x1, "swarm_outflow", &parthenon::BoundaryFunction::SwarmOutflowInnerX1); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::inner_x1, "swarm_outflow", + &parthenon::BoundaryFunction::SwarmOutflowInnerX1); } } else if (outer == 1) { if (rad_method == "mocmc") { - pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::outer_x1, "swarm_no_bc", &Boundaries::SwarmNoWorkBC); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::outer_x1, "swarm_no_bc", + &Boundaries::SwarmNoWorkBC); } else { - pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::outer_x1, "swarm_outflow", &parthenon::BoundaryFunction::SwarmOutflowOuterX1); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::outer_x1, "swarm_outflow", + &parthenon::BoundaryFunction::SwarmOutflowOuterX1); } } } if (d == 2) { if (outer == 0) { if (rad_method == "mocmc") { - pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::inner_x2, "swarm_no_bc", &Boundaries::SwarmNoWorkBC); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::inner_x2, "swarm_no_bc", + &Boundaries::SwarmNoWorkBC); } else { - pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::inner_x2, "swarm_outflow", &parthenon::BoundaryFunction::SwarmOutflowInnerX1); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::inner_x2, "swarm_outflow", + &parthenon::BoundaryFunction::SwarmOutflowInnerX1); } } else if (outer == 1) { if (rad_method == "mocmc") { - pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::outer_x2, "swarm_no_bc", &Boundaries::SwarmNoWorkBC); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::outer_x2, "swarm_no_bc", + &Boundaries::SwarmNoWorkBC); } else { - pman.app_input->RegisterSwarmBoundaryCondition(parthenon::BoundaryFace::outer_x2, "swarm_outflow", &parthenon::BoundaryFunction::SwarmOutflowOuterX2); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::outer_x2, "swarm_outflow", + &parthenon::BoundaryFunction::SwarmOutflowOuterX2); } } } diff --git a/src/tracers/tracers.cpp b/src/tracers/tracers.cpp index 39c99cb8c..a3b449f1d 100644 --- a/src/tracers/tracers.cpp +++ b/src/tracers/tracers.cpp @@ -86,7 +86,9 @@ TaskStatus AdvectTracers(MeshBlockData *rc, const Real dt) { // tracer position swarm pack const auto swarm_name = "tracers"; - static auto desc_tracer = MakeSwarmPackDescriptor(swarm_name); + static auto desc_tracer = + MakeSwarmPackDescriptor( + swarm_name); auto pack_tracers = desc_tracer.GetPack(rc); static const auto vars = {p::velocity::name()}; @@ -112,8 +114,7 @@ TaskStatus AdvectTracers(MeshBlockData *rc, const Real dt) { const Real z = pack_tracers(b, swarm_position::z(), n); // predictor - tracers_rhs(pack, geom, pvel_lo, pvel_hi, ndim, dt, x, y, z, rhs1, - rhs2, rhs3); + tracers_rhs(pack, geom, pvel_lo, pvel_hi, ndim, dt, x, y, z, rhs1, rhs2, rhs3); const Real kx = x + 0.5 * dt * rhs1; const Real ky = y + 0.5 * dt * rhs2; const Real kz = z + 0.5 * dt * rhs3; @@ -155,13 +156,16 @@ void FillTracers(MeshBlockData *rc) { // tracer swarm pack const auto swarm_name = "tracers"; - static auto desc_tracer_pos = MakeSwarmPackDescriptor(swarm_name); + static auto desc_tracer_pos = + MakeSwarmPackDescriptor( + swarm_name); auto pack_tracers_pos = desc_tracer_pos.GetPack(rc); // tracer vars pack - std::vector swarm_vars = {"vel_x", "vel_y", "vel_z", "rho", "temperature", - "ye", "entropy", "energy", "lorentz", "lapse", "shift_x", "shift_y", - "shift_z", "detgamma", "pressure", "bernoulli"}; + std::vector swarm_vars = { + "vel_x", "vel_y", "vel_z", "rho", "temperature", "ye", + "entropy", "energy", "lorentz", "lapse", "shift_x", "shift_y", + "shift_z", "detgamma", "pressure", "bernoulli"}; if (mhd) { swarm_vars.push_back("B_x"); @@ -220,11 +224,12 @@ void FillTracers(MeshBlockData *rc) { // update loop. const int max_active_index = swarm->GetMaxActiveIndex(); pmb->par_for( - "Fill Tracers", 0, pack_tracers_pos.GetMaxFlatIndex(), KOKKOS_LAMBDA(const int idx) { + "Fill Tracers", 0, pack_tracers_pos.GetMaxFlatIndex(), + KOKKOS_LAMBDA(const int idx) { const auto [b, n] = pack_tracers_pos.GetBlockParticleIndices(idx); const auto swarm_d = pack_tracers_pos.GetContext(b); - //TODO(BLB): clean up + // TODO(BLB): clean up const int ivel_x = pack_tracers_vars.GetLowerBound(b, spi_vel_x); const int ivel_y = pack_tracers_vars.GetLowerBound(b, spi_vel_y); const int ivel_z = pack_tracers_vars.GetLowerBound(b, spi_vel_z); From 55e5ef31ea8f7407c267bbbb9f2868413a534136 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Mon, 6 Jan 2025 13:11:14 -0500 Subject: [PATCH 12/48] fast logs disabled by default --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7aa9ad11c..4700cfd1f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -166,6 +166,7 @@ set(SINGULARITY_USE_FORTRAN OFF CACHE BOOL "" FORCE) set(SINGULARITY_FORCE_SUBMODULE_MODE ON CACHE BOOL "" FORCE) set(SINGULARITY_USE_KOKKOSKERNELS OFF CACHE BOOL "" FORCE) set(SINGULARITY_BUILD_CLOSURE OFF CACHE BOOL "" FORCE) +set(SINGULARITY_USE_TRUE_LOG_GRIDDING ON CACHE BOOL "" FORCE) # Kill cmake's package registry because it can interfere set(CMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY ON CACHE BOOL "" FORCE) From 94955cb4a7c202815a7af4d855edb3b81d504e2c Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Mon, 6 Jan 2025 13:29:32 -0500 Subject: [PATCH 13/48] ubuntu clang format broke. attempting to switch to pip provided.. more control.. pray --- .github/workflows/formatting.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/formatting.yml b/.github/workflows/formatting.yml index a1c441815..33e942171 100644 --- a/.github/workflows/formatting.yml +++ b/.github/workflows/formatting.yml @@ -17,7 +17,11 @@ jobs: - name: Set system to non-interactive mode run: export DEBIAN_FRONTEND=noninteractive - name: install dependencies - run: | - sudo apt-get install -y --force-yes -qq git clang-format-12 + - uses: BSFishy/pip-action@v1 + with: + packages: | + clang-format==12.0.1 + - run: | + sudo apt-get install -y --force-yes -qq git - name: check formatting - run: find . -regex '.*\.\(cpp\|hpp\)' | xargs clang-format-12 -style=file -i && git diff --exit-code --ignore-submodules + run: find . -regex '.*\.\(cpp\|hpp\)' | xargs clang-format -style=file -i && git diff --exit-code --ignore-submodules From ac9945b9e5ebc4dc13bec4ee28e71ac7bb73991f Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Mon, 6 Jan 2025 13:32:02 -0500 Subject: [PATCH 14/48] broke yaml --- .github/workflows/formatting.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/formatting.yml b/.github/workflows/formatting.yml index 33e942171..fbc1b3962 100644 --- a/.github/workflows/formatting.yml +++ b/.github/workflows/formatting.yml @@ -17,11 +17,11 @@ jobs: - name: Set system to non-interactive mode run: export DEBIAN_FRONTEND=noninteractive - name: install dependencies - - uses: BSFishy/pip-action@v1 - with: - packages: | - clang-format==12.0.1 - - run: | + uses: BSFishy/pip-action@v1 + with: + packages: | + lang-format==12.0.1 + run: | sudo apt-get install -y --force-yes -qq git - name: check formatting run: find . -regex '.*\.\(cpp\|hpp\)' | xargs clang-format -style=file -i && git diff --exit-code --ignore-submodules From 6800e14a4de3d991d4df8ad1c31ce98e9a6dae25 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Mon, 6 Jan 2025 13:34:47 -0500 Subject: [PATCH 15/48] hate actions --- .github/workflows/formatting.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/formatting.yml b/.github/workflows/formatting.yml index fbc1b3962..6ef9375c9 100644 --- a/.github/workflows/formatting.yml +++ b/.github/workflows/formatting.yml @@ -17,11 +17,12 @@ jobs: - name: Set system to non-interactive mode run: export DEBIAN_FRONTEND=noninteractive - name: install dependencies + run: | + sudo apt-get install -y --force-yes -qq git + - name: Install pip, clang-format uses: BSFishy/pip-action@v1 with: packages: | - lang-format==12.0.1 - run: | - sudo apt-get install -y --force-yes -qq git + clang-format==12.0.1 - name: check formatting run: find . -regex '.*\.\(cpp\|hpp\)' | xargs clang-format -style=file -i && git diff --exit-code --ignore-submodules From 27d1be3c293f15217818dc652d17135d719220e9 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Mon, 6 Jan 2025 13:36:48 -0500 Subject: [PATCH 16/48] clang format is the most version unstable software --- src/radiation/mocmc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/radiation/mocmc.cpp b/src/radiation/mocmc.cpp index d84d40144..d4a8c349d 100644 --- a/src/radiation/mocmc.cpp +++ b/src/radiation/mocmc.cpp @@ -454,8 +454,8 @@ TaskStatus MOCMCReconstruction(T *rc) { } break; } // if inside - } // m = 0..n - } // n = 0..nsamp + } // m = 0..n + } // n = 0..nsamp }); return TaskStatus::complete; From 6423e9beb7e64a2db2a5122eee553285005b2c59 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Mon, 6 Jan 2025 15:43:25 -0500 Subject: [PATCH 17/48] mean opac param type --- src/microphysics/opac_phoebus/opac_phoebus.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/microphysics/opac_phoebus/opac_phoebus.cpp b/src/microphysics/opac_phoebus/opac_phoebus.cpp index 928ccdb7e..58cf72676 100644 --- a/src/microphysics/opac_phoebus/opac_phoebus.cpp +++ b/src/microphysics/opac_phoebus/opac_phoebus.cpp @@ -285,7 +285,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { } auto opacity_device = params.Get("d.opacity"); - auto &mean_opac_device = params.Get("d.mean_opacity"); + auto &mean_opac_device = params.Get("d.mean_opacity"); auto &s_opacity_device = params.Get("d.s_opacity"); auto &mean_s_opac_device = params.Get("d.mean_s_opacity"); Opacities opacities(opacity_device, mean_opac_device, s_opacity_device, From 8c338cd1100a4848db5ae2d8d20a9c0d313c3b42 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Tue, 7 Jan 2025 14:40:41 -0500 Subject: [PATCH 18/48] cleanup --- src/fluid/fluid.cpp | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/fluid/fluid.cpp b/src/fluid/fluid.cpp index d7874c2cb..a23fa3edd 100644 --- a/src/fluid/fluid.cpp +++ b/src/fluid/fluid.cpp @@ -786,25 +786,11 @@ TaskStatus FluxCT(MeshBlockData *rc) { IndexRange kb = rc->GetBoundsK(IndexDomain::interior); auto &resolved_pkgs = pmesh->resolved_packages; - // static auto desc = MakePackDescriptor(resolved_pkgs.get(), - // {parthenon::Metadata::WithFluxes}, {parthenon::PDOpt::WithFluxes}); static auto desc = MakePackDescriptor( resolved_pkgs.get(), {}, {parthenon::PDOpt::WithFluxes}); - // const auto v = - // rc->PackVariablesAndFluxes(std::vector{fluid_cons::bfield::name()}); - // auto f1 = rc->Get("bnd_flux::" + fluid_cons::bfield::name()).data.Get(0,0,0); - // auto f2 = rc->Get("bnd_flux::" + fluid_cons::bfield::name()).data.Get(1,0,0); - // auto f3 = rc->Get("bnd_flux::" + fluid_cons::bfield::name()).data.Get(2,0,0); auto v = desc.GetPack(rc); - // auto f1 = v.flux(0, X1DIR, fluid_cons::bfield::name(), k, j, i); - // auto f2 = v.flux(0, X2DIR, fluid_cons::bfield::name(), k, j, i); - // auto f3 = v.flux(0, X3DIR, fluid_cons::bfield::name(), k, j, i); - // auto f1 = rc->Get(fluid_cons::bfield::name()).flux[X1DIR]; - // auto f2 = rc->Get(fluid_cons::bfield::name()).flux[X2DIR]; - // auto f3 = rc->Get(fluid_cons::bfield::name()).flux[X3DIR]; auto emf = rc->Get(internal_variables::emf::name()).data; - // print if (ndim == 2) { parthenon::par_for( DEFAULT_LOOP_PATTERN, "FluxCT::EMF::2D", DevExecSpace(), kb.s, kb.e, jb.s, From ec1499e57e1de726d93b3360a5502eecbdcb3419 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Tue, 7 Jan 2025 14:51:45 -0500 Subject: [PATCH 19/48] pack fix? --- src/radiation/moments.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/radiation/moments.cpp b/src/radiation/moments.cpp index aa1633cd5..368782489 100644 --- a/src/radiation/moments.cpp +++ b/src/radiation/moments.cpp @@ -513,6 +513,7 @@ TaskStatus CalculateFluxesImpl(T *rc) { namespace pr = radmoment_prim; namespace ir = radmoment_internal; + // TODO(@astrtobarker) move to sparse packs PackIndexMap imap_ql, imap_qr, imap; std::vector vars{ir::ql::name(), ir::qr::name(), ir::ql_v::name(), ir::qr_v::name(), ir::dJ::name(), ir::kappaH::name()}; @@ -520,6 +521,9 @@ TaskStatus CalculateFluxesImpl(T *rc) { auto v = rc->PackVariablesAndFluxes(vars, flxs, imap); + PackIndexMap imap_fl; + auto v_fl = rc->PackVariablesAndFluxes(flxs, flxs, imap_fl); + auto idx_qlv = imap.GetFlatIdx(ir::ql_v::name()); auto idx_qrv = imap.GetFlatIdx(ir::qr_v::name()); auto idx_ql = imap.GetFlatIdx(ir::ql::name()); @@ -527,8 +531,8 @@ TaskStatus CalculateFluxesImpl(T *rc) { auto idx_dJ = imap.GetFlatIdx(ir::dJ::name()); auto idx_kappaH = imap.GetFlatIdx(ir::kappaH::name()); - auto idx_Ff = imap.GetFlatIdx(cr::F::name()); - auto idx_Ef = imap.GetFlatIdx(cr::E::name()); + auto idx_Ff = imap_fl.GetFlatIdx(cr::F::name()); + auto idx_Ef = imap_fl.GetFlatIdx(cr::E::name()); auto num_species = rad_pkg->Param("num_species"); @@ -737,14 +741,14 @@ TaskStatus CalculateFluxesImpl(T *rc) { 0.5 * sdetgam * (conFl(idir) + conFr(idir) + sigspeed * (El - Er)); SPACELOOP(ii) { - v.flux(idir_in, idx_Ff(ispec, ii), k, j, i) = + v_fl.flux(idir_in, idx_Ff(ispec, ii), k, j, i) = 0.5 * sdetgam * (Pl(idir, ii) + Pr(idir, ii) + sigspeed * (covFl(ii) - covFr(ii))); } if (sdetgam < robust::SMALL()) { - v.flux(idir_in, idx_Ef(ispec), k, j, i) = 0.0; - SPACELOOP(ii) v.flux(idir_in, idx_Ff(ispec, ii), k, j, i) = 0.0; + v_fl.flux(idir_in, idx_Ef(ispec), k, j, i) = 0.0; + SPACELOOP(ii) v_fl.flux(idir_in, idx_Ff(ispec, ii), k, j, i) = 0.0; } } }); From 64aae211449cd790bb6d01cd7291e24e0202606f Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Tue, 7 Jan 2025 18:21:18 -0500 Subject: [PATCH 20/48] fighting types --- src/microphysics/opac_phoebus/opac_phoebus.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/microphysics/opac_phoebus/opac_phoebus.cpp b/src/microphysics/opac_phoebus/opac_phoebus.cpp index 58cf72676..f1610c970 100644 --- a/src/microphysics/opac_phoebus/opac_phoebus.cpp +++ b/src/microphysics/opac_phoebus/opac_phoebus.cpp @@ -145,7 +145,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { { auto opacity_host = - params.Get("h.opacity_baseunits"); + params.Get("h.opacity"); const Real YeMin = pin->GetOrAddReal("mean_opacity", "yemin", 0.1); const Real YeMax = pin->GetOrAddReal("mean_opacity", "yemax", 0.5); const int NYe = pin->GetOrAddInteger("mean_opacity", "nye", 10); @@ -162,7 +162,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { auto mean_opac_host = MeanOpacityBase(opacity_host, lRhoMin, lRhoMax, NRho, lTMin, lTMax, NT, YeMin, YeMax, NYe, lNuMin, lNuMax, NNu); - auto mean_opac_device = mean_opac_host.GetOnDevice(); + MeanOpacity mean_opac_device = mean_opac_host.GetOnDevice(); params.Add("h.mean_opacity", mean_opac_host); params.Add("d.mean_opacity", mean_opac_device); } else { @@ -182,7 +182,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { auto mean_opac_host = MeanNonCGSUnits( std::forward(cgs_mean_opacity), time_unit, mass_unit, length_unit, temp_unit); - auto mean_opac_device = mean_opac_host.GetOnDevice(); + MeanOpacity mean_opac_device = mean_opac_host.GetOnDevice(); params.Add("h.mean_opacity", mean_opac_host); params.Add("d.mean_opacity", mean_opac_device); } @@ -285,7 +285,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { } auto opacity_device = params.Get("d.opacity"); - auto &mean_opac_device = params.Get("d.mean_opacity"); + auto &mean_opac_device = params.Get("d.mean_opacity"); auto &s_opacity_device = params.Get("d.s_opacity"); auto &mean_s_opac_device = params.Get("d.mean_s_opacity"); Opacities opacities(opacity_device, mean_opac_device, s_opacity_device, From 2bc3a910afe7ad064abf16ac60b3ae1091de33ae Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Tue, 7 Jan 2025 18:22:10 -0500 Subject: [PATCH 21/48] weird changed behavior in regression testing --- tst/regression/regression_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tst/regression/regression_test.py b/tst/regression/regression_test.py index 6f1450e3b..4667d8be8 100644 --- a/tst/regression/regression_test.py +++ b/tst/regression/regression_test.py @@ -29,7 +29,7 @@ BUILD_DIR = "build" RUN_DIR = "run" SOURCE_DIR = "../../../" -NUM_PROCS = 4 # Default values for cmake --build --parallel can overwhelm CI systems +NUM_PROCS = 8 # Default values for cmake --build --parallel can overwhelm CI systems TEMPORARY_INPUT_FILE = "test_input.pin" SCRIPT_NAME = os.path.basename(__main__.__file__).split(".py")[0] @@ -298,7 +298,7 @@ def gold_comparison( if len(variable.shape) > 1: dim = variable.shape[0] for d in range(dim): - variables_data = np.concatenate((variables_data, variable[d, :])) + variables_data = np.concatenate((variables_data, variable[d, :].flatten())) else: variables_data = np.concatenate((variables_data, variable)) From e76ecccb308818c129cc142c71a25d1f5999577c Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Tue, 7 Jan 2025 18:24:21 -0500 Subject: [PATCH 22/48] fmt --- src/microphysics/opac_phoebus/opac_phoebus.cpp | 3 +-- tst/regression/regression_test.py | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/microphysics/opac_phoebus/opac_phoebus.cpp b/src/microphysics/opac_phoebus/opac_phoebus.cpp index f1610c970..b7ffe3aa5 100644 --- a/src/microphysics/opac_phoebus/opac_phoebus.cpp +++ b/src/microphysics/opac_phoebus/opac_phoebus.cpp @@ -144,8 +144,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { } { - auto opacity_host = - params.Get("h.opacity"); + auto opacity_host = params.Get("h.opacity"); const Real YeMin = pin->GetOrAddReal("mean_opacity", "yemin", 0.1); const Real YeMax = pin->GetOrAddReal("mean_opacity", "yemax", 0.5); const int NYe = pin->GetOrAddInteger("mean_opacity", "nye", 10); diff --git a/tst/regression/regression_test.py b/tst/regression/regression_test.py index 4667d8be8..2c6e9ae25 100644 --- a/tst/regression/regression_test.py +++ b/tst/regression/regression_test.py @@ -298,7 +298,9 @@ def gold_comparison( if len(variable.shape) > 1: dim = variable.shape[0] for d in range(dim): - variables_data = np.concatenate((variables_data, variable[d, :].flatten())) + variables_data = np.concatenate( + (variables_data, variable[d, :].flatten()) + ) else: variables_data = np.concatenate((variables_data, variable)) From bf398c467613f09e2c96d1f5c729ce33d92aca51 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Thu, 9 Jan 2025 20:49:48 +0000 Subject: [PATCH 23/48] feat: tracers work on MeshData, use SWARM_VARIABLES for tracer vars --- src/phoebus_driver.cpp | 18 ++-- src/phoebus_utils/variables.hpp | 23 +++++ src/tracers/tracers.cpp | 176 +++++++++++--------------------- src/tracers/tracers.hpp | 2 +- 4 files changed, 98 insertions(+), 121 deletions(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 23df07a04..60e44aa7c 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -586,19 +586,25 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { } } - TaskRegion &async_region_tr = tc.AddRegion(blocks.size()); - for (int n = 0; n < blocks.size(); n++) { + TaskRegion &async_region_tr = tc.AddRegion(num_partitions); + for (int n = 0; n < num_partitions; n++) { auto &tl = async_region_tr[n]; auto &pmb = blocks[n]; - auto &sc = pmb->meshblock_data.Get()->GetSwarmData(); + auto &base = pmesh->mesh_data.GetOrAdd("base", n); auto &mbd0 = pmb->meshblock_data.Get(stage_name[stage]); - auto tracerAdvect = tl.AddTask(none, tracers::AdvectTracers, mbd0.get(), dt); + auto tracerAdvect = tl.AddTask(none, tracers::AdvectTracers, base.get(), dt); auto tracerPurge = tl.AddTask(tracerAdvect, fixup::PurgeParticles, mbd0.get(), swarmName); + } - auto send = tl.AddTask(tracerPurge, &SwarmContainer::Send, sc.get(), - BoundaryCommSubset::all); + TaskRegion &async_region_tr_comm = tc.AddRegion(blocks.size()); + for (int n = 0; n < blocks.size(); n++) { + auto &tl = async_region_tr_comm[n]; + auto &pmb = blocks[n]; + auto &sc = pmb->meshblock_data.Get()->GetSwarmData(); + auto send = + tl.AddTask(none, &SwarmContainer::Send, sc.get(), BoundaryCommSubset::all); auto receive = tl.AddTask(send, &SwarmContainer::Receive, sc.get(), BoundaryCommSubset::all); diff --git a/src/phoebus_utils/variables.hpp b/src/phoebus_utils/variables.hpp index 2c127ea2e..6a78c5477 100644 --- a/src/phoebus_utils/variables.hpp +++ b/src/phoebus_utils/variables.hpp @@ -118,6 +118,29 @@ VARIABLE_CUSTOM(cell_coords, g.c.coord); VARIABLE_CUSTOM(node_coords, g.n.coord); } // namespace geometric_variables +namespace tracer_variables { +SWARM_VARIABLE(Real, tr, rho); +SWARM_VARIABLE(Real, tr, temperature); +SWARM_VARIABLE(Real, tr, ye); +SWARM_VARIABLE(Real, tr, entropy); +SWARM_VARIABLE(Real, tr, pressure); +SWARM_VARIABLE(Real, tr, energy); +SWARM_VARIABLE(Real, tr, vel_x); +SWARM_VARIABLE(Real, tr, vel_y); +SWARM_VARIABLE(Real, tr, vel_z); +SWARM_VARIABLE(Real, tr, lorentz); +SWARM_VARIABLE(Real, tr, lapse); +SWARM_VARIABLE(Real, tr, detgamma); +SWARM_VARIABLE(Real, tr, shift_x); +SWARM_VARIABLE(Real, tr, shift_y); +SWARM_VARIABLE(Real, tr, shift_z); +SWARM_VARIABLE(Real, tr, mass); +SWARM_VARIABLE(Real, tr, bernoulli); +SWARM_VARIABLE(Real, tr, B_x); +SWARM_VARIABLE(Real, tr, B_y); +SWARM_VARIABLE(Real, tr, B_z); +} // namespace tracer_variables + namespace diagnostic_variables { VARIABLE_NONS(divb); VARIABLE_NONS(ratio_divv_cs); diff --git a/src/tracers/tracers.cpp b/src/tracers/tracers.cpp index a3b449f1d..6ab347583 100644 --- a/src/tracers/tracers.cpp +++ b/src/tracers/tracers.cpp @@ -47,55 +47,57 @@ std::shared_ptr Initialize(ParameterInput *pin) { physics->AddSwarmValue("id", swarm_name, Metadata({Metadata::Integer})); // thermo variables - physics->AddSwarmValue("rho", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("temperature", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("ye", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("entropy", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("pressure", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("energy", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("vel_x", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("vel_y", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("vel_z", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("lorentz", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("lapse", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("detgamma", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("shift_x", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("shift_y", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("shift_z", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("mass", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("bernoulli", swarm_name, real_swarmvalue_metadata); + namespace tv = tracer_variables; + physics->AddSwarmValue(tv::rho::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::temperature::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::ye::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::entropy::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::pressure::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::energy::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::vel_x::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::vel_y::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::vel_z::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::lorentz::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::lapse::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::detgamma::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::shift_x::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::shift_y::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::shift_z::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::mass::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::bernoulli::name(), swarm_name, real_swarmvalue_metadata); const bool mhd = pin->GetOrAddBoolean("fluid", "mhd", false); if (mhd) { - physics->AddSwarmValue("B_x", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("B_y", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("B_z", swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::B_x::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::B_y::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(tv::B_z::name(), swarm_name, real_swarmvalue_metadata); } return physics; } // Initialize -TaskStatus AdvectTracers(MeshBlockData *rc, const Real dt) { +TaskStatus AdvectTracers(MeshData *rc, const Real dt) { namespace p = fluid_prim; auto *pmb = rc->GetParentPointer(); - Mesh *pmesh = rc->GetMeshPointer(); - const auto ndim = pmb->pmy_mesh->ndim; + static const auto ndim = pmb->ndim; - // tracer position swarm pack - const auto swarm_name = "tracers"; - static auto desc_tracer = + // tracer swarm pack + static const auto swarm_name = "tracers"; + static const auto desc_tracer = MakeSwarmPackDescriptor( swarm_name); auto pack_tracers = desc_tracer.GetPack(rc); static const auto vars = {p::velocity::name()}; + static const auto desc = MakePackDescriptor(rc); PackIndexMap imap; auto pack = rc->PackVariables(vars, imap); + // TODO(BLB): move to sparse packs. requires reworking tracer_rhs const int pvel_lo = imap[p::velocity::name()].first; const int pvel_hi = imap[p::velocity::name()].second; @@ -145,6 +147,7 @@ TaskStatus AdvectTracers(MeshBlockData *rc, const Real dt) { void FillTracers(MeshBlockData *rc) { using namespace LCInterp; namespace p = fluid_prim; + namespace tv = tracer_variables; auto *pmb = rc->GetParentPointer(); auto fluid = pmb->packages.Get("fluid"); @@ -156,48 +159,14 @@ void FillTracers(MeshBlockData *rc) { // tracer swarm pack const auto swarm_name = "tracers"; - static auto desc_tracer_pos = - MakeSwarmPackDescriptor( + static auto desc_tracers = + MakeSwarmPackDescriptor( swarm_name); - auto pack_tracers_pos = desc_tracer_pos.GetPack(rc); - - // tracer vars pack - std::vector swarm_vars = { - "vel_x", "vel_y", "vel_z", "rho", "temperature", "ye", - "entropy", "energy", "lorentz", "lapse", "shift_x", "shift_y", - "shift_z", "detgamma", "pressure", "bernoulli"}; - - if (mhd) { - swarm_vars.push_back("B_x"); - swarm_vars.push_back("B_y"); - swarm_vars.push_back("B_z"); - } - - // make pack and get index map - static auto desc_tracer_vars = MakeSwarmPackDescriptor(swarm_name, swarm_vars); - auto pack_tracers_vars = desc_tracer_vars.GetPack(rc); - auto pack_tracers_vars_map = desc_tracer_vars.GetMap(); - - // TODO(BLB): way to clean this up? - parthenon::PackIdx spi_vel_x(pack_tracers_vars_map["vel_x"]); - parthenon::PackIdx spi_vel_y(pack_tracers_vars_map["vel_y"]); - parthenon::PackIdx spi_vel_z(pack_tracers_vars_map["vel_z"]); - parthenon::PackIdx spi_B_x(pack_tracers_vars_map["B_x"]); - parthenon::PackIdx spi_B_y(pack_tracers_vars_map["B_y"]); - parthenon::PackIdx spi_B_z(pack_tracers_vars_map["B_z"]); - parthenon::PackIdx spi_rho(pack_tracers_vars_map["rho"]); - parthenon::PackIdx spi_temperature(pack_tracers_vars_map["temperature"]); - parthenon::PackIdx spi_ye(pack_tracers_vars_map["ye"]); - parthenon::PackIdx spi_entropy(pack_tracers_vars_map["entropy"]); - parthenon::PackIdx spi_energy(pack_tracers_vars_map["energy"]); - parthenon::PackIdx spi_lorentz(pack_tracers_vars_map["lorentz"]); - parthenon::PackIdx spi_lapse(pack_tracers_vars_map["lapse"]); - parthenon::PackIdx spi_shift_x(pack_tracers_vars_map["shift_x"]); - parthenon::PackIdx spi_shift_y(pack_tracers_vars_map["shift_y"]); - parthenon::PackIdx spi_shift_z(pack_tracers_vars_map["shift_z"]); - parthenon::PackIdx spi_detgamma(pack_tracers_vars_map["detgamma"]); - parthenon::PackIdx spi_pressure(pack_tracers_vars_map["pressure"]); - parthenon::PackIdx spi_bernoulli(pack_tracers_vars_map["bernoulli"]); + auto pack_tracers = desc_tracers.GetPack(rc); // hydro vars pack std::vector vars = {p::density::name(), p::temperature::name(), @@ -224,36 +193,15 @@ void FillTracers(MeshBlockData *rc) { // update loop. const int max_active_index = swarm->GetMaxActiveIndex(); pmb->par_for( - "Fill Tracers", 0, pack_tracers_pos.GetMaxFlatIndex(), - KOKKOS_LAMBDA(const int idx) { - const auto [b, n] = pack_tracers_pos.GetBlockParticleIndices(idx); - const auto swarm_d = pack_tracers_pos.GetContext(b); - - // TODO(BLB): clean up - const int ivel_x = pack_tracers_vars.GetLowerBound(b, spi_vel_x); - const int ivel_y = pack_tracers_vars.GetLowerBound(b, spi_vel_y); - const int ivel_z = pack_tracers_vars.GetLowerBound(b, spi_vel_z); - const int iB_x = pack_tracers_vars.GetLowerBound(b, spi_B_x); - const int iB_y = pack_tracers_vars.GetLowerBound(b, spi_B_y); - const int iB_z = pack_tracers_vars.GetLowerBound(b, spi_B_z); - const int irho = pack_tracers_vars.GetLowerBound(b, spi_rho); - const int itemperature = pack_tracers_vars.GetLowerBound(b, spi_temperature); - const int iye = pack_tracers_vars.GetLowerBound(b, spi_ye); - const int ientropy = pack_tracers_vars.GetLowerBound(b, spi_entropy); - const int ienergy = pack_tracers_vars.GetLowerBound(b, spi_energy); - const int ilorentz = pack_tracers_vars.GetLowerBound(b, spi_lorentz); - const int ilapse = pack_tracers_vars.GetLowerBound(b, spi_lapse); - const int ishift_x = pack_tracers_vars.GetLowerBound(b, spi_shift_x); - const int ishift_y = pack_tracers_vars.GetLowerBound(b, spi_shift_y); - const int ishift_z = pack_tracers_vars.GetLowerBound(b, spi_shift_z); - const int idetgamma = pack_tracers_vars.GetLowerBound(b, spi_detgamma); - const int ipressure = pack_tracers_vars.GetLowerBound(b, spi_pressure); - const int ibernoulli = pack_tracers_vars.GetLowerBound(b, spi_bernoulli); + "Fill Tracers", 0, pack_tracers.GetMaxFlatIndex(), KOKKOS_LAMBDA(const int idx) { + const auto [b, n] = pack_tracers.GetBlockParticleIndices(idx); + const auto swarm_d = pack_tracers.GetContext(b); + if (swarm_d.IsActive(n)) { - const Real x = pack_tracers_pos(b, swarm_position::x(), n); - const Real y = pack_tracers_pos(b, swarm_position::y(), n); - const Real z = pack_tracers_pos(b, swarm_position::z(), n); + const Real x = pack_tracers(b, swarm_position::x(), n); + const Real y = pack_tracers(b, swarm_position::y(), n); + const Real z = pack_tracers(b, swarm_position::z(), n); // geom quantities Real gcov4[4][4]; @@ -300,26 +248,26 @@ void FillTracers(MeshBlockData *rc) { const Real bernoulli = -(W / lapse) * h - 1.0; // store - pack_tracers_vars(b, irho, n) = rho; - pack_tracers_vars(b, itemperature, n) = temperature; - pack_tracers_vars(b, iye, n) = ye; - pack_tracers_vars(b, ienergy, n) = energy; - pack_tracers_vars(b, ientropy, n) = entropy; - pack_tracers_vars(b, ivel_x, n) = vel_X1; - pack_tracers_vars(b, ivel_y, n) = vel_X2; - pack_tracers_vars(b, ivel_z, n) = vel_X3; - pack_tracers_vars(b, ishift_x, n) = shift[0]; - pack_tracers_vars(b, ishift_y, n) = shift[1]; - pack_tracers_vars(b, ishift_z, n) = shift[2]; - pack_tracers_vars(b, ilapse, n) = lapse; - pack_tracers_vars(b, ilorentz, n) = W; - pack_tracers_vars(b, idetgamma, n) = gdet; - pack_tracers_vars(b, ipressure, n) = pressure; - pack_tracers_vars(b, ibernoulli, n) = bernoulli; + pack_tracers(b, tv::rho(), n) = rho; + pack_tracers(b, tv::temperature(), n) = temperature; + pack_tracers(b, tv::ye(), n) = ye; + pack_tracers(b, tv::energy(), n) = energy; + pack_tracers(b, tv::entropy(), n) = entropy; + pack_tracers(b, tv::vel_x(), n) = vel_X1; + pack_tracers(b, tv::vel_y(), n) = vel_X2; + pack_tracers(b, tv::vel_z(), n) = vel_X3; + pack_tracers(b, tv::shift_x(), n) = shift[0]; + pack_tracers(b, tv::shift_y(), n) = shift[1]; + pack_tracers(b, tv::shift_z(), n) = shift[2]; + pack_tracers(b, tv::lapse(), n) = lapse; + pack_tracers(b, tv::lorentz(), n) = W; + pack_tracers(b, tv::detgamma(), n) = gdet; + pack_tracers(b, tv::pressure(), n) = pressure; + pack_tracers(b, tv::bernoulli(), n) = bernoulli; if (mhd) { - pack_tracers_vars(b, iB_x, n) = B_X1; - pack_tracers_vars(b, iB_y, n) = B_X2; - pack_tracers_vars(b, iB_z, n) = B_X3; + pack_tracers(b, tv::B_x(), n) = B_X1; + pack_tracers(b, tv::B_y(), n) = B_X2; + pack_tracers(b, tv::B_z(), n) = B_X3; } bool on_current_mesh_block = true; diff --git a/src/tracers/tracers.hpp b/src/tracers/tracers.hpp index 595696d37..d4cf0d45f 100644 --- a/src/tracers/tracers.hpp +++ b/src/tracers/tracers.hpp @@ -76,7 +76,7 @@ KOKKOS_INLINE_FUNCTION void tracers_rhs(Pack &pack, Geometry &geom, const int pv } } -TaskStatus AdvectTracers(MeshBlockData *rc, const Real dt); +TaskStatus AdvectTracers(MeshData *rc, const Real dt); void FillTracers(MeshBlockData *rc); From 13a84dd712e808f061f239ff26e3d10583624f5c Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Fri, 17 Jan 2025 00:05:21 +0000 Subject: [PATCH 24/48] WIP + compile time number of tracies for MOCMC --- CMakeLists.txt | 3 +++ cmake/Transport.cmake | 13 +++++++++++++ src/compile_constants.hpp.in | 2 ++ src/radiation/radiation.cpp | 24 +++++++++++++++--------- 4 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 cmake/Transport.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 4700cfd1f..bb8d5daca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -185,6 +185,9 @@ include(cmake/Geometry.cmake) # Fluid include(cmake/Fluid.cmake) +# Transport +include(cmake/Transport.cmake) + # Phoebus src message("\nConfiguring src") add_subdirectory(src) diff --git a/cmake/Transport.cmake b/cmake/Transport.cmake new file mode 100644 index 000000000..6e97fb6e5 --- /dev/null +++ b/cmake/Transport.cmake @@ -0,0 +1,13 @@ +# © 2021-2025. Triad National Security, LLC. All rights reserved. This +# program was produced under U.S. Government contract 89233218CNA000001 +# for Los Alamos National Laboratory (LANL), which is operated by Triad +# National Security, LLC for the U.S. Department of Energy/National +# Nuclear Security Administration. All rights in the program are +# reserved by Triad National Security, LLC, and the U.S. Department of +# Energy/National Nuclear Security Administration. The Government is +# granted for itself and others acting on its behalf a nonexclusive, +# paid-up, irrevocable worldwide license in this material to reproduce, +# prepare derivative works, distribute copies to the public, perform +# publicly and display publicly, and to permit others to do so. + +set(MOCMC_NUM_SPECIES 3 CACHE INTEGER "Number of radiation species for MOCMC.") diff --git a/src/compile_constants.hpp.in b/src/compile_constants.hpp.in index 4fdf0db20..44c69c8a7 100644 --- a/src/compile_constants.hpp.in +++ b/src/compile_constants.hpp.in @@ -16,6 +16,8 @@ #define NCONS_MAX (@MAX_NUMBER_CONSERVED_VARS@) +#define MOCMC_NUM_SPECIES (@MOCMC_NUM_SPECIES@) + #define PHOEBUS_GEOMETRY Geometry::@PHOEBUS_GEOMETRY@ #define GEOMETRY_MESH @GEOMETRY_MESH@ #define GEOMETRY_MESH_BLOCK @GEOMETRY_MESH_BLOCK@ diff --git a/src/radiation/radiation.cpp b/src/radiation/radiation.cpp index cb0cb633e..0ca6f58cb 100644 --- a/src/radiation/radiation.cpp +++ b/src/radiation/radiation.cpp @@ -144,20 +144,26 @@ std::shared_ptr Initialize(ParameterInput *pin) { } if (method == "mocmc") { - std::string swarm_name = "mocmc"; + static constexpr auto swarm_name = "mocmc"; Metadata swarm_metadata({Metadata::Provides}); physics->AddSwarm(swarm_name, swarm_metadata); Metadata real_swarmvalue_metadata({Metadata::Real}); - physics->AddSwarmValue("t", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("mu_lo", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("mu_hi", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("phi_lo", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("phi_hi", swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(mocmc_core::t::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(mocmc_core::mu_lo::name(), swarm_name, + real_swarmvalue_metadata); + physics->AddSwarmValue(mocmc_core::mu_hi::name(), swarm_name, + real_swarmvalue_metadata); + physics->AddSwarmValue(mocmc_core::phi_lo::name(), swarm_name, + real_swarmvalue_metadata); + physics->AddSwarmValue(mocmc_core::phi_hi::name(), swarm_name, + real_swarmvalue_metadata); Metadata fourv_swarmvalue_metadata({Metadata::Real}, std::vector{4}); - physics->AddSwarmValue("ncov", swarm_name, fourv_swarmvalue_metadata); + physics->AddSwarmValue(mocmc_core::ncov::name(), swarm_name, + fourv_swarmvalue_metadata); Metadata Inu_swarmvalue_metadata({Metadata::Real}, - std::vector{num_species, nu_bins}); - physics->AddSwarmValue("Inuinv", swarm_name, Inu_swarmvalue_metadata); + std::vector{MOCMC_NUM_SPECIES, nu_bins}); + physics->AddSwarmValue(mocmc_core::Inuinv::name(), swarm_name, + Inu_swarmvalue_metadata); // Boundary temperatures for outflow sample boundary conditions const std::string ix1_bc = pin->GetOrAddString("phoebus", "ix1_bc", "None"); From a2990dd74d3902c6986ca3ba6991f34b30ac8c86 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Fri, 17 Jan 2025 00:15:32 +0000 Subject: [PATCH 25/48] WIP MOCMC refactor to use swarm packs and MeshData and work with modern parthenon --- src/phoebus_driver.cpp | 109 +++++-- src/phoebus_utils/variables.hpp | 24 ++ src/radiation/geodesics.hpp | 16 +- src/radiation/mocmc.cpp | 541 +++++++++++++++++--------------- src/radiation/radiation.hpp | 12 +- src/tracers/tracers.cpp | 24 +- 6 files changed, 422 insertions(+), 304 deletions(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 60e44aa7c..813b91bd6 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -282,6 +282,7 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { } // Goal: make async regions go away + // TODO(BLB): move moments code to packs / MeshData TaskRegion &async_region_1 = tc.AddRegion(num_independent_task_lists); for (int ib = 0; ib < num_independent_task_lists; ib++) { auto pmb = blocks[ib].get(); @@ -320,24 +321,68 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { auto fix_flux = tl.AddTask(sndrcv_flux_depend, fixup::FixFluxes, sc0.get()); sndrcv_flux_depend = sndrcv_flux_depend | fix_flux; } + } - if (rad_mocmc_active) { - using MDT = std::remove_pointer::type; + if (rad_mocmc_active) { + TaskRegion &sync_region_tr = tc.AddRegion(1); + { + for (int i = 0; i < blocks.size(); i++) { + auto &tl = sync_region_tr[0]; + auto &pmb = blocks[i]; + auto &sc = pmb->meshblock_data.Get()->GetSwarmData(); + auto reset_comms = + tl.AddTask(none, &SwarmContainer::ResetCommunication, sc.get()); + } + } + } + + // TODO(BLB) merge ifs + // if (rad_mocmc_active && stage == integrator->nstages) { + if (rad_mocmc_active) { + TaskRegion &sync_region_mocmc = tc.AddRegion(num_partitions); + for (int n = 0; n < num_partitions; n++) { + auto &tl = sync_region_mocmc[n]; + + auto &base = pmesh->mesh_data.GetOrAdd("base", n); + using MDT = std::remove_pointer::type; // TODO(BRR) stage_name[stage - 1]? - auto &sd0 = pmb->meshblock_data.Get()->GetSwarmData(); auto samples_transport = - tl.AddTask(none, radiation::MOCMCTransport, sc0.get(), dt); - auto send = tl.AddTask(samples_transport, &SwarmContainer::Send, sd0.get(), - BoundaryCommSubset::all); + tl.AddTask(none, radiation::MOCMCTransport, base.get(), beta * dt); + } + } + + if (rad_mocmc_active) { + TaskRegion &sync_region_mocmc_comm = tc.AddRegion(blocks.size()); + for (int ib = 0; ib < blocks.size(); ib++) { + auto &tl = sync_region_mocmc_comm[ib]; + auto pmb = blocks[ib].get(); + + // TODO(BRR) stage_name[stage - 1]? + auto &sc = pmb->meshblock_data.Get()->GetSwarmData(); + auto send = + tl.AddTask(none, &SwarmContainer::Send, sc.get(), BoundaryCommSubset::all); auto receive = - tl.AddTask(send, &SwarmContainer::Receive, sd0.get(), BoundaryCommSubset::all); + tl.AddTask(send, &SwarmContainer::Receive, sc.get(), BoundaryCommSubset::all); + } + } + + if (rad_mocmc_active) { + TaskRegion &async_region_mocmc_2 = tc.AddRegion(num_partitions); + for (int n = 0; n < num_partitions; n++) { + auto &tl = async_region_mocmc_2[n]; + + auto &base = pmesh->mesh_data.GetOrAdd("base", n); + auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], n); + using MDT = std::remove_pointer::type; + // TODO(BRR) stage_name[stage - 1]? auto sample_bounds = - tl.AddTask(receive, radiation::MOCMCSampleBoundaries, sc0.get()); - auto sample_recon = - tl.AddTask(sample_bounds, radiation::MOCMCReconstruction, sc0.get()); + tl.AddTask(none, radiation::MOCMCSampleBoundaries, base.get(), sc1.get()); + auto sample_recon = tl.AddTask(sample_bounds, radiation::MOCMCReconstruction, + base.get(), sc1.get()); auto eddington = - tl.AddTask(sample_recon, radiation::MOCMCEddington, sc0.get()); + tl.AddTask(sample_recon, radiation::MOCMCEddington, base.get(), sc1.get()); + TaskID geom_src(0); geom_src = geom_src | eddington; } } @@ -524,13 +569,16 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { } // Fix up flux failures + // TODO (DO NOT MERGE) is this loop right + // TODO (BLB) once the fixups can work with MeshData, + // -- then stitch this section back together TaskRegion &async_region_2 = tc.AddRegion(num_independent_task_lists); for (int ib = 0; ib < num_independent_task_lists; ib++) { auto pmb = blocks[ib].get(); auto &tl = async_region_2[ib]; // first make other useful containers - auto &base = pmb->meshblock_data.Get(); + // auto &base = pmb->meshblock_data.Get(); // pull out the container we'll use to get fluxes and/or compute RHSs auto &sc1 = pmb->meshblock_data.Get(stage_name[stage]); @@ -548,23 +596,40 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { auto floors = tl.AddTask(radfixup, fixup::ApplyFloors>, sc1.get()); + } - TaskID gas_rad_int(0); + TaskRegion &sync_region_fixup = tc.AddRegion(num_partitions); + TaskID gas_rad_int(0); + for (int ib = 0; ib < num_partitions; ib++) { + auto &tl = sync_region_fixup[ib]; + + auto &base = pmesh->mesh_data.GetOrAdd("base", ib); + auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], ib); + using MDT = std::remove_pointer::type; if (rad_mocmc_active) { - auto impl_update = - tl.AddTask(floors, radiation::MOCMCFluidSource>, sc1.get(), - beta * dt, fluid_active); - auto impl_edd = tl.AddTask( - impl_update, radiation::MOCMCEddington>, sc1.get()); + auto impl_update = tl.AddTask(none, radiation::MOCMCFluidSource, base.get(), + sc1.get(), beta * dt, fluid_active); + auto impl_edd = + tl.AddTask(impl_update, radiation::MOCMCEddington, base.get(), sc1.get()); gas_rad_int = gas_rad_int | impl_edd; } else if (rad_moments_active) { + // auto impl_update = + // tl.AddTask(none, radiation::MomentFluidSource>, sc1.get(), + // beta * dt, fluid_active); + // gas_rad_int = gas_rad_int | impl_update; + } + } + + TaskRegion &async_region_fixup2 = tc.AddRegion(num_independent_task_lists); + for (int ib = 0; ib < num_independent_task_lists; ib++) { + auto pmb = blocks[ib].get(); + auto &tl = async_region_fixup2[ib]; + auto &sc1 = pmb->meshblock_data.Get(stage_name[stage]); + if (rad_moments_active) { // TODO(PRE MERGE) move outside auto impl_update = - tl.AddTask(floors, radiation::MomentFluidSource>, sc1.get(), + tl.AddTask(none, radiation::MomentFluidSource>, sc1.get(), beta * dt, fluid_active); gas_rad_int = gas_rad_int | impl_update; - } - - if (rad_moments_active) { // Only apply floors because MomentFluidSource already ensured that a sensible state // was returned auto floors = diff --git a/src/phoebus_utils/variables.hpp b/src/phoebus_utils/variables.hpp index 6a78c5477..f2b4f833c 100644 --- a/src/phoebus_utils/variables.hpp +++ b/src/phoebus_utils/variables.hpp @@ -14,6 +14,8 @@ #ifndef PHOEBUS_UTILS_VARIABLES_HPP_ #define PHOEBUS_UTILS_VARIABLES_HPP_ +#include "compile_constants.hpp" + #include #include @@ -41,6 +43,17 @@ static std::string name() { return #varstring; } \ } +#define TENSOR_SWARM(type, ns, varname, ...) \ + struct varname : public parthenon::swarm_variable_names::base_t { \ + template \ + KOKKOS_INLINE_FUNCTION varname(Ts &&...args) \ + : parthenon::swarm_variable_names::base_t( \ + std::forward(args)...) {} \ + static std::string name() { return #ns "." #varname; } \ + } + +using parthenon::variable_names::ANYDIM; + namespace fluid_prim { VARIABLE(p, density); VARIABLE(p, velocity); @@ -96,6 +109,16 @@ VARIABLE(mocmc.i, Inu1); VARIABLE(mocmc.i, jinvs); } // namespace mocmc_internal +namespace mocmc_core { +SWARM_VARIABLE(Real, mocmc.c, t); +SWARM_VARIABLE(Real, mocmc.c, mu_lo); +SWARM_VARIABLE(Real, mocmc.c, mu_hi); +SWARM_VARIABLE(Real, mocmc.c, phi_lo); +SWARM_VARIABLE(Real, mocmc.c, phi_hi); +SWARM_VARIABLE(Real, mocmc.c, ncov); +TENSOR_SWARM(Real, mocmc.c, Inuinv, ANYDIM, MOCMC_NUM_SPECIES); +} // namespace mocmc_core + namespace internal_variables { VARIABLE_NONS(face_signal_speed); VARIABLE_NONS(cell_signal_speed); @@ -139,6 +162,7 @@ SWARM_VARIABLE(Real, tr, bernoulli); SWARM_VARIABLE(Real, tr, B_x); SWARM_VARIABLE(Real, tr, B_y); SWARM_VARIABLE(Real, tr, B_z); +TENSOR_SWARM(Real, tr, test, ANYDIM, MOCMC_NUM_SPECIES); } // namespace tracer_variables namespace diagnostic_variables { diff --git a/src/radiation/geodesics.hpp b/src/radiation/geodesics.hpp index fcfda22f2..f3706e65e 100644 --- a/src/radiation/geodesics.hpp +++ b/src/radiation/geodesics.hpp @@ -35,17 +35,17 @@ void GetXSource(Real &Kcon0, Real &Kcon1, Real &Kcon2, Real &Kcon3, Real src[NDF src[3] = Kcon3 / Kcon0; } -KOKKOS_INLINE_FUNCTION -void GetKSource(Real &X0, Real &X1, Real &X2, Real &X3, Real &Kcov0, Real &Kcov1, - Real &Kcov2, Real &Kcov3, Real &Kcon0, - const Geometry::CoordSysMeshBlock &geom, Real source[4]) { +template +KOKKOS_INLINE_FUNCTION void GetKSource(Real &X0, Real &X1, Real &X2, Real &X3, + Real &Kcov0, Real &Kcov1, Real &Kcov2, Real &Kcov3, + Real &Kcon0, Geom &geom, Real source[4]) { SPACETIMELOOP(mu) { source[mu] = 0.; } } -KOKKOS_INLINE_FUNCTION -void PushParticle(Real &X0, Real &X1, Real &X2, Real &X3, Real &Kcov0, Real &Kcov1, - Real &Kcov2, Real &Kcov3, const Real &dt, - const Geometry::CoordSysMeshBlock &geom) { +template +KOKKOS_INLINE_FUNCTION void PushParticle(Real &X0, Real &X1, Real &X2, Real &X3, + Real &Kcov0, Real &Kcov1, Real &Kcov2, + Real &Kcov3, const Real &dt, Geom &geom) { Real c1[NDFULL], c2[NDFULL], d1[NDFULL], d2[NDFULL]; Real Xtmp[NDFULL], Kcontmp[NDFULL], Kcovtmp[NDFULL]; Real Kcov[NDFULL] = {Kcov0, Kcov1, Kcov2, Kcov3}; diff --git a/src/radiation/mocmc.cpp b/src/radiation/mocmc.cpp index d4a8c349d..d05508bc2 100644 --- a/src/radiation/mocmc.cpp +++ b/src/radiation/mocmc.cpp @@ -163,8 +163,8 @@ void MOCMCInitSamples(T *rc) { const auto &x = swarm->template Get(swarm_position::x::name()).Get(); const auto &y = swarm->template Get(swarm_position::y::name()).Get(); const auto &z = swarm->template Get(swarm_position::z::name()).Get(); - const auto &ncov = swarm->template Get("ncov").Get(); - const auto &Inuinv = swarm->template Get("Inuinv").Get(); + const auto &ncov = swarm->template Get(mocmc_core::ncov::name()).Get(); + const auto &Inuinv = swarm->template Get(mocmc_core::Inuinv::name()).Get(); auto swarm_d = swarm->GetDeviceContext(); @@ -225,22 +225,20 @@ void MOCMCInitSamples(T *rc) { }); // Initialize eddington tensor and opacities for first step - MOCMCReconstruction(rc); - MOCMCEddington(rc); - MOCMCFluidSource(rc, 0., false); // Update opacities for asymptotic fluxes + MOCMCReconstruction(rc, rc); + MOCMCEddington(rc, rc); + MOCMCFluidSource(rc, rc, 0., false); // Update opacities for asymptotic fluxes } template -TaskStatus MOCMCSampleBoundaries(T *rc) { +TaskStatus MOCMCSampleBoundaries(T *rc_base, T *rc) { auto *pmb = rc->GetParentPointer(); - auto &sc = rc->GetSwarmData(); - auto &swarm = sc->Get("mocmc"); StateDescriptor *rad = pmb->packages.Get("radiation").get(); const auto geom = Geometry::GetCoordinateSystem(rc); - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + IndexRange ib = rc->GetBoundsI(IndexDomain::interior); + IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); + IndexRange kb = rc->GetBoundsK(IndexDomain::interior); std::vector variables{pr::J::name(), pf::velocity::name(), ir::tilPi::name()}; @@ -251,15 +249,14 @@ TaskStatus MOCMCSampleBoundaries(T *rc) { auto opac_pkg = pmb->packages.Get("opacity"); const auto opac = opac_pkg->template Param("opacities"); - const auto &x = swarm->template Get(swarm_position::x::name()).Get(); - const auto &y = swarm->template Get(swarm_position::y::name()).Get(); - const auto &z = swarm->template Get(swarm_position::z::name()).Get(); - const auto &ncov = swarm->template Get("ncov").Get(); - const auto &Inuinv = swarm->template Get("Inuinv").Get(); - const auto &mu_lo = swarm->template Get("mu_lo").Get(); - const auto &mu_hi = swarm->template Get("mu_hi").Get(); - const auto &phi_lo = swarm->template Get("phi_lo").Get(); - const auto &phi_hi = swarm->template Get("phi_hi").Get(); + static constexpr auto swarm_name = "mocmc"; + static const auto desc_mocmc = + MakeSwarmPackDescriptor( + swarm_name); + auto pack_mocmc = desc_mocmc.GetPack(rc_base); + auto pv = imap.GetFlatIdx(pf::velocity::name()); auto iTilPi = imap.GetFlatIdx(ir::tilPi::name()); auto iJ = imap.GetFlatIdx(pr::J::name()); @@ -273,8 +270,6 @@ TaskStatus MOCMCSampleBoundaries(T *rc) { species_d[s] = species[s]; } - auto swarm_d = swarm->GetDeviceContext(); - // TODO(BRR) provide *all* MOCMC boundaries auto ix1_bc = rad->Param("ix1_bc"); @@ -289,26 +284,33 @@ TaskStatus MOCMCSampleBoundaries(T *rc) { // ox1_temp = rad->Param("ox1_temp"); //} - pmb->par_for( - "Temporary MOCMC boundaries", 0, swarm->GetMaxActiveIndex(), - KOKKOS_LAMBDA(const int n) { + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "MOCMC::SampleBoundaries", DevExecSpace(), 0, + pack_mocmc.GetMaxFlatIndex(), KOKKOS_LAMBDA(const int idx) { + const auto [b, n] = pack_mocmc.GetBlockParticleIndices(idx); + const auto swarm_d = pack_mocmc.GetContext(b); if (swarm_d.IsActive(n)) { + Real &x = pack_mocmc(b, swarm_position::x(), n); + Real &y = pack_mocmc(b, swarm_position::y(), n); + Real &z = pack_mocmc(b, swarm_position::z(), n); // Store zone before reflections int i, j, k; - swarm_d.Xtoijk(x(n), y(n), z(n), i, j, k); + swarm_d.Xtoijk(x, y, z, i, j, k); - if (x(n) < swarm_d.x_min_global_) { + if (x < swarm_d.x_min_global_) { // Reflect particle across boundary - x(n) = swarm_d.x_min_global_ + (swarm_d.x_min_global_ - x(n)); + x = swarm_d.x_min_global_ + (swarm_d.x_min_global_ - x); // TODO(BRR) normalized in relativity? - ncov(1, n) = -ncov(1, n); + pack_mocmc(b, mocmc_core::ncov(1), n) = + -pack_mocmc(b, mocmc_core::ncov(1), n); for (int s = 0; s < num_species; s++) { Real temp = 0.; if (ix1_bc == MOCMCBoundaries::outflow) { // Temperature from J in ghost zone - temp = opac.TemperatureFromEnergyDensity(v(iJ(s), k, j, i), species_d[s]); + temp = + opac.TemperatureFromEnergyDensity(v(b, iJ(s), k, j, i), species_d[s]); } else { // Fixed temperature temp = ix1_temp; @@ -317,7 +319,7 @@ TaskStatus MOCMCSampleBoundaries(T *rc) { // Reset intensities for (int nubin = 0; nubin < nu_bins; nubin++) { const Real nu = nusamp(nubin); - Inuinv(nubin, s, n) = + pack_mocmc(b, mocmc_core::Inuinv(nubin, s), n) = std::max(robust::SMALL(), opac.ThermalDistributionOfTNu( temp, species_d[s], nu)) / std::pow(nu, 3); @@ -325,21 +327,22 @@ TaskStatus MOCMCSampleBoundaries(T *rc) { } } - if (x(n) > swarm_d.x_max_global_) { + if (x > swarm_d.x_max_global_) { // Reflect particle across boundary - x(n) = swarm_d.x_max_global_ - (x(n) - swarm_d.x_max_global_); - ncov(1, n) = -ncov(1, n); + x = swarm_d.x_max_global_ - (x - swarm_d.x_max_global_); + pack_mocmc(b, mocmc_core::ncov(1), n) = + -pack_mocmc(b, mocmc_core::ncov(1), n); // Reset intensities for (int nubin = 0; nubin < nu_bins; nubin++) { for (int s = 0; s < num_species; s++) { - Inuinv(nubin, s, n) = robust::SMALL(); + pack_mocmc(b, mocmc_core::Inuinv(nubin, s), n) = robust::SMALL(); } } } bool on_current_mesh_block = true; - swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); + swarm_d.GetNeighborBlockIndex(n, x, y, z, on_current_mesh_block); } }); @@ -347,67 +350,72 @@ TaskStatus MOCMCSampleBoundaries(T *rc) { } template -TaskStatus MOCMCReconstruction(T *rc) { +TaskStatus MOCMCReconstruction(T *rc_base, T *rc) { auto *pmb = rc->GetParentPointer(); - auto &sc = rc->GetSwarmData(); - auto &swarm = sc->Get("mocmc"); StateDescriptor *rad = pmb->packages.Get("radiation").get(); + Real num_total = 0; + // Sort and accumulate total active particles + // Sorting must be done before forming packs. + for (int b = 0; b <= rc->NumBlocks() - 1; b++) { + rc_base->GetSwarmData(b)->Get("mocmc")->SortParticlesByCell(); + num_total += rc_base->GetSwarmData(b)->Get("mocmc")->GetNumActive(); + } + rad->UpdateParam("num_total", num_total); + const auto geom = Geometry::GetCoordinateSystem(rc); - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + IndexRange ib = rc->GetBoundsI(IndexDomain::interior); + IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); + IndexRange kb = rc->GetBoundsK(IndexDomain::interior); std::vector variables{pf::velocity::name(), ir::tilPi::name()}; PackIndexMap imap; auto v = rc->PackVariables(variables, imap); - const auto &x = swarm->template Get(swarm_position::x::name()).Get(); - const auto &y = swarm->template Get(swarm_position::y::name()).Get(); - const auto &z = swarm->template Get(swarm_position::z::name()).Get(); - const auto &ncov = swarm->template Get("ncov").Get(); - const auto &Inuinv = swarm->template Get("Inuinv").Get(); - const auto &mu_lo = swarm->template Get("mu_lo").Get(); - const auto &mu_hi = swarm->template Get("mu_hi").Get(); - const auto &phi_lo = swarm->template Get("phi_lo").Get(); - const auto &phi_hi = swarm->template Get("phi_hi").Get(); + static constexpr auto swarm_name = "mocmc"; + static const auto desc_mocmc = + MakeSwarmPackDescriptor( + swarm_name); + auto pack_mocmc = desc_mocmc.GetPack(rc_base); auto pv = imap.GetFlatIdx(pf::velocity::name()); auto iTilPi = imap.GetFlatIdx(ir::tilPi::name()); auto nusamp = rad->Param>("nusamp"); const int nu_bins = rad->Param("nu_bins"); - swarm->SortParticlesByCell(); - auto swarm_d = swarm->GetDeviceContext(); - - rad->UpdateParam("num_total", swarm->GetNumActive()); - auto mocmc_recon = rad->Param("mocmc_recon"); parthenon::par_for( - DEFAULT_LOOP_PATTERN, "MOCMC::kdgrid", DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, - ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { + DEFAULT_LOOP_PATTERN, "MOCMC::kdgrid", DevExecSpace(), 0, rc->NumBlocks() - 1, kb.s, + kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto swarm_d = pack_mocmc.GetContext(b); const int nsamp = swarm_d.GetParticleCountPerCell(k, j, i); for (int n = 0; n < nsamp; n++) { const int nswarm = swarm_d.GetFullIndex(k, j, i, n); if (n == 0) { - mu_lo(nswarm) = 0.0; - mu_hi(nswarm) = 2.0; - phi_lo(nswarm) = 0.0; - phi_hi(nswarm) = 2.0 * M_PI; + pack_mocmc(b, mocmc_core::mu_lo(), nswarm) = 0.0; + pack_mocmc(b, mocmc_core::mu_hi(), nswarm) = 2.0; + pack_mocmc(b, mocmc_core::phi_lo(), nswarm) = 0.0; + pack_mocmc(b, mocmc_core::phi_hi(), nswarm) = 2.0 * M_PI; continue; } Real cov_g[4][4]; - geom.SpacetimeMetric(CellLocation::Cent, k, j, i, cov_g); - const Real vpcon[] = {v(pv(0), k, j, i), v(pv(1), k, j, i), v(pv(2), k, j, i)}; + geom.SpacetimeMetric(CellLocation::Cent, b, k, j, i, cov_g); + const Real vpcon[] = {v(b, pv(0), k, j, i), v(b, pv(1), k, j, i), + v(b, pv(2), k, j, i)}; const Real trial[4] = {0., 1., 0., 0.}; Real ucon[4]; - GetFourVelocity(vpcon, geom, CellLocation::Cent, k, j, i, ucon); + GetFourVelocity(vpcon, geom, CellLocation::Cent, b, k, j, i, ucon); Geometry::Tetrads tetrads(ucon, trial, cov_g); - Real ncov_coord[4] = {ncov(0, nswarm), ncov(1, nswarm), ncov(2, nswarm), - ncov(3, nswarm)}; + Real ncov_coord[4] = {pack_mocmc(b, mocmc_core::ncov(0), nswarm), + pack_mocmc(b, mocmc_core::ncov(1), nswarm), + pack_mocmc(b, mocmc_core::ncov(2), nswarm), + pack_mocmc(b, mocmc_core::ncov(3), nswarm)}; Real ncov_tetrad[4]; tetrads.CoordToTetradCov(ncov_coord, ncov_tetrad); @@ -417,40 +425,53 @@ TaskStatus MOCMCReconstruction(T *rc) { for (int m = 0; m < n; m++) { const int mswarm = swarm_d.GetFullIndex(k, j, i, m); PARTHENON_DEBUG_REQUIRE(mswarm != nswarm, "Comparing the same particle!"); - if (mu > mu_lo(mswarm) && mu < mu_hi(mswarm) && phi > phi_lo(mswarm) && - phi < phi_hi(mswarm)) { - Real mcov_tetrad[4] = {-1., ncov(1, mswarm), ncov(2, mswarm), - ncov(3, mswarm)}; - Real mu0 = mu_hi(mswarm) - mu_lo(mswarm); - Real phi0 = phi_hi(mswarm) - phi_lo(mswarm); + if (mu > pack_mocmc(b, mocmc_core::mu_lo(), mswarm) && + mu < pack_mocmc(b, mocmc_core::mu_hi(), mswarm) && + phi > pack_mocmc(b, mocmc_core::phi_lo(), mswarm) && + phi < pack_mocmc(b, mocmc_core::phi_hi(), mswarm)) { + Real mcov_tetrad[4] = {-1., pack_mocmc(b, mocmc_core::ncov(1), mswarm), + pack_mocmc(b, mocmc_core::ncov(2), mswarm), + pack_mocmc(b, mocmc_core::ncov(3), mswarm)}; + Real mu0 = pack_mocmc(b, mocmc_core::mu_hi(), mswarm) - + pack_mocmc(b, mocmc_core::mu_lo(), mswarm); + Real phi0 = pack_mocmc(b, mocmc_core::phi_hi(), mswarm) - + pack_mocmc(b, mocmc_core::phi_lo(), mswarm); if (mu0 > phi0) { const Real mu_m = 1.0 - ncov_tetrad[1]; mu0 = 0.5 * (mu + mu_m); if (mu < mu0) { - mu_lo(nswarm) = mu_lo(mswarm); - mu_hi(nswarm) = mu0; - mu_lo(mswarm) = mu0; + pack_mocmc(b, mocmc_core::mu_lo(), nswarm) = + pack_mocmc(b, mocmc_core::mu_lo(), mswarm); + pack_mocmc(b, mocmc_core::mu_hi(), nswarm) = mu0; + pack_mocmc(b, mocmc_core::mu_lo(), mswarm) = mu0; } else { - mu_lo(nswarm) = mu0; - mu_hi(nswarm) = mu_hi(mswarm); - mu_hi(mswarm) = mu0; + pack_mocmc(b, mocmc_core::mu_lo(), nswarm) = mu0; + pack_mocmc(b, mocmc_core::mu_hi(), nswarm) = + pack_mocmc(b, mocmc_core::mu_hi(), mswarm); + pack_mocmc(b, mocmc_core::mu_hi(), mswarm) = mu0; } - phi_lo(nswarm) = phi_lo(mswarm); - phi_hi(nswarm) = phi_hi(mswarm); + pack_mocmc(b, mocmc_core::phi_lo(), nswarm) = + pack_mocmc(b, mocmc_core::phi_lo(), mswarm); + pack_mocmc(b, mocmc_core::phi_hi(), nswarm) = + pack_mocmc(b, mocmc_core::phi_hi(), mswarm); } else { const Real phi_m = atan2(ncov_tetrad[3], ncov_tetrad[2]); phi0 = 0.5 * (phi + phi_m); if (phi < phi0) { - phi_lo(nswarm) = phi_lo(mswarm); - phi_hi(nswarm) = phi0; - phi_lo(mswarm) = phi0; + pack_mocmc(b, mocmc_core::phi_lo(), nswarm) = + pack_mocmc(b, mocmc_core::phi_lo(), mswarm); + pack_mocmc(b, mocmc_core::phi_hi(), nswarm) = phi0; + pack_mocmc(b, mocmc_core::phi_lo(), mswarm) = phi0; } else { - phi_lo(nswarm) = phi0; - phi_hi(nswarm) = phi_hi(mswarm); - phi_hi(mswarm) = phi0; + pack_mocmc(b, mocmc_core::phi_lo(), nswarm) = phi0; + pack_mocmc(b, mocmc_core::phi_hi(), nswarm) = + pack_mocmc(b, mocmc_core::phi_hi(), mswarm); + pack_mocmc(b, mocmc_core::phi_hi(), mswarm) = phi0; } - mu_lo(nswarm) = mu_lo(mswarm); - mu_hi(nswarm) = mu_hi(mswarm); + pack_mocmc(b, mocmc_core::mu_lo(), nswarm) = + pack_mocmc(b, mocmc_core::mu_lo(), mswarm); + pack_mocmc(b, mocmc_core::mu_hi(), nswarm) = + pack_mocmc(b, mocmc_core::mu_hi(), mswarm); } break; } // if inside @@ -464,56 +485,54 @@ TaskStatus MOCMCReconstruction(T *rc) { template TaskStatus MOCMCTransport(T *rc, const Real dt) { auto *pmb = rc->GetParentPointer(); - auto &sc = rc->GetSwarmData(); - auto &swarm = sc->Get("mocmc"); auto geom = Geometry::GetCoordinateSystem(rc); - auto &t = swarm->template Get("t").Get(); - auto &x = swarm->template Get(swarm_position::x::name()).Get(); - auto &y = swarm->template Get(swarm_position::y::name()).Get(); - auto &z = swarm->template Get(swarm_position::z::name()).Get(); - auto &ncov = swarm->template Get("ncov").Get(); - auto swarm_d = swarm->GetDeviceContext(); + static constexpr auto swarm_name = "mocmc"; + static const auto desc_mocmc = + MakeSwarmPackDescriptor(swarm_name); + auto pack_mocmc = desc_mocmc.GetPack(rc); - pmb->par_for( - "MOCMC::Transport", 0, swarm->GetMaxActiveIndex(), KOKKOS_LAMBDA(const int n) { + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "MOCMC::Transport", DevExecSpace(), 0, + pack_mocmc.GetMaxFlatIndex(), KOKKOS_LAMBDA(const int idx) { + const auto [b, n] = pack_mocmc.GetBlockParticleIndices(idx); + const auto swarm_d = pack_mocmc.GetContext(b); if (swarm_d.IsActive(n)) { - Real y0 = y(n); - Real z0 = z(n); - PushParticle(t(n), x(n), y(n), z(n), ncov(0, n), ncov(1, n), ncov(2, n), - ncov(3, n), dt, geom); + Real &t = pack_mocmc(b, mocmc_core::t(), n); + Real &x = pack_mocmc(b, swarm_position::x(), n); + if (n == 240) std::printf("t, x, n, b = %f %f %d %d\n", t, x, n, b); + Real &y = pack_mocmc(b, swarm_position::y(), n); + Real &z = pack_mocmc(b, swarm_position::z(), n); + Real y0 = y; + Real z0 = z; + PushParticle(t, x, y, z, pack_mocmc(b, mocmc_core::ncov(0), n), + pack_mocmc(b, mocmc_core::ncov(1), n), + pack_mocmc(b, mocmc_core::ncov(2), n), + pack_mocmc(b, mocmc_core::ncov(3), n), dt, geom); + if (n == 240) std::printf("t, x, n, b = %f %f %d %d\n", t, x, n, b); bool on_current_mesh_block = true; - swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); + swarm_d.GetNeighborBlockIndex(n, x, y, z, on_current_mesh_block); } }); return TaskStatus::complete; } -// TODO(BRR): Hack to get around current lack of support for packing parthenon swarms -template <> -TaskStatus MOCMCFluidSource(MeshData *rc, const Real dt, const bool update_fluid) { - for (int n = 0; n < rc->NumBlocks(); n++) { - MOCMCFluidSource(rc->GetBlockData(n).get(), dt, update_fluid); - } - return TaskStatus::complete; -} - template -TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { +TaskStatus MOCMCFluidSource(T *rc_base, T *rc, const Real dt, const bool update_fluid) { // Assume particles are already sorted from MOCMCReconstruction call! auto *pmb = rc->GetParentPointer(); - auto &sc = rc->GetSwarmData(); - auto &swarm = sc->Get("mocmc"); + // swarm pack StateDescriptor *rad = pmb->packages.Get("radiation").get(); // Meshblock geometry const auto geom = Geometry::GetCoordinateSystem(rc); - const IndexRange &ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - const IndexRange &jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - const IndexRange &kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + const IndexRange &ib = rc->GetBoundsI(IndexDomain::interior); + const IndexRange &jb = rc->GetBoundsJ(IndexDomain::interior); + const IndexRange &kb = rc->GetBoundsK(IndexDomain::interior); // Microphysics auto opac = pmb->packages.Get("opacity"); @@ -556,13 +575,12 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { cye = imap[cf::ye::name()].first; } - const auto &ncov = swarm->template Get("ncov").Get(); - const auto &mu_lo = swarm->template Get("mu_lo").Get(); - const auto &mu_hi = swarm->template Get("mu_hi").Get(); - const auto &phi_lo = swarm->template Get("phi_lo").Get(); - const auto &phi_hi = swarm->template Get("phi_hi").Get(); - const auto &Inuinv = swarm->template Get("Inuinv").Get(); - const auto swarm_d = swarm->GetDeviceContext(); + static constexpr auto swarm_name = "mocmc"; + static const auto desc_mocmc = + MakeSwarmPackDescriptor( + swarm_name); + auto pack_mocmc = desc_mocmc.GetPack(rc_base); auto nusamp = rad->Param>("nusamp"); const int nu_bins = rad->Param("nu_bins"); @@ -574,19 +592,21 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { species_d[s] = species[s]; } - const int iblock = 0; // No meshblockpacks right now int nspec = idx_E.DimSize(1); if (true) { // update = lagged parthenon::par_for( - DEFAULT_LOOP_PATTERN, "MOCMC::FluidSource", DevExecSpace(), kb.s, kb.e, jb.s, - jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { + DEFAULT_LOOP_PATTERN, "MOCMC::FluidSource", DevExecSpace(), 0, + rc->NumBlocks() - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto swarm_d = pack_mocmc.GetContext(b); Real cov_g[4][4]; - geom.SpacetimeMetric(CellLocation::Cent, iblock, k, j, i, cov_g); - Real alpha = geom.Lapse(CellLocation::Cent, iblock, k, j, i); + geom.SpacetimeMetric(CellLocation::Cent, b, k, j, i, cov_g); + Real alpha = geom.Lapse(CellLocation::Cent, b, k, j, i); Real con_beta[3]; - geom.ContravariantShift(CellLocation::Cent, iblock, k, j, i, con_beta); - const Real vpcon[] = {v(pv(0), k, j, i), v(pv(1), k, j, i), v(pv(2), k, j, i)}; + geom.ContravariantShift(CellLocation::Cent, b, k, j, i, con_beta); + const Real vpcon[] = {v(b, pv(0), k, j, i), v(b, pv(1), k, j, i), + v(b, pv(2), k, j, i)}; const Real W = phoebus::GetLorentzFactor(vpcon, cov_g); const Real ucon[4] = {W / alpha, vpcon[0] - con_beta[0] * W / alpha, vpcon[1] - con_beta[1] * W / alpha, @@ -596,23 +616,22 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { const int nsamp = swarm_d.GetParticleCountPerCell(k, j, i); // Set up the background state - Vec con_v{{v(iblock, pv(0), k, j, i), v(iblock, pv(1), k, j, i), - v(iblock, pv(2), k, j, i)}}; + Vec con_v{{v(b, pv(0), k, j, i), v(b, pv(1), k, j, i), v(b, pv(2), k, j, i)}}; Tens2 cov_gamma; - geom.Metric(CellLocation::Cent, iblock, k, j, i, cov_gamma.data); - Real alpha = geom.Lapse(CellLocation::Cent, iblock, k, j, i); - Real sdetgam = geom.DetGamma(CellLocation::Cent, iblock, k, j, i); - LocalThreeGeometry g(geom, CellLocation::Cent, iblock, k, j, i); - Real Estar = v(iblock, idx_E(ispec), k, j, i) / sdetgam; - Vec cov_Fstar{v(iblock, idx_F(ispec, 0), k, j, i) / sdetgam, - v(iblock, idx_F(ispec, 1), k, j, i) / sdetgam, - v(iblock, idx_F(ispec, 2), k, j, i) / sdetgam}; + geom.Metric(CellLocation::Cent, b, k, j, i, cov_gamma.data); + Real alpha = geom.Lapse(CellLocation::Cent, b, k, j, i); + Real sdetgam = geom.DetGamma(CellLocation::Cent, b, k, j, i); + LocalThreeGeometry g(geom, CellLocation::Cent, b, k, j, i); + Real Estar = v(b, idx_E(ispec), k, j, i) / sdetgam; + Vec cov_Fstar{v(b, idx_F(ispec, 0), k, j, i) / sdetgam, + v(b, idx_F(ispec, 1), k, j, i) / sdetgam, + v(b, idx_F(ispec, 2), k, j, i) / sdetgam}; Tens2 con_tilPi; SPACELOOP2(ii, jj) { - con_tilPi(ii, jj) = v(iblock, iTilPi(ispec, ii, jj), k, j, i); + con_tilPi(ii, jj) = v(b, iTilPi(ispec, ii, jj), k, j, i); } - Real JBB = opac_d.EnergyDensityFromTemperature(v(iblock, pT, k, j, i), - species_d[ispec]); + Real JBB = + opac_d.EnergyDensityFromTemperature(v(b, pT, k, j, i), species_d[ispec]); ClosureMOCMC<> c(con_v, &g); @@ -621,15 +640,19 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { // Calculate Inu0 for (int bin = 0; bin < nu_bins; bin++) { - v(iblock, Inu0(ispec, bin), k, j, i) = 0.; + v(b, Inu0(ispec, bin), k, j, i) = 0.; } for (int n = 0; n < nsamp; n++) { const int nswarm = swarm_d.GetFullIndex(k, j, i, n); - const Real dOmega = (mu_hi(nswarm) - mu_lo(nswarm)) * - (phi_hi(nswarm) - phi_lo(nswarm)) / (4. * M_PI); + const Real dOmega = (pack_mocmc(b, mocmc_core::mu_hi(), nswarm) - + pack_mocmc(b, mocmc_core::mu_lo(), nswarm)) * + (pack_mocmc(b, mocmc_core::phi_hi(), nswarm) - + pack_mocmc(b, mocmc_core::phi_lo(), nswarm)) / + (4. * M_PI); for (int bin = 0; bin < nu_bins; bin++) { - v(iblock, Inu0(ispec, bin), k, j, i) += - Inuinv(bin, ispec, nswarm) * pow(nusamp(bin), 3) * dOmega; + v(b, Inu0(ispec, bin), k, j, i) += + pack_mocmc(b, mocmc_core::Inuinv(bin, ispec), nswarm) * + pow(nusamp(bin), 3) * dOmega; } } @@ -640,42 +663,39 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { Real Itot = 0.; for (int bin = 0; bin < nu_bins; bin++) { kappaJ += opac_d.AngleAveragedAbsorptionCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nusamp(bin)) * - v(iblock, Inu0(ispec, bin), k, j, i) * nusamp(bin); + v(b, pdens, k, j, i), v(b, pT, k, j, i), v(b, pye, k, j, i), + species_d[ispec], nusamp(bin)) * + v(b, Inu0(ispec, bin), k, j, i) * nusamp(bin); kappaH += opac_d.TotalScatteringCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nusamp(bin)) * - v(iblock, Inu0(ispec, bin), k, j, i) * nusamp(bin); - Itot += v(iblock, Inu0(ispec, bin), k, j, i) * nusamp(bin); + v(b, pdens, k, j, i), v(b, pT, k, j, i), v(b, pye, k, j, i), + species_d[ispec], nusamp(bin)) * + v(b, Inu0(ispec, bin), k, j, i) * nusamp(bin); + Itot += v(b, Inu0(ispec, bin), k, j, i) * nusamp(bin); } // Trapezoidal rule kappaJ -= 0.5 * opac_d.AngleAveragedAbsorptionCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nusamp(0)) * - v(iblock, Inu0(ispec, 0), k, j, i) * nusamp(0); - kappaJ -= - 0.5 * - opac_d.AngleAveragedAbsorptionCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nusamp(nu_bins - 1)) * - v(iblock, Inu0(ispec, nu_bins - 1), k, j, i) * nusamp(nu_bins - 1); + v(b, pdens, k, j, i), v(b, pT, k, j, i), v(b, pye, k, j, i), + species_d[ispec], nusamp(0)) * + v(b, Inu0(ispec, 0), k, j, i) * nusamp(0); + kappaJ -= 0.5 * + opac_d.AngleAveragedAbsorptionCoefficient( + v(b, pdens, k, j, i), v(b, pT, k, j, i), v(b, pye, k, j, i), + species_d[ispec], nusamp(nu_bins - 1)) * + v(b, Inu0(ispec, nu_bins - 1), k, j, i) * nusamp(nu_bins - 1); + kappaH -= 0.5 * + opac_d.TotalScatteringCoefficient( + v(b, pdens, k, j, i), v(b, pT, k, j, i), v(b, pye, k, j, i), + species_d[ispec], nusamp(0)) * + v(b, Inu0(ispec, 0), k, j, i) * nusamp(0); kappaH -= 0.5 * opac_d.TotalScatteringCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nusamp(0)) * - v(iblock, Inu0(ispec, 0), k, j, i) * nusamp(0); - kappaH -= - 0.5 * - opac_d.TotalScatteringCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nusamp(nu_bins - 1)) * - v(iblock, Inu0(ispec, nu_bins - 1), k, j, i) * nusamp(nu_bins - 1); - Itot -= 0.5 * - (v(iblock, Inu0(ispec, 0), k, j, i) * nusamp(0) + - v(iblock, Inu0(ispec, nu_bins - 1), k, j, i) * nusamp(nu_bins - 1)); + v(b, pdens, k, j, i), v(b, pT, k, j, i), v(b, pye, k, j, i), + species_d[ispec], nusamp(nu_bins - 1)) * + v(b, Inu0(ispec, nu_bins - 1), k, j, i) * nusamp(nu_bins - 1); + Itot -= 0.5 * (v(b, Inu0(ispec, 0), k, j, i) * nusamp(0) + + v(b, Inu0(ispec, nu_bins - 1), k, j, i) * nusamp(nu_bins - 1)); kappaJ = robust::ratio(kappaJ, Itot); kappaH = robust::ratio(kappaH, Itot) + kappaJ; @@ -683,24 +703,24 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { Real tauH = alpha * dt * kappaH; // Store kappaH for asymptotic fluxes - v(iblock, idx_kappaH(ispec), k, j, i) = kappaH; + v(b, idx_kappaH(ispec), k, j, i) = kappaH; auto status = c.LinearSourceUpdate(Estar, cov_Fstar, con_tilPi, JBB, tauJ, tauH, &dE, &cov_dF); // Add source corrections to conserved radiation variables - v(iblock, idx_E(ispec), k, j, i) += sdetgam * dE; + v(b, idx_E(ispec), k, j, i) += sdetgam * dE; for (int idir = 0; idir < 3; ++idir) { - v(iblock, idx_F(ispec, idir), k, j, i) += sdetgam * cov_dF(idir); + v(b, idx_F(ispec, idir), k, j, i) += sdetgam * cov_dF(idir); } // Add source corrections to conserved fluid variables if (update_fluid) { - v(iblock, cye, k, j, i) -= sdetgam * 0.0; - v(iblock, ceng, k, j, i) -= sdetgam * dE; - v(iblock, cmom_lo + 0, k, j, i) -= sdetgam * cov_dF(0); - v(iblock, cmom_lo + 1, k, j, i) -= sdetgam * cov_dF(1); - v(iblock, cmom_lo + 2, k, j, i) -= sdetgam * cov_dF(2); + v(b, cye, k, j, i) -= sdetgam * 0.0; + v(b, ceng, k, j, i) -= sdetgam * dE; + v(b, cmom_lo + 0, k, j, i) -= sdetgam * cov_dF(0); + v(b, cmom_lo + 1, k, j, i) -= sdetgam * cov_dF(1); + v(b, cmom_lo + 2, k, j, i) -= sdetgam * cov_dF(2); } // Update sample intensities @@ -709,7 +729,9 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { Real nu_fluid0 = nusamp(0); Real nu_lab0 = 0.; - SPACETIMELOOP(nu) { nu_lab0 -= ncov(nu, nswarm) * ucon[nu]; } + SPACETIMELOOP(nu) { + nu_lab0 -= pack_mocmc(b, mocmc_core::ncov(nu), nswarm) * ucon[nu]; + } nu_lab0 *= nusamp(0); const Real shift = (log(nusamp(0)) - log(nu_lab0)) / dlnu; @@ -723,11 +745,11 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { const Real nu_fluid = nusamp(bin); const Real alphainv_s = nu_fluid * opac_d.TotalScatteringCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nu_fluid); - v(iblock, ijinvs(ispec, bin), k, j, i) = - v(iblock, Inu0(ispec, bin), k, j, i) / - (nu_fluid * nu_fluid * nu_fluid) * alphainv_s; + v(b, pdens, k, j, i), v(b, pT, k, j, i), + v(b, pye, k, j, i), species_d[ispec], nu_fluid); + v(b, ijinvs(ispec, bin), k, j, i) = v(b, Inu0(ispec, bin), k, j, i) / + (nu_fluid * nu_fluid * nu_fluid) * + alphainv_s; } for (int bin = 0; bin < nu_bins; bin++) { @@ -736,27 +758,27 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { const Real ds = dt / ucon[0] / nu_fluid; const Real alphainv_a = nu_fluid * opac_d.TotalScatteringCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nu_fluid); + v(b, pdens, k, j, i), v(b, pT, k, j, i), + v(b, pye, k, j, i), species_d[ispec], nu_fluid); const Real alphainv_s = nu_fluid * opac_d.TotalScatteringCoefficient( - v(iblock, pdens, k, j, i), v(iblock, pT, k, j, i), - v(iblock, pye, k, j, i), species_d[ispec], nu_fluid); - const Real jinv_a = - opac_d.ThermalDistributionOfTNu(v(iblock, pT, k, j, i), - species_d[ispec], nu_fluid) / - (nu_fluid * nu_fluid * nu_fluid) * alphainv_a; + v(b, pdens, k, j, i), v(b, pT, k, j, i), + v(b, pye, k, j, i), species_d[ispec], nu_fluid); + const Real jinv_a = opac_d.ThermalDistributionOfTNu( + v(b, pT, k, j, i), species_d[ispec], nu_fluid) / + (nu_fluid * nu_fluid * nu_fluid) * alphainv_a; // Interpolate invariant scattering emissivity to lab frame Real jinv_s = 0.; interp.GetIndicesAndWeights(bin, nubin_shift, nubin_wgt); for (int isup = 0; isup < interp.StencilSize(); isup++) { - jinv_s += nubin_wgt[isup] * - v(iblock, ijinvs(ispec, nubin_shift[isup]), k, j, i); + jinv_s += + nubin_wgt[isup] * v(b, ijinvs(ispec, nubin_shift[isup]), k, j, i); } - Inuinv(bin, ispec, nswarm) = - (Inuinv(bin, ispec, nswarm) + ds * (jinv_a + jinv_s)) / + pack_mocmc(b, mocmc_core::Inuinv(bin, ispec), nswarm) = + (pack_mocmc(b, mocmc_core::Inuinv(bin, ispec), nswarm) + + ds * (jinv_a + jinv_s)) / (1. + ds * (alphainv_a + alphainv_s)); } } @@ -773,28 +795,17 @@ TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid) { return TaskStatus::complete; } -// TODO(BRR): Hack to get around current lack of support for packing parthenon swarms -template <> -TaskStatus MOCMCEddington(MeshData *rc) { - for (int n = 0; n < rc->NumBlocks(); n++) { - MOCMCEddington(rc->GetBlockData(n).get()); - } - return TaskStatus::complete; -} - template -TaskStatus MOCMCEddington(T *rc) { +TaskStatus MOCMCEddington(T *rc_base, T *rc) { // Assume list is sorted! namespace ir = radmoment_internal; auto *pmb = rc->GetParentPointer(); - auto &sc = rc->GetSwarmData(); - auto &swarm = sc->Get("mocmc"); StateDescriptor *rad = pmb->packages.Get("radiation").get(); - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + IndexRange ib = rc->GetBoundsI(IndexDomain::interior); + IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); + IndexRange kb = rc->GetBoundsK(IndexDomain::interior); auto geom = Geometry::GetCoordinateSystem(rc); @@ -802,12 +813,12 @@ TaskStatus MOCMCEddington(T *rc) { PackIndexMap imap; auto v = rc->PackVariables(variables, imap); - const auto &Inuinv = swarm->template Get("Inuinv").Get(); - const auto &ncov = swarm->template Get("ncov").Get(); - const auto &mu_lo = swarm->template Get("mu_lo").Get(); - const auto &mu_hi = swarm->template Get("mu_hi").Get(); - const auto &phi_lo = swarm->template Get("phi_lo").Get(); - const auto &phi_hi = swarm->template Get("phi_hi").Get(); + static constexpr auto swarm_name = "mocmc"; + static const auto desc_mocmc = + MakeSwarmPackDescriptor( + swarm_name); + auto pack_mocmc = desc_mocmc.GetPack(rc_base); auto iTilPi = imap.GetFlatIdx(ir::tilPi::name()); const auto pvel_lo = imap[pf::velocity::name()].first; @@ -816,18 +827,16 @@ TaskStatus MOCMCEddington(T *rc) { auto nusamp = rad->Param>("nusamp"); auto num_species = rad->Param("num_species"); - // TODO(BRR) block packing eventually - const int iblock = 0; - - auto swarm_d = swarm->GetDeviceContext(); parthenon::par_for( - DEFAULT_LOOP_PATTERN, "MOCMC::kdgrid", DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, - ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { + DEFAULT_LOOP_PATTERN, "MOCMC::kdgrid", DevExecSpace(), 0, rc->NumBlocks() - 1, kb.s, + kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto swarm_d = pack_mocmc.GetContext(b); // initialize eddington to zero for (int s = 0; s < num_species; s++) { for (int ii = 0; ii < 3; ii++) { for (int jj = ii; jj < 3; jj++) { - v(iTilPi(s, ii, jj), k, j, i) = 0.0; + v(b, iTilPi(s, ii, jj), k, j, i) = 0.0; } } } @@ -835,12 +844,12 @@ TaskStatus MOCMCEddington(T *rc) { const int nsamp = swarm_d.GetParticleCountPerCell(k, j, i); Real cov_g[4][4]; - geom.SpacetimeMetric(CellLocation::Cent, iblock, k, j, i, cov_g); - Real alpha = geom.Lapse(CellLocation::Cent, iblock, k, j, i); + geom.SpacetimeMetric(CellLocation::Cent, b, k, j, i, cov_g); + Real alpha = geom.Lapse(CellLocation::Cent, b, k, j, i); Real con_beta[3]; - geom.ContravariantShift(CellLocation::Cent, iblock, k, j, i, con_beta); - const Real vpcon[] = {v(pvel_lo, k, j, i), v(pvel_lo + 1, k, j, i), - v(pvel_lo + 2, k, j, i)}; + geom.ContravariantShift(CellLocation::Cent, b, k, j, i, con_beta); + const Real vpcon[] = {v(b, pvel_lo, k, j, i), v(b, pvel_lo + 1, k, j, i), + v(b, pvel_lo + 2, k, j, i)}; const Real W = phoebus::GetLorentzFactor(vpcon, cov_g); const Real ucon[4] = {W / alpha, vpcon[0] - con_beta[0] * W / alpha, vpcon[1] - con_beta[1] * W / alpha, @@ -851,7 +860,9 @@ TaskStatus MOCMCEddington(T *rc) { Real nu_fluid0 = nusamp(0); Real nu_lab0 = 0.; - SPACETIMELOOP(nu) { nu_lab0 -= ncov(nu, nswarm) * ucon[nu]; } + SPACETIMELOOP(nu) { + nu_lab0 -= pack_mocmc(b, mocmc_core::ncov(nu), nswarm) * ucon[nu]; + } nu_lab0 *= nusamp(0); const Real shift = (log(nusamp(0)) - log(nu_lab0)) / dlnu; @@ -866,26 +877,33 @@ TaskStatus MOCMCEddington(T *rc) { for (int s = 0; s < num_species; s++) { interp.GetIndicesAndWeights(nubin, nubin_shift, nubin_wgt); for (int isup = 0; isup < interp.StencilSize(); isup++) { - I[s] += nubin_wgt[isup] * Inuinv(nubin_shift[isup], s, nswarm) * + I[s] += nubin_wgt[isup] * + pack_mocmc(b, mocmc_core::Inuinv(nubin_shift[isup], s), nswarm) * pow(nusamp(nubin), 4); } } } Real wgts[6]; - kdgrid::integrate_ninj_domega_quad(mu_lo(nswarm), mu_hi(nswarm), phi_lo(nswarm), - phi_hi(nswarm), wgts); + kdgrid::integrate_ninj_domega_quad(pack_mocmc(b, mocmc_core::mu_lo(), nswarm), + pack_mocmc(b, mocmc_core::mu_hi(), nswarm), + pack_mocmc(b, mocmc_core::phi_lo(), nswarm), + pack_mocmc(b, mocmc_core::phi_hi(), nswarm), + wgts); for (int ii = 0; ii < 3; ii++) { for (int jj = ii; jj < 3; jj++) { const int ind = Geometry::Utils::Flatten2(ii, jj, 3); for (int s = 0; s < num_species; s++) { - v(iTilPi(s, ii, jj), k, j, i) += wgts[ind] * I[s]; + v(b, iTilPi(s, ii, jj), k, j, i) += wgts[ind] * I[s]; } } } for (int s = 0; s < num_species; s++) { - energy[s] += (mu_hi(nswarm) - mu_lo(nswarm)) * - (phi_hi(nswarm) - phi_lo(nswarm)) * I[s]; + energy[s] += (pack_mocmc(b, mocmc_core::mu_hi(), nswarm) - + pack_mocmc(b, mocmc_core::mu_lo(), nswarm)) * + (pack_mocmc(b, mocmc_core::phi_hi(), nswarm) - + pack_mocmc(b, mocmc_core::phi_lo(), nswarm)) * + I[s]; } } @@ -893,17 +911,17 @@ TaskStatus MOCMCEddington(T *rc) { for (int s = 0; s < num_species; s++) { for (int ii = 0; ii < 3; ii++) { for (int jj = ii; jj < 3; jj++) { - v(iTilPi(s, ii, jj), k, j, i) /= energy[s]; + v(b, iTilPi(s, ii, jj), k, j, i) /= energy[s]; } - v(iTilPi(s, ii, ii), k, j, i) -= 1. / 3.; + v(b, iTilPi(s, ii, ii), k, j, i) -= 1. / 3.; } } } for (int s = 0; s < num_species; s++) { - v(iTilPi(s, 1, 0), k, j, i) = v(iTilPi(s, 0, 1), k, j, i); - v(iTilPi(s, 2, 0), k, j, i) = v(iTilPi(s, 0, 2), k, j, i); - v(iTilPi(s, 2, 1), k, j, i) = v(iTilPi(s, 1, 2), k, j, i); + v(b, iTilPi(s, 1, 0), k, j, i) = v(b, iTilPi(s, 0, 1), k, j, i); + v(b, iTilPi(s, 2, 0), k, j, i) = v(b, iTilPi(s, 0, 2), k, j, i); + v(b, iTilPi(s, 2, 1), k, j, i) = v(b, iTilPi(s, 1, 2), k, j, i); } }); @@ -920,16 +938,21 @@ TaskStatus MOCMCUpdateParticleCount(Mesh *pmesh, std::vector *resolution) return TaskStatus::complete; } -template TaskStatus MOCMCSampleBoundaries>(MeshBlockData *); -template TaskStatus MOCMCReconstruction>(MeshBlockData *); -template TaskStatus MOCMCEddington>(MeshBlockData *rc); +template TaskStatus MOCMCSampleBoundaries>(MeshData *rc_base, + MeshData *rc); +// template TaskStatus MOCMCSampleBoundaries>(MeshData *); +template TaskStatus MOCMCReconstruction>(MeshData *rc_base, + MeshData *rc); +// template TaskStatus MOCMCReconstruction>(MeshData *); +template TaskStatus MOCMCEddington>(MeshData *rc_base, + MeshData *rc); +// template TaskStatus MOCMCEddington>(MeshData *rc); template void MOCMCInitSamples>(MeshBlockData *); -template TaskStatus MOCMCTransport>(MeshBlockData *rc, - const Real dt); +template TaskStatus MOCMCTransport>(MeshData *rc, const Real dt); // template TaskStatus MOCMCFluidSource>(MeshData *rc, const Real dt, -// const bool update_fluid); -template TaskStatus MOCMCFluidSource>(MeshBlockData *rc, - const Real dt, - const bool update_fluid); +// const bool update_fluid); +template TaskStatus MOCMCFluidSource>(MeshData *rc_base, + MeshData *rc, const Real dt, + const bool update_fluid); } // namespace radiation diff --git a/src/radiation/radiation.hpp b/src/radiation/radiation.hpp index 924e3199f..83c0839b1 100644 --- a/src/radiation/radiation.hpp +++ b/src/radiation/radiation.hpp @@ -154,16 +154,20 @@ template TaskStatus MOCMCTransport(T *rc, const Real dt); template -TaskStatus MOCMCSampleBoundaries(T *rc); +TaskStatus MOCMCSampleBoundaries(T *rc_base, T *rc); +// TaskStatus MOCMCSampleBoundaries(T *rc); template -TaskStatus MOCMCReconstruction(T *rc); +TaskStatus MOCMCReconstruction(T *rc_base, T *rc); +// TaskStatus MOCMCReconstruction(T *rc); template -TaskStatus MOCMCEddington(T *rc); +TaskStatus MOCMCEddington(T *rc_base, T *rc); +// TaskStatus MOCMCEddington(T *rc); template -TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid); +// TaskStatus MOCMCFluidSource(T *rc, const Real dt, const bool update_fluid); +TaskStatus MOCMCFluidSource(T *rc_base, T *rc, const Real dt, const bool update_fluid); TaskStatus MOCMCUpdateParticleCount(Mesh *pmesh, std::vector *resolution); diff --git a/src/tracers/tracers.cpp b/src/tracers/tracers.cpp index 6ab347583..148bdfe65 100644 --- a/src/tracers/tracers.cpp +++ b/src/tracers/tracers.cpp @@ -40,7 +40,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { physics->AddParam<>("rng_pool", rng_pool); // Add swarm of tracers - std::string swarm_name = "tracers"; + static constexpr auto swarm_name = "tracers"; Metadata swarm_metadata({Metadata::Provides, Metadata::None}); physics->AddSwarm(swarm_name, swarm_metadata); Metadata real_swarmvalue_metadata({Metadata::Real}); @@ -66,6 +66,10 @@ std::shared_ptr Initialize(ParameterInput *pin) { physics->AddSwarmValue(tv::mass::name(), swarm_name, real_swarmvalue_metadata); physics->AddSwarmValue(tv::bernoulli::name(), swarm_name, real_swarmvalue_metadata); + // TEST + Metadata Inu_swarmvalue_metadata({Metadata::Real}, std::vector{3, 100}); + physics->AddSwarmValue(tv::test::name(), swarm_name, Inu_swarmvalue_metadata); + const bool mhd = pin->GetOrAddBoolean("fluid", "mhd", false); if (mhd) { @@ -85,7 +89,7 @@ TaskStatus AdvectTracers(MeshData *rc, const Real dt) { static const auto ndim = pmb->ndim; // tracer swarm pack - static const auto swarm_name = "tracers"; + static constexpr auto swarm_name = "tracers"; static const auto desc_tracer = MakeSwarmPackDescriptor( swarm_name); @@ -158,14 +162,12 @@ void FillTracers(MeshBlockData *rc) { const auto mhd = fluid->Param("mhd"); // tracer swarm pack - const auto swarm_name = "tracers"; - static auto desc_tracers = - MakeSwarmPackDescriptor( - swarm_name); + static constexpr auto swarm_name = "tracers"; + static auto desc_tracers = MakeSwarmPackDescriptor< + swarm_position::x, swarm_position::y, swarm_position::z, tv::vel_x, tv::vel_y, + tv::vel_z, tv::rho, tv::temperature, tv::ye, tv::entropy, tv::energy, tv::lorentz, + tv::lapse, tv::shift_x, tv::shift_y, tv::shift_z, tv::detgamma, tv::pressure, + tv::bernoulli, tv::B_x, tv::B_y, tv::B_z, tv::test>(swarm_name); auto pack_tracers = desc_tracers.GetPack(rc); // hydro vars pack @@ -191,7 +193,6 @@ void FillTracers(MeshBlockData *rc) { auto geom = Geometry::GetCoordinateSystem(rc); // update loop. - const int max_active_index = swarm->GetMaxActiveIndex(); pmb->par_for( "Fill Tracers", 0, pack_tracers.GetMaxFlatIndex(), KOKKOS_LAMBDA(const int idx) { const auto [b, n] = pack_tracers.GetBlockParticleIndices(idx); @@ -264,6 +265,7 @@ void FillTracers(MeshBlockData *rc) { pack_tracers(b, tv::detgamma(), n) = gdet; pack_tracers(b, tv::pressure(), n) = pressure; pack_tracers(b, tv::bernoulli(), n) = bernoulli; + pack_tracers(b, tv::test(2 + 5), n) = 1.1238; if (mhd) { pack_tracers(b, tv::B_x(), n) = B_X1; pack_tracers(b, tv::B_y(), n) = B_X2; From 41c06e130c3e32d4728a89eb4117b1e6231f86d4 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Fri, 17 Jan 2025 00:18:37 +0000 Subject: [PATCH 26/48] remove debugging prints --- src/radiation/mocmc.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/radiation/mocmc.cpp b/src/radiation/mocmc.cpp index d05508bc2..b7c130819 100644 --- a/src/radiation/mocmc.cpp +++ b/src/radiation/mocmc.cpp @@ -501,7 +501,6 @@ TaskStatus MOCMCTransport(T *rc, const Real dt) { if (swarm_d.IsActive(n)) { Real &t = pack_mocmc(b, mocmc_core::t(), n); Real &x = pack_mocmc(b, swarm_position::x(), n); - if (n == 240) std::printf("t, x, n, b = %f %f %d %d\n", t, x, n, b); Real &y = pack_mocmc(b, swarm_position::y(), n); Real &z = pack_mocmc(b, swarm_position::z(), n); Real y0 = y; @@ -510,7 +509,6 @@ TaskStatus MOCMCTransport(T *rc, const Real dt) { pack_mocmc(b, mocmc_core::ncov(1), n), pack_mocmc(b, mocmc_core::ncov(2), n), pack_mocmc(b, mocmc_core::ncov(3), n), dt, geom); - if (n == 240) std::printf("t, x, n, b = %f %f %d %d\n", t, x, n, b); bool on_current_mesh_block = true; swarm_d.GetNeighborBlockIndex(n, x, y, z, on_current_mesh_block); From 3691f4b1c49e12b82cb16b58cb85539ed9bbd53e Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Fri, 17 Jan 2025 17:22:30 +0000 Subject: [PATCH 27/48] add check that number of species is consistent with compile time --- src/radiation/radiation.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/radiation/radiation.cpp b/src/radiation/radiation.cpp index 0ca6f58cb..ac1d1b405 100644 --- a/src/radiation/radiation.cpp +++ b/src/radiation/radiation.cpp @@ -96,6 +96,10 @@ std::shared_ptr Initialize(ParameterInput *pin) { params.Add("dlnu", dlnu); } + // NOTE(BLB): This will have to change if we ever support more than 3 species. + PARTHENON_REQUIRE(do_nu_electron + do_nu_electron_anti + do_nu_heavy == MOCMC_NUM_SPECIES, + "Requested species must be equal to compile time requested number of species."); + // TODO(BLB) With compile time num species this can be a static array. std::vector species; if (do_nu_electron) { species.push_back(RadiationType::NU_ELECTRON); @@ -106,7 +110,8 @@ std::shared_ptr Initialize(ParameterInput *pin) { if (do_nu_heavy) { species.push_back(RadiationType::NU_HEAVY); } - const int num_species = species.size(); + //const int num_species = species.size(); + static constexpr int num_species = MOCMC_NUM_SPECIES; params.Add("num_species", num_species); params.Add("species", species); @@ -161,7 +166,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { physics->AddSwarmValue(mocmc_core::ncov::name(), swarm_name, fourv_swarmvalue_metadata); Metadata Inu_swarmvalue_metadata({Metadata::Real}, - std::vector{MOCMC_NUM_SPECIES, nu_bins}); + std::vector{num_species, nu_bins}); physics->AddSwarmValue(mocmc_core::Inuinv::name(), swarm_name, Inu_swarmvalue_metadata); From e03dfeb97274ecb8e5ef314234f49f8f77db5f36 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Tue, 21 Jan 2025 23:26:24 +0000 Subject: [PATCH 28/48] thread compile time neutrino species into tests --- tst/regression/bondi.py | 1 + tst/regression/friedmann.py | 1 + tst/regression/homogeneous_sphere.py | 1 + tst/regression/linear_modes.py | 1 + tst/regression/mocmc_diffusion.py | 3 +++ tst/regression/mocmc_equilibration.py | 4 ++++ tst/regression/radiation_diffusion.py | 3 +++ tst/regression/radiation_equilibration.py | 3 +++ tst/regression/regression_test.py | 9 +++++++-- tst/regression/thincooling.py | 3 +++ tst/regression/thincooling_coolingfunction.py | 3 +++ 11 files changed, 30 insertions(+), 2 deletions(-) diff --git a/tst/regression/bondi.py b/tst/regression/bondi.py index c57ca6ce4..9d86f6e83 100755 --- a/tst/regression/bondi.py +++ b/tst/regression/bondi.py @@ -46,6 +46,7 @@ input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args="", geometry="FMKS", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/friedmann.py b/tst/regression/friedmann.py index 3a521fb4d..0f639826c 100755 --- a/tst/regression/friedmann.py +++ b/tst/regression/friedmann.py @@ -39,6 +39,7 @@ variables=["p.density", "p.energy"], input_file=args.input, executable=args.executable, + cmake_extra_args="", geometry="FLRW", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/homogeneous_sphere.py b/tst/regression/homogeneous_sphere.py index aaf3935c3..c4f91762d 100755 --- a/tst/regression/homogeneous_sphere.py +++ b/tst/regression/homogeneous_sphere.py @@ -52,6 +52,7 @@ input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args="", geometry="SphericalMinkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/linear_modes.py b/tst/regression/linear_modes.py index 99a821b0f..350ff2cd3 100755 --- a/tst/regression/linear_modes.py +++ b/tst/regression/linear_modes.py @@ -46,6 +46,7 @@ input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args="", geometry="Minkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/mocmc_diffusion.py b/tst/regression/mocmc_diffusion.py index f77bd09a8..6cbc85460 100755 --- a/tst/regression/mocmc_diffusion.py +++ b/tst/regression/mocmc_diffusion.py @@ -56,11 +56,14 @@ modified_inputs["parthenon/mesh/nx1"] = 64 modified_inputs["parthenon/meshblock/nx1"] = 64 +cmake_extra_args = "-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off" + code = rt.gold_comparison( variables=["r.p.J", "r.p.H"], input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args=cmake_extra_args, geometry="Minkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/mocmc_equilibration.py b/tst/regression/mocmc_equilibration.py index ab4c23680..8ea4bbf54 100755 --- a/tst/regression/mocmc_equilibration.py +++ b/tst/regression/mocmc_equilibration.py @@ -47,11 +47,15 @@ modified_inputs["radiation/mocmc/nsamp_per_zone"] = 32 modified_inputs["radiation_equilibration/J"] = 0.2 + +cmake_extra_args = "-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off" + code = rt.gold_comparison( variables=["r.p.J", "r.p.H"], input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args=cmake_extra_args, geometry="Minkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/radiation_diffusion.py b/tst/regression/radiation_diffusion.py index 1e36fc001..f8a296630 100755 --- a/tst/regression/radiation_diffusion.py +++ b/tst/regression/radiation_diffusion.py @@ -50,11 +50,14 @@ modified_inputs["radiation_advection/width"] = 0.0333 modified_inputs["radiation_advection/kappas_init"] = 1.0e3 +cmake_extra_args = "-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off" + code = rt.gold_comparison( variables=["r.p.J", "r.p.H"], input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args=cmake_extra_args, geometry="Minkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/radiation_equilibration.py b/tst/regression/radiation_equilibration.py index dfd11e2fc..4dc511599 100755 --- a/tst/regression/radiation_equilibration.py +++ b/tst/regression/radiation_equilibration.py @@ -43,11 +43,14 @@ modified_inputs["opacity/gray_kappa"] = 0.1 modified_inputs["radiation_equilibration/J"] = 0.2 +cmake_extra_args = "-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off" + code = rt.gold_comparison( variables=["r.p.J", "r.p.H"], input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args=cmake_extra_args, geometry="Minkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/regression_test.py b/tst/regression/regression_test.py index 2c6e9ae25..7a26a1770 100644 --- a/tst/regression/regression_test.py +++ b/tst/regression/regression_test.py @@ -148,7 +148,7 @@ def modify_input(dict_key, value, input_file): # -- Configure and build phoebus with problem-specific options -def build_code(geometry, use_gpu=False, build_type="Release"): +def build_code(geometry, use_gpu=False, build_type="Release", cmake_extra_args=""): if os.path.isdir(BUILD_DIR): print( f'BUILD_DIR "{BUILD_DIR}" already exists! Clean up before calling a regression test script!' @@ -166,6 +166,9 @@ def build_code(geometry, use_gpu=False, build_type="Release"): else: print(f'Build type "{build_type}" not known!') sys.exit(os.EX_SOFTWARE) + #configure_options.append(cmake_extra_args) + for arg in cmake_extra_args.split(): + configure_options.append(arg) configure_options.append("-DPHOEBUS_ENABLE_UNIT_TESTS=OFF") configure_options.append("-DMAX_NUMBER_CONSERVED_VARS=10") configure_options.append("-DPHOEBUS_CACHE_GEOMETRY=ON") @@ -187,6 +190,7 @@ def build_code(geometry, use_gpu=False, build_type="Release"): for option in configure_options: cmake_call.append(option) cmake_call.append(SOURCE_DIR) + print(cmake_call) # Configure call(cmake_call) @@ -238,6 +242,7 @@ def gold_comparison( input_file, modified_inputs={}, executable=None, + cmake_extra_args="", geometry="Minkowski", use_gpu=False, use_mpiexec=False, @@ -260,7 +265,7 @@ def gold_comparison( if executable is None: executable = os.path.join(BUILD_DIR, "src", "phoebus") - build_code(geometry, use_gpu, build_type) + build_code(geometry, use_gpu, build_type, cmake_extra_args) if os.path.isdir(RUN_DIR): print( diff --git a/tst/regression/thincooling.py b/tst/regression/thincooling.py index 39789b360..96177dbe9 100755 --- a/tst/regression/thincooling.py +++ b/tst/regression/thincooling.py @@ -38,11 +38,14 @@ modified_inputs = {} modified_inputs["radiation/method"] = "monte_carlo" +cmake_extra_args = "-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off" + code = rt.gold_comparison( variables=["p.density", "p.energy"], input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args=cmake_extra_args, geometry="Minkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/thincooling_coolingfunction.py b/tst/regression/thincooling_coolingfunction.py index e97d477ef..bf6f98644 100755 --- a/tst/regression/thincooling_coolingfunction.py +++ b/tst/regression/thincooling_coolingfunction.py @@ -38,11 +38,14 @@ modified_inputs = {} modified_inputs["radiation/method"] = "cooling_function" +cmake_extra_args = "-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off" + code = rt.gold_comparison( variables=["p.density", "p.energy"], input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, + cmake_extra_args=cmake_extra_args, geometry="Minkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, From 2d3ce81fe89f9ed7fa7134fb884d5b14cae41076 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Tue, 28 Jan 2025 21:30:46 +0000 Subject: [PATCH 29/48] update golds --- tst/regression/homogeneous_sphere.py | 3 +- tst/regression/linear_modes.gold | 2458 +++++++++++------------ tst/regression/mocmc_diffusion.gold | 50 +- tst/regression/mocmc_equilibration.gold | 32 +- 4 files changed, 1272 insertions(+), 1271 deletions(-) diff --git a/tst/regression/homogeneous_sphere.py b/tst/regression/homogeneous_sphere.py index c4f91762d..30633ea5b 100755 --- a/tst/regression/homogeneous_sphere.py +++ b/tst/regression/homogeneous_sphere.py @@ -46,13 +46,14 @@ modified_inputs["radiation_advection/vx"] = 0.0 modified_inputs["radiation_advection/radius"] = 1.0 +cmake_extra_args = "-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off" code = rt.gold_comparison( variables=["r.p.J", "r.p.H"], input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, - cmake_extra_args="", + cmake_extra_args=cmake_extra_args, geometry="SphericalMinkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/linear_modes.gold b/tst/regression/linear_modes.gold index 571fe701f..56b52b9c9 100644 --- a/tst/regression/linear_modes.gold +++ b/tst/regression/linear_modes.gold @@ -1,1232 +1,1232 @@ -1.000005632655319499e+00 -1.000002760808383684e+00 -9.999972996949745285e-01 -9.999943565266583567e-01 -9.999962802371848980e-01 -1.000001649208487331e+00 -1.000005620483547686e+00 -1.000004150727892061e+00 -9.999989029694753562e-01 -9.999945052546052038e-01 -9.999951245617294537e-01 -9.999999666961703060e-01 -1.000004883968774960e+00 -1.000005153020244952e+00 -1.000000599734518048e+00 -9.999955005058174251e-01 -9.999944869738877085e-01 -9.999982906507698388e-01 -1.000003652558603573e+00 -1.000005628965033422e+00 -1.000002244374423199e+00 -9.999968100084866141e-01 -9.999943649241527233e-01 -9.999967457378612368e-01 -1.000002185287540257e+00 -1.000005642989404420e+00 -1.000003719999225105e+00 -9.999983506855543158e-01 -9.999943790434682178e-01 -9.999954615166134486e-01 -1.000000534120877971e+00 -1.000005228450156824e+00 -1.000004875483002431e+00 -1.000000033345461725e+00 -9.999951164123052072e-01 -9.999946323421767813e-01 -9.999988401746482403e-01 -1.000004088373919187e+00 -1.000005589093612368e+00 -1.000001709466136646e+00 -9.999963473093869126e-01 -9.999943761293018651e-01 -9.999972393619038780e-01 -1.000002700093578278e+00 -1.000005644115180781e+00 -1.000003254457284108e+00 -9.999978146264839607e-01 -9.999943568631461943e-01 -9.999958495106286982e-01 -1.000001096971994130e+00 -1.000005495535843947e+00 -1.000004875450465347e+00 -9.999994658331410324e-01 -9.999947709284359165e-01 -9.999948465842898315e-01 -9.999994003330860970e-01 -1.000004499339977482e+00 -1.000005513960985581e+00 -1.000001159938610495e+00 -9.999959108496043747e-01 -9.999944056627527722e-01 -9.999977557711768883e-01 -1.000003189886431443e+00 -1.000005636202187853e+00 -1.000002760808383684e+00 -9.999972998744335317e-01 -9.999943559924534542e-01 -9.999962802054666033e-01 -1.000001649278303040e+00 -1.000005620628910519e+00 -1.000004538623388450e+00 -9.999989029677013308e-01 -9.999945028654575463e-01 -9.999951245676423905e-01 -9.999999667078377508e-01 -1.000004883749974649e+00 -1.000005367520260258e+00 -1.000000599734518048e+00 -9.999955009783095816e-01 -9.999944860242014899e-01 -9.999982906563524843e-01 -1.000003652592456493e+00 -1.000005626612752385e+00 -1.000002760794253653e+00 -9.999968099938674193e-01 -9.999943548367293822e-01 -9.999967457329563825e-01 -1.000002185325947535e+00 -1.000005642809836504e+00 -1.000004150697972882e+00 -9.999983506855543158e-01 -9.999943783760981697e-01 -9.999954615040358430e-01 -1.000000534158342669e+00 -1.000005228311358962e+00 -1.000005153536246860e+00 -1.000000033271464250e+00 -9.999951155518908008e-01 -9.999946330534588146e-01 -9.999988401823904915e-01 -1.000004088583912543e+00 -1.000005588376624566e+00 -1.000002244346593239e+00 -9.999963473093869126e-01 -9.999943743511141570e-01 -9.999972393620715216e-01 -1.000002700174408510e+00 -1.000005643771175512e+00 -1.000003719963573623e+00 -9.999978144589464213e-01 -9.999943569558090717e-01 -9.999958495208238762e-01 -1.000001096972067183e+00 -1.000005495200696481e+00 -1.000004875516921521e+00 -1.000000033248674702e+00 -9.999947709284359165e-01 -9.999948467812824759e-01 -9.999994003578855928e-01 -1.000004499543848624e+00 -1.000005513648303479e+00 -1.000001709420340612e+00 -9.999959110738979584e-01 -9.999944112094069082e-01 -9.999977557586826604e-01 -1.000003189792319835e+00 -1.000005635582378538e+00 -1.000003254426833799e+00 -9.999972996949745285e-01 -9.999943559924534542e-01 -9.999962802051375332e-01 -1.000001649284675054e+00 -1.000005621498109232e+00 -1.000004538640674401e+00 -9.999994658109249146e-01 -9.999945035818743611e-01 -9.999951246263967253e-01 -9.999999666998068415e-01 -1.000004883316716331e+00 -1.000005367234137355e+00 -1.000001159884960744e+00 -9.999955005058174251e-01 -9.999944860242014899e-01 -9.999982906307588459e-01 -1.000003652605387927e+00 -1.000005624334296517e+00 -1.000002760792013445e+00 -9.999972997383014262e-01 -9.999943614488633337e-01 -9.999967457341174537e-01 -1.000002185393187970e+00 -1.000005642143994233e+00 -1.000004150708544870e+00 -9.999989029061402412e-01 -9.999943790434682178e-01 -9.999954615040358430e-01 -1.000000534081493475e+00 -1.000005228295131277e+00 -1.000005153196704244e+00 -1.000000599682605351e+00 -9.999951170624205021e-01 -9.999946333588105585e-01 -9.999988401642738722e-01 -1.000004088706652361e+00 -1.000005595181765905e+00 -1.000002244379371907e+00 -9.999968101646514729e-01 -9.999943761293018651e-01 -9.999972393620715216e-01 -1.000002700108178377e+00 -1.000005642882210610e+00 -1.000003720001498175e+00 -9.999983506322962512e-01 -9.999943596158442638e-01 -9.999958494945223597e-01 -1.000001096905887232e+00 -1.000005495644664899e+00 -1.000004875475132726e+00 -1.000000033312556269e+00 -9.999951170408881707e-01 -9.999948465842898315e-01 -9.999994003578855928e-01 -1.000004499522637147e+00 -1.000005512885039138e+00 -1.000001709484434675e+00 -9.999963472312672907e-01 -9.999944020628815933e-01 -9.999977557613660695e-01 -1.000003189716275331e+00 -1.000005639801760493e+00 -1.000003254451671930e+00 -9.999978145217349734e-01 -9.999943565266583567e-01 -9.999962802054666033e-01 -1.000001649284675054e+00 -1.000005620784933713e+00 -1.000004538678776589e+00 -9.999994658770350320e-01 -9.999947714133969878e-01 -9.999951245978127012e-01 -9.999999667342928111e-01 -1.000004883138554179e+00 -1.000005367418171476e+00 -1.000001159934313488e+00 -9.999959109807341484e-01 -9.999944869738877085e-01 -9.999982906563524843e-01 -1.000003652605387927e+00 -1.000005619484136021e+00 -1.000002760804144186e+00 -9.999972997261444840e-01 -9.999943567613929218e-01 -9.999967457339060672e-01 -1.000002185403360278e+00 -1.000005642411317064e+00 -1.000004150689643767e+00 -9.999989030539646606e-01 -9.999945040795561457e-01 -9.999954615166134486e-01 -1.000000534158342669e+00 -1.000005228295131277e+00 -1.000005153882767006e+00 -1.000000599705133553e+00 -9.999955003056808511e-01 -9.999946334501745859e-01 -9.999988401734757337e-01 -1.000004088639646405e+00 -1.000005593670743709e+00 -1.000002244387094841e+00 -9.999968100584690767e-01 -9.999943599343450451e-01 -9.999972393619038780e-01 -1.000002700174408510e+00 -1.000005642882210610e+00 -1.000003719982992090e+00 -9.999983506613785433e-01 -9.999943787217002722e-01 -9.999958495178359330e-01 -1.000001096954051372e+00 -1.000005496312835085e+00 -1.000004875478968547e+00 -1.000000033287579804e+00 -9.999951168764663612e-01 -9.999946331056811522e-01 -9.999994003330860970e-01 -1.000004499543848624e+00 -1.000005512885039138e+00 -1.000001709477368106e+00 -9.999963473939638137e-01 -9.999943780502227320e-01 -9.999977557667091288e-01 -1.000003189830621864e+00 -1.000005637416242932e+00 -1.000003254456625301e+00 -9.999978144411570957e-01 -9.999943589123083676e-01 -9.999962802371848980e-01 -1.000001649278303040e+00 -1.000005621498109232e+00 -1.000004538678776589e+00 -9.999994658622803900e-01 -9.999947722049712384e-01 -9.999948463156687062e-01 -9.999999666883998550e-01 -1.000004883059735450e+00 -1.000005366258192918e+00 -1.000001159913265214e+00 -9.999959109662573953e-01 -9.999944021225303237e-01 -9.999982906507698388e-01 -1.000003652592456493e+00 -1.000005624334296517e+00 -1.000002760804144186e+00 -9.999972997290592636e-01 -9.999943563340063513e-01 -9.999962802322818201e-01 -1.000002185330710391e+00 -1.000005640836852283e+00 -1.000004150751921728e+00 -9.999989029865307133e-01 -9.999945034238885144e-01 -9.999951246482331468e-01 -1.000000534120877971e+00 -1.000005228311358962e+00 -1.000005153196704244e+00 -1.000000599705133553e+00 -9.999955002942768623e-01 -9.999944865621374079e-01 -9.999988401662210924e-01 -1.000004088660306545e+00 -1.000005597738118590e+00 -1.000002244387627748e+00 -9.999968100301691587e-01 -9.999943593569411426e-01 -9.999967457255370951e-01 -1.000002700093578278e+00 -1.000005643771175512e+00 -1.000003720001498175e+00 -9.999983506613785433e-01 -9.999943800857316045e-01 -9.999954615287282023e-01 -1.000001096930779321e+00 -1.000005496538477034e+00 -1.000004875481131261e+00 -1.000000033332056004e+00 -9.999951162199782750e-01 -9.999946336329671448e-01 -9.999988401593464804e-01 -1.000004499339977482e+00 -1.000005513648303479e+00 -1.000001709484434675e+00 -9.999963473939638137e-01 -9.999943649475041552e-01 -9.999972393553445693e-01 -1.000003189733285724e+00 -1.000005644760634915e+00 -1.000003254444560064e+00 -9.999978145439466504e-01 -9.999943571569811507e-01 -9.999958494474100457e-01 -1.000001649208487331e+00 -1.000005620628910519e+00 -1.000004538640674401e+00 -9.999994658770350320e-01 -9.999947722049712384e-01 -9.999948468252387590e-01 -9.999994003217230754e-01 -1.000004883005678025e+00 -1.000005366956320030e+00 -1.000001159915184790e+00 -9.999959114697550833e-01 -9.999944071037120130e-01 -9.999977557691764884e-01 -1.000003652558603573e+00 -1.000005626612752385e+00 -1.000002760792013445e+00 -9.999972997261444840e-01 -9.999943563340063513e-01 -9.999962801845050375e-01 -1.000001649218661415e+00 -1.000005641543503909e+00 -1.000004150665656066e+00 -9.999989029996332324e-01 -9.999945047613947313e-01 -9.999951247846718960e-01 -9.999999667155246019e-01 -1.000005228450156824e+00 -1.000005153536246860e+00 -1.000000599682605351e+00 -9.999955003056808511e-01 -9.999944865621374079e-01 -9.999982906611800670e-01 -1.000004088755765075e+00 -1.000005598271136220e+00 -1.000002244348032532e+00 -9.999968100916505342e-01 -9.999943640835056158e-01 -9.999967457235865442e-01 -1.000002185347521610e+00 -1.000005644115180781e+00 -1.000003719963573623e+00 -9.999983506322962512e-01 -9.999943787217002722e-01 -9.999954615287282023e-01 -1.000000534153414167e+00 -1.000005496702192076e+00 -1.000004875448733621e+00 -1.000000033272859801e+00 -9.999951163138114385e-01 -9.999946326145957576e-01 -9.999988401736024102e-01 -1.000004088469371721e+00 -1.000005513960985581e+00 -1.000001709420340612e+00 -9.999963472312672907e-01 -9.999943780502227320e-01 -9.999972393553445693e-01 -1.000002700212754503e+00 -1.000005645330762860e+00 -1.000003254434076005e+00 -9.999978145630771253e-01 -9.999943576993874927e-01 -9.999958495086429533e-01 -1.000001096954989510e+00 -1.000005620483547686e+00 -1.000004538623388450e+00 -9.999994658109249146e-01 -9.999947714133969878e-01 -9.999948463156687062e-01 -9.999994003217230754e-01 -1.000004499638560196e+00 -1.000005366980012411e+00 -1.000001159875671508e+00 -9.999959114989783737e-01 -9.999944078390725632e-01 -9.999977557599983857e-01 -1.000003189869540288e+00 -1.000005628965033422e+00 -1.000002760794253653e+00 -9.999972997383014262e-01 -9.999943567613929218e-01 -9.999962802322818201e-01 -1.000001649218661415e+00 -1.000005620196233735e+00 -1.000004150703476480e+00 -9.999989029721221279e-01 -9.999945044132163563e-01 -9.999951245066918126e-01 -9.999999666814920474e-01 -1.000004883573349268e+00 -1.590367539041457537e-06 -1.850912408548692162e-07 --1.388671263919384155e-06 --1.701402812589428375e-06 --5.275245537920439392e-07 -1.127207486787609383e-06 -1.725513118501249725e-06 -6.926330706003840947e-07 --9.843926778009791954e-07 --1.741847553483955041e-06 --1.004286466214067983e-06 -6.744469082427657453e-07 -1.741808948583900944e-06 -1.148018083313236715e-06 --5.089801355624572556e-07 --1.734300937657732912e-06 --1.400619844916229366e-06 -1.648243680517588008e-07 -1.613372567611652862e-06 -1.504581194016223573e-06 -1.027057519218336824e-08 --1.506965207472381200e-06 --1.656413222052862091e-06 --3.579387071900327507e-07 -1.261832909686356920e-06 -1.726530510927856447e-06 -5.275505036001899248e-07 --1.127283476069760551e-06 --1.738991200931393030e-06 --8.519558241342447753e-07 -8.332683681776776070e-07 -1.741543189546330788e-06 -1.004348184969015373e-06 --6.744859534995672576e-07 --1.741128142201565000e-06 --1.280871119453824414e-06 -3.385081253452617528e-07 -1.695890104207147870e-06 -1.400661979984600656e-06 --1.648645316816077902e-07 --1.613580376379654210e-06 --1.590413693388966491e-06 --1.850598198757230344e-07 -1.388569531686582505e-06 -1.701594467880254092e-06 -3.579597281925138896e-07 --1.262011462870778688e-06 --1.727044134608643941e-06 --6.925960486877090200e-07 -9.843413329179845028e-07 -1.741951384656060736e-06 -8.520003192028401959e-07 --8.334344866947607621e-07 --1.741882352976100373e-06 --1.147954552841384033e-06 -5.089651065550712443e-07 -1.734499890489319603e-06 -1.400657029138366716e-06 --3.385102729909338517e-07 --1.696068296688998337e-06 --1.504578035365303085e-06 --1.027792459486570175e-08 -1.506889434950186658e-06 -1.656390619447936956e-06 -1.850579395021460447e-07 --1.388808107645545854e-06 --1.701449497581291826e-06 --5.275302993430160891e-07 -1.127129333673650356e-06 -1.739139556716176440e-06 -6.926275968399265429e-07 --9.843286574677046199e-07 --1.738833342797969636e-06 --1.004290676010142285e-06 -6.744290190047559574e-07 -1.741608192516847482e-06 -1.280938630120717270e-06 --5.089962051112259242e-07 --1.734595962635921047e-06 --1.400613937252086356e-06 -1.648187793660271902e-07 -1.613277429219103755e-06 -1.590372361279220448e-06 -1.026314296979694244e-08 --1.507199462265586231e-06 --1.656498548720226997e-06 --3.579462891905561392e-07 -1.261786933656091673e-06 -1.727036888705413617e-06 -6.926335290490646140e-07 --1.127305826552776190e-06 --1.734666481337623469e-06 --8.519554876257128211e-07 -8.332817916134800692e-07 -1.741570517560293097e-06 -1.148022411434485064e-06 --6.745069916869999551e-07 --1.740893134080086125e-06 --1.280863526478345686e-06 -3.385007361086511826e-07 -1.695925012644290348e-06 -1.504607413513912528e-06 --1.648357971565737352e-07 --1.613293906584121117e-06 --1.590371049557741721e-06 --1.850698468576710981e-07 -1.388584273480631005e-06 -1.701619351222137502e-06 -5.275664409881972963e-07 --1.261890835872668019e-06 --1.726952582184757882e-06 --6.925939866257344096e-07 -9.843387710206114620e-07 -1.741827775922222212e-06 -1.004355416021173411e-06 --8.332887313101556406e-07 --1.741493618313671237e-06 --1.147954127011784552e-06 -5.089536504049958006e-07 -1.734405951374010952e-06 -1.400665060755398470e-06 --1.648147270693007677e-07 --1.696026167294465809e-06 --1.504516816255540277e-06 --1.028570061164219897e-08 -1.506966518059593660e-06 -1.656403618202785754e-06 -3.579714705349063427e-07 --1.388536818637949710e-06 --1.701761632405453606e-06 --5.275227261590045916e-07 -1.127176760524699177e-06 -1.738899137111849145e-06 -8.520110766617004472e-07 --9.843379307777278582e-07 --1.741806689421038490e-06 --1.004295364866518706e-06 -6.743993238623917024e-07 -1.741251734869187301e-06 -1.280939977094745456e-06 --3.385363576962681561e-07 --1.734477658715999417e-06 --1.400598327554048858e-06 -1.648342583341889763e-07 -1.613392098859980280e-06 -1.590351309578219981e-06 -1.850793076461093879e-07 --1.506990158082079201e-06 --1.656348914111078322e-06 --3.579349890952416767e-07 -1.261748446896396022e-06 -1.726954622768805722e-06 -6.926407633502231383e-07 --9.843909964545581157e-07 --1.737771171945459064e-06 --8.519585978329506073e-07 -8.332596957649066877e-07 -1.741883632174271865e-06 -1.148021789733716254e-06 --5.089646463921921279e-07 --1.741108470592745993e-06 --1.280864836161371862e-06 -3.385379663369058124e-07 -1.696309625747959570e-06 -1.504609226014816693e-06 -1.029970426659359921e-08 --1.613350527324997949e-06 --1.590302724258083714e-06 --1.850619383300565058e-07 -1.388450251314567264e-06 -1.701199448021600421e-06 -5.275580778973880034e-07 --1.127288025163849908e-06 --1.726563348516592840e-06 --6.925950592092194851e-07 -9.843972034917504489e-07 -1.741986631626964369e-06 -1.004347296229844827e-06 --6.744476113853419399e-07 --1.741626188449242454e-06 --1.147951644189643252e-06 -5.089802152540511556e-07 -1.734464186140258252e-06 -1.400680181652775881e-06 --1.648386383732958587e-07 --1.613458033031303169e-06 --1.504577710435353542e-06 --1.028608283527124951e-08 -1.506928969757272522e-06 -1.656256499901239233e-06 -3.579525644479939614e-07 --1.261872342220463811e-06 --1.701610355631484363e-06 --5.275271344634275999e-07 -1.127198902199723192e-06 -1.738062302039671397e-06 -8.520017892937290931e-07 --8.333539228821495243e-07 --1.739065206108945372e-06 --1.004291260290182602e-06 -6.744101958019806226e-07 -1.740803442299043410e-06 -1.280937036840117323e-06 --3.385636630176459017e-07 --1.696207774224466475e-06 --1.400602580196387577e-06 -1.648167040171728496e-07 -1.613295641558825584e-06 -1.590472857050408798e-06 -1.850645034035481419e-07 --1.388546378061128693e-06 --1.656265722646537589e-06 --3.579423946587949524e-07 -1.261757825222560384e-06 -1.727582033192779542e-06 -6.926310400115253552e-07 --9.844404762875896280e-07 --1.740739744440883911e-06 --8.519585516470582605e-07 -8.332593986167771238e-07 -1.742379863835736858e-06 -1.148012931887122658e-06 --5.090402292842649397e-07 --1.734407858537690999e-06 --1.280868972595230153e-06 -3.384999746174194464e-07 -1.696185029077676697e-06 -1.504623543518964511e-06 -1.028223359323060603e-08 --1.507075600978145329e-06 --1.590475652061203466e-06 --1.850639222710864662e-07 -1.388557421002558836e-06 -1.701684251364760647e-06 -5.275512265607479172e-07 --1.127286411336689808e-06 --1.737395596505640334e-06 --6.925970737149970458e-07 -9.843206542173732238e-07 -1.741832499020683313e-06 -1.004348077440545941e-06 --6.744556533661682400e-07 --1.740774003594915025e-06 --1.147947685019042703e-06 -5.089704719231065102e-07 -1.734460067992656717e-06 -1.400663872155841370e-06 --1.648378112527584822e-07 --1.613128151027974754e-06 --1.590478454220823225e-06 --1.027735954261626783e-08 -1.506985638456615903e-06 -1.656094988251150500e-06 -3.579615998315412844e-07 --1.261784008739208135e-06 --1.726486447474969463e-06 --5.275304255328125951e-07 -1.127163313841367346e-06 -1.738319230226277259e-06 -8.520030490968967593e-07 --8.333047458639765553e-07 --1.741716031385066482e-06 --1.004287515401397678e-06 -6.744269820492903251e-07 -1.741063303667368066e-06 -1.280944197266247243e-06 --3.385375740156370755e-07 --1.696114681665394089e-06 --1.504607320341262842e-06 -1.648247549435342491e-07 -1.613279411811395556e-06 -1.590272262240389021e-06 -1.850763445089354181e-07 --1.388543432206800262e-06 --1.701579535502788515e-06 --3.579417799535259882e-07 -1.261825112859412937e-06 -1.726522186015714916e-06 -6.926298797425628815e-07 --9.844451534597710166e-07 --1.739154462062785886e-06 --1.004287950956870439e-06 -8.332800073558788083e-07 -1.741705419103144528e-06 -1.148018190478056903e-06 --5.089998106922283231e-07 --1.734196579128229177e-06 --1.400607469785205336e-06 -3.385147902400103756e-07 -1.696188697615942383e-06 -1.504598469109849392e-06 -1.028602692912153095e-08 --1.507149469115720502e-06 --1.656302740691107855e-06 --1.850615676620913606e-07 -1.388597063993844476e-06 -1.701598914005255437e-06 -5.275488683394952047e-07 --1.127246230087635691e-06 --1.735630160281970369e-06 --8.519545802429056960e-07 -9.843770273985061775e-07 -1.738994245936507767e-06 -1.004344838744613408e-06 --6.744224451044774395e-07 --1.741648171681971972e-06 --1.280885326922608207e-06 -5.089825455281064240e-07 -1.734625715760534243e-06 -1.400662582495071480e-06 --1.648427259867010545e-07 --1.613604219551383351e-06 --1.590309690913316708e-06 --1.850585260142685255e-07 -1.507093017892425353e-06 -1.656433997566742238e-06 -3.579501898796839463e-07 --1.261749084508135586e-06 --1.724793762976749918e-06 --6.925917484604561872e-07 -1.127192347382897711e-06 -1.734482175110829926e-06 -8.520028454191943851e-07 --8.332712925714974420e-07 --1.741297315315238620e-06 --1.147961367548260921e-06 -6.744510731501280489e-07 -1.741062871064385555e-06 -1.280935962263143594e-06 --3.385459381808079238e-07 --1.695877980024644523e-06 --1.504538951390613589e-06 --1.027454453753881087e-08 -1.613311494828473454e-06 -1.590490523396869758e-06 -1.850695136122860865e-07 --1.388564292089949174e-06 --1.701054595687863222e-06 --5.275258794933226751e-07 -1.261802895159962122e-06 -1.725896452052280683e-06 -6.926385417940661657e-07 --9.843876064109259510e-07 --1.742343816791561549e-06 --1.004292673205916525e-06 -6.744367176308645790e-07 -1.741774388432576809e-06 -1.148021657984709214e-06 --5.090048672716291640e-07 --1.734424760616408426e-06 --1.400604165194647486e-06 -1.648261974188203075e-07 -1.696104137809677338e-06 -1.504611812673658253e-06 -1.028752509823473208e-08 --1.506835049173077311e-06 --1.656367124297569766e-06 --3.579465736156574359e-07 -1.388594163091088900e-06 -1.701600197062117496e-06 -5.275662193295627331e-07 --1.127189509723428754e-06 --1.739140251596924240e-06 --8.519555146344949567e-07 -8.332803391730492215e-07 -1.739083119561703616e-06 -1.004355888144604765e-06 --6.744274295560516419e-07 --1.741157556063786170e-06 --1.280874370092632454e-06 -3.384979849638089985e-07 -1.734512488043795267e-06 -1.400653876346155224e-06 --1.648191437038570666e-07 --1.613357878702286843e-06 --1.590415680388212322e-06 --1.850664008729975969e-07 -1.388616654257874299e-06 -1.656410100155959239e-06 -3.579718078982790880e-07 --1.261742194831940182e-06 --1.727519293799483035e-06 --6.925931453648272117e-07 -9.842945020186080504e-07 -1.734544076054722863e-06 -8.520144031590257785e-07 --8.332964747435799942e-07 --1.742269274102159542e-06 --1.147947761448823896e-06 -5.089688992861910061e-07 -1.741034279003410450e-06 -1.280937689492520769e-06 --3.385423636830984105e-07 --1.696046231791591712e-06 --1.504618301607008335e-06 --1.027719594533354700e-08 -1.506805351404585447e-06 -1.590436401080494455e-06 -1.851002823797434524e-07 --1.388659116484511387e-06 --1.701540621950765053e-06 --5.275256108826528761e-07 -1.127166171203310547e-06 -1.725110778275020941e-06 -6.926393770930866481e-07 --9.844323146218771484e-07 --1.741784308782007532e-06 --1.004286311019523964e-06 -6.744500208297343382e-07 -1.741938329515638949e-06 -1.148014876225550924e-06 --5.089785309167015760e-07 --1.734497322243950889e-06 --1.400614927829939636e-06 -1.648214631468875859e-07 -1.613184576308203552e-06 -1.504606788480569668e-06 -1.027859207800901093e-08 --1.507276578279591926e-06 --1.656275217839760236e-06 --3.579405943675265619e-07 -1.261816236083089866e-06 -1.700849706196831590e-06 -5.275493876965313214e-07 --1.127198411639394720e-06 --1.734249179605773024e-06 --8.519561298198576426e-07 -8.332521341154560291e-07 -1.741764638932950718e-06 -1.004347431623458245e-06 --6.744596310186115941e-07 --1.741748314809797923e-06 --1.280869111820689546e-06 -3.385171037921494635e-07 -1.695901592563524869e-06 -1.400678617835233387e-06 --1.648565488028164046e-07 --1.613561807138776761e-06 --1.590478919667941100e-06 --1.850638870732223280e-07 -1.388590675464703539e-06 -1.701433691138633940e-06 -3.579618944853186432e-07 --1.261779512233267517e-06 --1.726504418213852149e-06 --6.925968883290548120e-07 -9.843697821249760282e-07 -1.741881945693986421e-06 -8.520030971635724024e-07 --8.333062008962849171e-07 --1.742331654135019675e-06 --1.147952122313779147e-06 -5.089734434596512497e-07 -1.734662045847583257e-06 -1.280942301929910373e-06 --3.384895853315081937e-07 --1.696161302241622052e-06 --1.504564104073877981e-06 --1.027659593009093002e-08 -1.507009727967658865e-06 -1.656445198085565982e-06 -1.850641972503618116e-07 --1.388693950052137218e-06 --1.701040840053128679e-06 --5.275290161123662693e-07 -1.127187285536431175e-06 -1.734723813921674311e-06 -6.926307761678843503e-07 --9.843077859588004430e-07 --1.739164557000898997e-06 --1.004292273601492378e-06 -6.744365888437088794e-07 -1.742010130835613369e-06 -1.280938872272856700e-06 --5.090256218436111423e-07 --1.734564342862144349e-06 --1.400616732860756689e-06 -1.648170136371230491e-07 -1.613363813253878648e-06 -1.590438272052297480e-06 -1.025305568602594536e-08 --1.506926596493520195e-06 --1.656431298117337317e-06 --3.579410847930974386e-07 -1.261875034073571594e-06 -1.726397781239574585e-06 -5.275604617026915171e-07 --1.127137339160950026e-06 --1.739180713235708337e-06 --8.519550466243470749e-07 -8.332820516221241366e-07 -1.741974380368915069e-06 -1.148021876526727446e-06 --6.744862491138618203e-07 --1.741113204422817864e-06 --1.280873208255707068e-06 -3.385022989464865713e-07 -1.695972778829141366e-06 -1.504589322247046242e-06 --1.648193240548547792e-07 --1.613117730023353339e-06 --1.590520135504460998e-06 --1.850727975776422814e-07 -1.388654221902107873e-06 -1.701326020156905512e-06 -5.275682709626586915e-07 --1.261947505715531078e-06 --1.726078486393322259e-06 --6.925948186026352393e-07 -9.843279579363112002e-07 -1.741660351150165014e-06 -1.004355483758935434e-06 --8.333099001568666338e-07 --1.742964785873333881e-06 --1.147945751530698009e-06 -5.089534599001360521e-07 -1.734599858759064124e-06 -1.400664681976267818e-06 --3.385016663087549197e-07 --1.695870676087512766e-06 --1.504554066993212823e-06 --1.028948149217973521e-08 -1.507067109023507444e-06 -1.656464995976781995e-06 -3.579755870487922869e-07 --1.388613154987747987e-06 --1.701443479170704020e-06 --5.275255413453798391e-07 -1.127166071302384452e-06 -1.734503200677021406e-06 -8.520142368580808288e-07 --9.844001539690870857e-07 --1.741656127705483338e-06 --1.004294452983678689e-06 -6.744018012792296245e-07 -1.741914613915466185e-06 -1.280937446172547368e-06 --3.385295895269946168e-07 --1.734981101659022615e-06 --1.400588052416376249e-06 -1.648316694002071529e-07 -1.613486565481482708e-06 -1.590483001996038449e-06 -1.850813389808183833e-07 --1.507046180103455067e-06 --1.656478180211654670e-06 --3.579349198709450374e-07 -1.261721756539615536e-06 -1.726441261398884717e-06 -6.926417775169320062e-07 --1.127277146577362529e-06 --1.735176888438899801e-06 --8.519584793928184132e-07 -8.332792681452140573e-07 -1.742157773790548184e-06 -1.148017886747695187e-06 --5.090103794232628682e-07 --1.741570079490782926e-06 --1.280866889089690358e-06 -3.385394777859105416e-07 -1.696098376610258425e-06 -1.504587084259393158e-06 -1.028785662284046328e-08 --1.613721909539123835e-06 --1.590494503503302450e-06 --1.850666926647154778e-07 -1.388542047507202778e-06 -1.701004739452484825e-06 -5.275550978710463273e-07 --1.127242889265855080e-06 --1.724557476696404319e-06 --6.925939616939340237e-07 -9.844457602245243228e-07 -1.739765484603756400e-06 -1.004345694661083607e-06 --6.744448456179310710e-07 --1.741858256020505802e-06 --1.147945817186612501e-06 -5.089811917854148751e-07 -1.734594781773205028e-06 -1.400685564424060619e-06 --1.648459173018137831e-07 --1.696301470419201054e-06 --1.504565791851446476e-06 --1.028660033549784524e-08 -1.507102629175416761e-06 -1.656165813660437616e-06 -3.579516650658177460e-07 --1.261767226410950790e-06 --1.701502509763890978e-06 --5.275255364334288959e-07 -1.127258241735445696e-06 -1.738397037242816289e-06 -8.519992627385563252e-07 --8.333155900301203234e-07 --1.739221311409126995e-06 --1.004289595998266989e-06 -6.743959648938684969e-07 -1.741615939271367849e-06 -1.280948672552883438e-06 --3.385524990879865896e-07 --1.696233800921703517e-06 --1.400615320269782821e-06 -1.648185908126635101e-07 -1.613437983733279720e-06 -1.590376068386415433e-06 -1.850698379245635073e-07 --1.388559203520277344e-06 --1.656422452523023210e-06 --3.579448456761001503e-07 -1.261793790873533917e-06 -1.724840191291488373e-06 -6.926291817612108503e-07 --9.844155398481135369e-07 --1.739034044670750390e-06 --8.519585025669481418e-07 -8.332434238338562824e-07 -1.742439124717288334e-06 -1.148015029905874473e-06 --5.090224365676433484e-07 --1.734886862324567029e-06 --1.280870799559112837e-06 -3.384997738350057584e-07 -1.695894495341171505e-06 -1.504610243716362610e-06 -1.028324945307817011e-08 --1.507023215201327417e-06 --1.590389675798358597e-06 --1.850716221977319632e-07 -1.388596165212837243e-06 -1.701872268799283530e-06 -5.275521592616450156e-07 --1.127213027036639802e-06 --1.738868422425264749e-06 --6.925967090994584708e-07 -9.843423265322836092e-07 -1.739658055559617132e-06 -1.004347820281614206e-06 --6.744326650540783592e-07 --1.741589486302994863e-06 --1.147951091273796405e-06 -5.089665116460267154e-07 -1.734732998372292519e-06 -1.400646831852552351e-06 --1.648303833899749357e-07 --1.613310940965321688e-06 --1.504616294783919238e-06 --1.028165214295071204e-08 -1.506992341721636064e-06 -1.656530616962355248e-06 -3.579615973366801713e-07 --1.261896119614707845e-06 --1.725099656955503595e-06 --5.275287952149862621e-07 -1.127169131548336438e-06 -1.736726332399427330e-06 -8.520031875083780306e-07 --8.332788361584564856e-07 --1.741739780205732736e-06 --1.004287541940401894e-06 -6.744270469032312699e-07 -1.741887255151474092e-06 -1.280932833180469980e-06 --3.385350963948087381e-07 --1.695992647258216778e-06 --1.504617839395090027e-06 -1.648217192121360978e-07 -1.613600533049282307e-06 -1.590344747451266808e-06 -1.850778758784153934e-07 --1.388497459901435881e-06 --1.701694326018869044e-06 --3.579403082721944544e-07 -1.261788553692148542e-06 -1.725249752752758465e-06 -6.926305704182332962e-07 --9.844267509989011877e-07 --1.739205736135453973e-06 --8.519543788221921723e-07 -8.332742280914557694e-07 -1.741817340405137131e-06 -1.148018306641354476e-06 --5.089990489542986116e-07 --1.734404740651791544e-06 --1.400611542120221849e-06 -3.385030299852301277e-07 -1.696213744038989187e-06 -1.504595461529449285e-06 -1.028736312899410621e-08 --1.507070054026904670e-06 --1.656396408560201997e-06 --1.850607566820000722e-07 -1.388605038353589448e-06 -1.701245856308883713e-06 -5.275505345612582854e-07 --1.127252558694745794e-06 --1.738070038224574988e-06 --8.519549505936305591e-07 -9.843313317470496922e-07 -1.739225041207011783e-06 -1.004346551016468279e-06 --6.744387716338965728e-07 --1.740866185716938826e-06 --1.280870248436358718e-06 -5.089666806394039543e-07 -1.734652445953194618e-06 -1.400686053030659741e-06 --1.648523921474952950e-07 --1.613681664011344824e-06 --1.590316014061567975e-06 --1.027950810415140155e-08 -1.506931218489344571e-06 -1.656346474519701670e-06 -3.579517781802106437e-07 --1.261829655859476321e-06 --1.727214126451728340e-06 --6.925930708928146570e-07 -1.127135099675787796e-06 -1.739042613578682417e-06 -8.520049857849844323e-07 --8.333550763716032250e-07 --1.741223931084412381e-06 --1.147960783092397317e-06 -6.744292292009189618e-07 -1.741077513878259454e-06 -1.280943363517365134e-06 --3.385752748583262396e-07 --1.696353572023426503e-06 --1.504547365475786380e-06 --1.027654991288985169e-08 -1.613222830497776154e-06 -1.590460028730889177e-06 -1.850729280492534397e-07 --1.388806688619792916e-06 --1.701267205312941893e-06 --5.275260701054195476e-07 -1.261773888119183541e-06 -1.725800301839937553e-06 -6.926372703249363807e-07 --9.844933858999205134e-07 --1.742084440336276215e-06 --1.004293243730852016e-06 -8.332837782442656153e-07 -1.741681429266680129e-06 -1.148018244108631514e-06 --5.090140825880851954e-07 --1.734648181794856410e-06 --1.400615529873207839e-06 -1.648194043834948003e-07 -1.696085223384804340e-06 -1.504625704785217146e-06 -1.029153370613536142e-08 --1.507118939351625321e-06 --1.655993897924089542e-06 --3.579469875844075512e-07 -1.388652301991538058e-06 -1.701558517642712207e-06 -5.275674631140396064e-07 --1.127395666019526808e-06 --1.734941680691810245e-06 --8.519556369383823392e-07 -8.332644240114245081e-07 -1.739256246116809773e-06 -1.004353996353650715e-06 --6.744530441635463773e-07 --1.742187448380313789e-06 --1.280886367995999871e-06 -3.384972043392220203e-07 -1.734506840627408643e-06 -1.400668240434210029e-06 --1.648250127578891211e-07 --1.613605146426856459e-06 --1.590345985028447486e-06 --1.850701461615776516e-07 -1.507153337590947545e-06 -1.656372251468303937e-06 -3.579702221657228762e-07 --1.261874832669292944e-06 --1.726027473690091609e-06 --6.925920677498453038e-07 -9.843190773326566283e-07 -1.739058466695084452e-06 -8.520157362474218902e-07 --8.333066511850163643e-07 --1.742433685415770509e-06 --1.147955603304021991e-06 -5.089720785437955275e-07 -1.741096844482296675e-06 -1.280939701917045734e-06 --3.385226925826214097e-07 --1.696133847163606545e-06 --1.504574588735058261e-06 --1.027584873402550068e-08 -1.506900793475349959e-06 -1.590413619220728538e-06 -1.850912530880386366e-07 --1.388642336697858351e-06 --1.701249147872202525e-06 --5.275242307718781953e-07 -1.127191767414049902e-06 -1.726028621147986852e-06 -6.926392242527476744e-07 --9.844647089534297558e-07 --1.739939757177752584e-06 --1.004288355418123485e-06 -6.744529207682478307e-07 -1.742380102408182492e-06 -1.148016912806621430e-06 --5.089940474249223597e-07 --1.734603309747968123e-06 --1.400616570006886963e-06 -1.648218615872110605e-07 -1.613295571530132512e-06 -1.504603161806901259e-06 -1.029429429793979145e-08 --1.506989540017813416e-06 --1.656398114490014354e-06 --3.579385752429690782e-07 -1.261835544753514274e-06 +1.000004666400361453e+00 +1.000002248011152339e+00 +9.999977404516279256e-01 +9.999953361638420635e-01 +9.999969712837875058e-01 +1.000001393321431120e+00 +1.000004592398310077e+00 +1.000003386213018208e+00 +9.999990615324131360e-01 +9.999955337770192187e-01 +9.999959715599420207e-01 +1.000000004744709070e+00 +1.000004023217759386e+00 +1.000004296599104503e+00 +1.000000465061607891e+00 +9.999962680697185302e-01 +9.999953748349462712e-01 +9.999986145838325458e-01 +1.000003047856125704e+00 +1.000004666399947117e+00 +1.000001825924000443e+00 +9.999973351656267218e-01 +9.999953333260460653e-01 +9.999973507013483198e-01 +1.000001834803263323e+00 +1.000004647713942374e+00 +1.000003028598724919e+00 +9.999986067435152615e-01 +9.999954076462023167e-01 +9.999962788503311550e-01 +1.000000474205735124e+00 +1.000004271376126797e+00 +1.000004028316689553e+00 +9.999999952551462679e-01 +9.999959768960008377e-01 +9.999954956180908683e-01 +9.999990698133630618e-01 +1.000003405033339510e+00 +1.000004659995234135e+00 +1.000001385351878991e+00 +9.999969522622452756e-01 +9.999953335994599124e-01 +9.999977518926730369e-01 +1.000002259646641800e+00 +1.000004663823589768e+00 +1.000002649190411930e+00 +9.999981652797611531e-01 +9.999953523026023916e-01 +9.999966136635332070e-01 +1.000000938512173088e+00 +1.000004466299231121e+00 +1.000004028316591409e+00 +9.999995258168943302e-01 +9.999957287239070736e-01 +9.999957032905217869e-01 +9.999995349155391677e-01 +1.000003732051085725e+00 +1.000004625133037894e+00 +1.000000930142094901e+00 +9.999965950888028976e-01 +9.999953399974228496e-01 +9.999981739944652226e-01 +1.000002664944579678e+00 +1.000004666686853616e+00 +1.000002248011124806e+00 +9.999977404516252610e-01 +9.999953361786149131e-01 +9.999969712837798452e-01 +1.000001393321430010e+00 +1.000004592398679337e+00 +1.000003721023358105e+00 +9.999990615324095833e-01 +9.999955337781045728e-01 +9.999959715597189769e-01 +1.000000004744709070e+00 +1.000004023217781368e+00 +1.000004504305695097e+00 +1.000000465061601895e+00 +9.999962680697194184e-01 +9.999953748341793291e-01 +9.999986145838234419e-01 +1.000003047856125260e+00 +1.000004666399358477e+00 +1.000002248011129691e+00 +9.999973351656280540e-01 +9.999953333195393812e-01 +9.999973507012623886e-01 +1.000001834803264211e+00 +1.000004647713058414e+00 +1.000003386212947820e+00 +9.999986067435119308e-01 +9.999954076461353702e-01 +9.999962788501154387e-01 +1.000000474205734013e+00 +1.000004271376119247e+00 +1.000004296598814957e+00 +9.999999952551494875e-01 +9.999959768959527651e-01 +9.999954956156673624e-01 +9.999990698133554012e-01 +1.000003405033344395e+00 +1.000004659995257006e+00 +1.000001825924003329e+00 +9.999969522622470519e-01 +9.999953336000279025e-01 +9.999977518926045361e-01 +1.000002259646642022e+00 +1.000004663827931184e+00 +1.000003028598684063e+00 +9.999981652797619303e-01 +9.999953522974022180e-01 +9.999966136631759372e-01 +1.000000938512173088e+00 +1.000004466299570849e+00 +1.000004028316538118e+00 +9.999999952551463789e-01 +9.999957287238498971e-01 +9.999957032903722398e-01 +9.999995349155340607e-01 +1.000003732051085059e+00 +1.000004625132793201e+00 +1.000001385351878769e+00 +9.999965950888151101e-01 +9.999953399980278101e-01 +9.999981739944291403e-01 +1.000002664944579234e+00 +1.000004666684458199e+00 +1.000002649190382398e+00 +9.999977404516281476e-01 +9.999953362018981773e-01 +9.999969712836818125e-01 +1.000001393321429566e+00 +1.000004592398526126e+00 +1.000003721023295711e+00 +9.999995258168982160e-01 +9.999955337806277766e-01 +9.999959715596077325e-01 +1.000000004744707960e+00 +1.000004023217778704e+00 +1.000004504304911279e+00 +1.000000930142098010e+00 +9.999962680697201955e-01 +9.999953748321477320e-01 +9.999986145838194451e-01 +1.000003047856125704e+00 +1.000004666399210818e+00 +1.000002248011114814e+00 +9.999977404516300350e-01 +9.999953332918282145e-01 +9.999973507012395180e-01 +1.000001834803263767e+00 +1.000004647712783967e+00 +1.000003386212862333e+00 +9.999990615324134691e-01 +9.999954076448548390e-01 +9.999962788498980570e-01 +1.000000474205733569e+00 +1.000004271376105924e+00 +1.000004296598671960e+00 +1.000000465061604116e+00 +9.999959768957805695e-01 +9.999954956105355786e-01 +9.999990698133564004e-01 +1.000003405033346837e+00 +1.000004659995051615e+00 +1.000001825923991783e+00 +9.999973351656272769e-01 +9.999953336051309316e-01 +9.999977518925733388e-01 +1.000002259646642244e+00 +1.000004663832038121e+00 +1.000003028598672739e+00 +9.999986067435135961e-01 +9.999953523000170152e-01 +9.999966136629715452e-01 +1.000000938512174642e+00 +1.000004466299900585e+00 +1.000004028316468618e+00 +9.999999952551471560e-01 +9.999959768958928130e-01 +9.999957032910258281e-01 +9.999995349155321733e-01 +1.000003732051085281e+00 +1.000004625132317360e+00 +1.000001385351880101e+00 +9.999969522622410567e-01 +9.999953399996251990e-01 +9.999981739943998305e-01 +1.000002664944579678e+00 +1.000004666682248633e+00 +1.000002649190380399e+00 +9.999981652797613751e-01 +9.999953361678434183e-01 +9.999969712837857294e-01 +1.000001393321430232e+00 +1.000004592398119119e+00 +1.000003721023209113e+00 +9.999995258168984380e-01 +9.999957287239132908e-01 +9.999959715603238264e-01 +1.000000004744704629e+00 +1.000004023217817339e+00 +1.000004504303755981e+00 +1.000000930142097788e+00 +9.999965950887820254e-01 +9.999953748355584482e-01 +9.999986145838228868e-01 +1.000003047856125926e+00 +1.000004666397982467e+00 +1.000002248011110373e+00 +9.999977404516288138e-01 +9.999953361605120605e-01 +9.999973507014099372e-01 +1.000001834803264211e+00 +1.000004647717442907e+00 +1.000003386212769074e+00 +9.999990615324144683e-01 +9.999955337764854235e-01 +9.999962788506364664e-01 +1.000000474205735790e+00 +1.000004271376145004e+00 +1.000004296598153486e+00 +1.000000465061604338e+00 +9.999962680697163098e-01 +9.999954956181416055e-01 +9.999990698133610634e-01 +1.000003405033335069e+00 +1.000004659994018219e+00 +1.000001825923998888e+00 +9.999973351656312737e-01 +9.999953333334294925e-01 +9.999977518926795872e-01 +1.000002259646640690e+00 +1.000004663816347117e+00 +1.000003028598642763e+00 +9.999986067435127080e-01 +9.999954076461047281e-01 +9.999966136634478309e-01 +1.000000938512173976e+00 +1.000004466298461736e+00 +1.000004028316343385e+00 +9.999999952551524851e-01 +9.999959768961173001e-01 +9.999954956188110700e-01 +9.999995349155367252e-01 +1.000003732051084615e+00 +1.000004625133427805e+00 +1.000001385351886984e+00 +9.999969522622440543e-01 +9.999953335954714362e-01 +9.999981739944518999e-01 +1.000002664944580122e+00 +1.000004666695629485e+00 +1.000002649190358417e+00 +9.999981652797603759e-01 +9.999953523091763552e-01 +9.999969712837353253e-01 +1.000001393321430454e+00 +1.000004592398613390e+00 +1.000003721023265291e+00 +9.999995258168957735e-01 +9.999957287239312764e-01 +9.999957032895571141e-01 +1.000000004744705739e+00 +1.000004023217897720e+00 +1.000004504306811981e+00 +1.000000930142101563e+00 +9.999965950887911292e-01 +9.999953399951342359e-01 +9.999986145838121177e-01 +1.000003047856125704e+00 +1.000004666395742259e+00 +1.000002248011120587e+00 +9.999977404516278146e-01 +9.999953361715600009e-01 +9.999969712837485369e-01 +1.000001834803264211e+00 +1.000004647717475992e+00 +1.000003386212974021e+00 +9.999990615324142462e-01 +9.999955337777335362e-01 +9.999959715592217080e-01 +1.000000474205733125e+00 +1.000004271376128795e+00 +1.000004296597990283e+00 +1.000000465061608335e+00 +9.999962680697170869e-01 +9.999953748338852311e-01 +9.999990698133466305e-01 +1.000003405033344395e+00 +1.000004659993012135e+00 +1.000001825924012655e+00 +9.999973351656316067e-01 +9.999953333197989513e-01 +9.999973507012055451e-01 +1.000002259646641578e+00 +1.000004663830784013e+00 +1.000003028598643873e+00 +9.999986067435108206e-01 +9.999954076455773722e-01 +9.999962788498614197e-01 +1.000000938512174198e+00 +1.000004466300216555e+00 +1.000004028316062055e+00 +9.999999952551527072e-01 +9.999959768960148265e-01 +9.999954956159711195e-01 +9.999990698133480738e-01 +1.000003732051082173e+00 +1.000004625131559965e+00 +1.000001385351883876e+00 +9.999969522622428331e-01 +9.999953335986523362e-01 +9.999977518925946551e-01 +1.000002664944577235e+00 +1.000004666674098930e+00 +1.000002649190306903e+00 +9.999981652797607090e-01 +9.999953523057741878e-01 +9.999966136632807423e-01 +1.000001393321430010e+00 +1.000004592398435976e+00 +1.000003721022906022e+00 +9.999995258168977719e-01 +9.999957287239165105e-01 +9.999957032900425036e-01 +9.999995349155347268e-01 +1.000004023217651250e+00 +1.000004504303320774e+00 +1.000000930142100897e+00 +9.999965950887975685e-01 +9.999953399963393830e-01 +9.999981739944453496e-01 +1.000003047856124594e+00 +1.000004666403346176e+00 +1.000002248011097494e+00 +9.999977404516284807e-01 +9.999953361761605430e-01 +9.999969712837305513e-01 +1.000001393321430676e+00 +1.000004647707787520e+00 +1.000003386212811485e+00 +9.999990615324095833e-01 +9.999955337783269504e-01 +9.999959715595423404e-01 +1.000000004744708182e+00 +1.000004271376105480e+00 +1.000004296599532827e+00 +1.000000465061610777e+00 +9.999962680697176420e-01 +9.999953748335558279e-01 +9.999986145838278828e-01 +1.000003405033346837e+00 +1.000004659997403511e+00 +1.000001825924002663e+00 +9.999973351656301634e-01 +9.999953333120960020e-01 +9.999973507012649421e-01 +1.000001834803264211e+00 +1.000004663829241025e+00 +1.000003028598694277e+00 +9.999986067435129300e-01 +9.999954076457778784e-01 +9.999962788499922040e-01 +1.000000474205734680e+00 +1.000004466299469152e+00 +1.000004028317039495e+00 +9.999999952551497095e-01 +9.999959768958892603e-01 +9.999954956152694585e-01 +9.999990698133587319e-01 +1.000003405033342174e+00 +1.000004625133476432e+00 +1.000001385351894312e+00 +9.999969522622431661e-01 +9.999953336026468076e-01 +9.999977518926256304e-01 +1.000002259646642688e+00 +1.000004666687664301e+00 +1.000002649190435244e+00 +9.999981652797605980e-01 +9.999953523010406409e-01 +9.999966136633081648e-01 +1.000000938512173532e+00 +1.000004592398902270e+00 +1.000003721023604131e+00 +9.999995258168995482e-01 +9.999957287239137349e-01 +9.999957032910183896e-01 +9.999995349155389457e-01 +1.000003732051084615e+00 +1.000004504305699982e+00 +1.000000930142109112e+00 +9.999965950887942379e-01 +9.999953399989767178e-01 +9.999981739944453496e-01 +1.000002664944578346e+00 +1.000004666399634923e+00 +1.000002248011125916e+00 +9.999977404516260382e-01 +9.999953361699313037e-01 +9.999969712837786240e-01 +1.000001393321428900e+00 +1.000004592398641368e+00 +1.000003386212862111e+00 +9.999990615324135801e-01 +9.999955337772126196e-01 +9.999959715601252075e-01 +1.000000004744708626e+00 +1.000004023217752946e+00 +-9.294055605829665049e-07 +-1.030291957496450776e-07 +8.046703979068646837e-07 +1.000549553077626191e-06 +3.019937159876685764e-07 +-6.562109180205328918e-07 +-1.006598455220162532e-06 +-3.971066828603020044e-07 +5.733964941424576805e-07 +1.007349474896367298e-06 +5.745175580737134868e-07 +-3.941875256883406693e-07 +-1.005080896372813576e-06 +-6.560460356071897101e-07 +2.988280090240379087e-07 +9.952472322291836494e-07 +8.048859651250844561e-07 +-1.001443213029248955e-07 +-9.232992404455057782e-07 +-8.711308617708640842e-07 +-1.404958832198173164e-09 +8.682932468611122998e-07 +9.747824518831289863e-07 +2.036114893984304378e-07 +-7.336134015618541773e-07 +-1.006598655095421562e-06 +-3.019790656009647984e-07 +6.561857221984301149e-07 +1.007380094339941091e-06 +4.881467828113725560e-07 +-4.859015582911342406e-07 +-1.007082731718890411e-06 +-5.744946763633820308e-07 +3.941703885595222443e-07 +1.005077803376744554e-06 +7.328936611983004574e-07 +-2.005485973645621850e-07 +-9.676535229571514956e-07 +-8.048589854925684981e-07 +1.001409230956606669e-07 +9.232768255384944365e-07 +9.294280201533293177e-07 +1.030357151124987955e-07 +-8.046964356243842865e-07 +-1.000543221413674344e-06 +-2.036006000799612629e-07 +7.335872418718312566e-07 +1.006598822150295398e-06 +3.971244869449107467e-07 +-5.734197853180035611e-07 +-1.007351720028704540e-06 +-4.881262794805896256e-07 +4.858809806757326190e-07 +1.007081825336202509e-06 +6.560708920619997719e-07 +-2.988410452247087619e-07 +-9.952571243814315649e-07 +-8.048589497438283035e-07 +2.005402049912768811e-07 +9.676356082333004592e-07 +8.711572815316465394e-07 +1.406631894765357807e-09 +-8.683180606476139550e-07 +-9.747665153267913961e-07 +-1.030291939310566975e-07 +8.046704067551571979e-07 +1.000548532352946841e-06 +3.019937204881005265e-07 +-6.562109176569878331e-07 +-1.007381271481910094e-06 +-3.971066810608646289e-07 +5.733964953263467370e-07 +1.007350482720288475e-06 +5.745175809304583819e-07 +-3.941875258143732426e-07 +-1.005080964527917202e-06 +-7.328675160914734301e-07 +2.988280095462563788e-07 +9.952476620641745090e-07 +8.048859608725118163e-07 +-1.001443209425435050e-07 +-9.232992367045987427e-07 +-9.294055980330326193e-07 +-1.404959517640701816e-09 +8.682932773388525434e-07 +9.747826957941115791e-07 +2.036114952335989597e-07 +-7.336134023099747463e-07 +-1.006598701388902975e-06 +-3.971066847334067353e-07 +6.561857226187680973e-07 +1.007380188439154189e-06 +4.881468076402621446e-07 +-4.859015583827707117e-07 +-1.007082837263389934e-06 +-6.560460353188880705e-07 +3.941703882469833392e-07 +1.005078004821177435e-06 +7.328938038143404117e-07 +-2.005485965280355433e-07 +-9.676536526718752940e-07 +-8.711308358705835906e-07 +1.001409227611797836e-07 +9.232768064746382478e-07 +9.294284726415213611e-07 +1.030357178216759599e-07 +-8.046964334422135385e-07 +-1.000543486380220574e-06 +-3.019790657874176612e-07 +7.335872359921279128e-07 +1.006599374420541780e-06 +3.971244997196466696e-07 +-5.734197851770628903e-07 +-1.007351405861018618e-06 +-5.744946709092609681e-07 +4.858809805013788277e-07 +1.007081014439225773e-06 +6.560709161344415048e-07 +-2.988410443891016538e-07 +-9.952569826471431678e-07 +-8.048589436051752351e-07 +1.001409222746548451e-07 +9.676348249515108273e-07 +8.711572942806880884e-07 +1.406631817966765624e-09 +-8.683180537623393543e-07 +-9.747664592654431377e-07 +-2.036006005066277991e-07 +8.046704244787415075e-07 +1.000547438048910773e-06 +3.019937219917811019e-07 +-6.562109182992073667e-07 +-1.007381223111862613e-06 +-4.881262767194982221e-07 +5.733964955760497205e-07 +1.007352598026047830e-06 +5.745175763699886305e-07 +-3.941875254487432124e-07 +-1.005080934378242984e-06 +-7.328674761556064905e-07 +2.005402049373490197e-07 +9.952490497271137940e-07 +8.048858539997143893e-07 +-1.001443202768902755e-07 +-9.232992286496215109e-07 +-9.294054590202618228e-07 +-1.030291953709523303e-07 +8.682933990494582503e-07 +9.747827016462158366e-07 +2.036114941945643957e-07 +-7.336134030057371382e-07 +-1.006598732942832460e-06 +-3.971066803716042206e-07 +5.733964963353193606e-07 +1.007381093907491412e-06 +4.881468083266627049e-07 +-4.859015574202071292e-07 +-1.007082892318401641e-06 +-6.560460319610472384e-07 +2.988280080283115568e-07 +1.005078475712284694e-06 +7.328939380294382444e-07 +-2.005485965233618537e-07 +-9.676537729434291982e-07 +-8.711308463507602519e-07 +-1.404958758512861952e-09 +9.232767224797481987e-07 +9.294295603992337567e-07 +1.030357181236277737e-07 +-8.046964303023859640e-07 +-1.000543566924423863e-06 +-3.019790665356147808e-07 +6.561857218223741678e-07 +1.006600252622787633e-06 +3.971245013732760583e-07 +-5.734197844482541431e-07 +-1.007351112727576302e-06 +-5.744946719570341593e-07 +3.941703888336498463e-07 +1.007081254426845351e-06 +6.560708973246380255e-07 +-2.988410449280466424e-07 +-9.952568583960853288e-07 +-8.048589446686297591e-07 +1.001409223708378053e-07 +9.232768318617425597e-07 +8.711570877529253580e-07 +1.406631382096227332e-09 +-8.683180591714414629e-07 +-9.747664209953362632e-07 +-2.036006006470662853e-07 +7.335872463039276591e-07 +1.000548577095291758e-06 +3.019937124766091721e-07 +-6.562109174690844793e-07 +-1.007381201659390136e-06 +-4.881262780308053861e-07 +4.858809794166910936e-07 +1.007349541517549313e-06 +5.745175345504756350e-07 +-3.941875254938029300e-07 +-1.005081017746366662e-06 +-7.328674576769726707e-07 +2.005402047787423386e-07 +9.676361943290861294e-07 +8.048857393434741411e-07 +-1.001443217229112800e-07 +-9.232991899817397058e-07 +-9.294053279097822050e-07 +-1.030291956228264048e-07 +8.046703946814131955e-07 +9.747825369079970173e-07 +2.036114859916464274e-07 +-7.336133983415780604e-07 +-1.006599312755879993e-06 +-3.971066810618459166e-07 +5.733964940734734114e-07 +1.007349051460694473e-06 +4.881467767607113878e-07 +-4.859015582501338225e-07 +-1.007082136642869904e-06 +-6.560460377401635139e-07 +2.988280091995979807e-07 +9.952468567128696503e-07 +7.328937091375577813e-07 +-2.005485978997841197e-07 +-9.676532399336291053e-07 +-8.711308755560668499e-07 +-1.404959259262984853e-09 +8.682931914722783436e-07 +9.294278784563814152e-07 +1.030357146174803797e-07 +-8.046964394762864135e-07 +-1.000542599308030562e-06 +-3.019790680866183458e-07 +6.561857225577792899e-07 +1.007379813889062744e-06 +3.971244933289400484e-07 +-5.734197853806958577e-07 +-1.007352364527693350e-06 +-5.744946714740483495e-07 +3.941703884418093874e-07 +1.005077455138730845e-06 +6.560709097079438769e-07 +-2.988410445870692943e-07 +-9.952575760200046695e-07 +-8.048588314377094271e-07 +1.001409232286070540e-07 +9.232769242132966126e-07 +9.294277527649889421e-07 +1.406632600900779972e-09 +-8.683181184598553366e-07 +-9.747664873476702331e-07 +-2.036006022580282950e-07 +7.335872480031750659e-07 +1.006597553859811437e-06 +3.019937247961385248e-07 +-6.562109174122158519e-07 +-1.007381570085975475e-06 +-4.881262837657042124e-07 +4.858809799792420963e-07 +1.007082840755948180e-06 +5.745175990700788054e-07 +-3.941875254435710704e-07 +-1.005081271373953455e-06 +-7.328675606231233482e-07 +2.005402042525645453e-07 +9.676359117096829464e-07 +8.711573206991186470e-07 +-1.001443197585308346e-07 +-9.232991430965190641e-07 +-9.294059393448317663e-07 +-1.030291967012917025e-07 +8.046704058239257876e-07 +1.000549640675806998e-06 +2.036114966551241155e-07 +-7.336133990798837410e-07 +-1.006599892045066235e-06 +-3.971066878600229781e-07 +5.733964951241903598e-07 +1.007350114803998982e-06 +5.745175943436896992e-07 +-4.859015579621007982e-07 +-1.007082106238151100e-06 +-6.560460369184712513e-07 +2.988280092777632399e-07 +9.952475812636651914e-07 +8.048861864825506316e-07 +-2.005485963684203371e-07 +-9.676536779881982506e-07 +-8.711308052841336562e-07 +-1.404959423938335499e-09 +8.682932579749920019e-07 +9.747826190325700268e-07 +1.030357164468299561e-07 +-8.046964274718452703e-07 +-1.000542868051341273e-06 +-3.019790651361441396e-07 +6.561857231891525666e-07 +1.007380232515156921e-06 +4.881468087301507669e-07 +-5.734197848813358422e-07 +-1.007350950294399677e-06 +-5.744946517783403105e-07 +3.941703877627450853e-07 +1.005077800626097384e-06 +7.328937381305914640e-07 +-2.988410443342520883e-07 +-9.952564819183330304e-07 +-8.048587592066878117e-07 +1.001409222760839353e-07 +9.232768647194232507e-07 +9.294283447542539941e-07 +1.030357170321422826e-07 +-8.683179910261983592e-07 +-9.747664841496742591e-07 +-2.036005970270483214e-07 +7.335872446343642043e-07 +1.006598370965810515e-06 +3.971244919329538360e-07 +-6.562109187731836047e-07 +-1.007380870163989117e-06 +-4.881262716706710956e-07 +4.858809802198892388e-07 +1.007082267466381430e-06 +6.560708945169878679e-07 +-3.941875255511650599e-07 +-1.005080570409457157e-06 +-7.328674796952083110e-07 +2.005402040854753321e-07 +9.676356970333445729e-07 +8.711572222901105394e-07 +1.406631048658445245e-09 +-9.232993319839070370e-07 +-9.294050849430872943e-07 +-1.030291957977054491e-07 +8.046704085642496373e-07 +1.000549175795806904e-06 +3.019937190892757479e-07 +-7.336134064288557290e-07 +-1.006597490885001862e-06 +-3.971066861696735465e-07 +5.733964954294078827e-07 +1.007350584890920456e-06 +5.745175783441651443e-07 +-3.941875256004935589e-07 +-1.007083653283044906e-06 +-6.560460501217274421e-07 +2.988280085134882174e-07 +9.952479636465961965e-07 +8.048860351608119245e-07 +-1.001443210646145978e-07 +-9.676537756389315584e-07 +-8.711309078986095250e-07 +-1.404960017817508730e-09 +8.682933158663948593e-07 +9.747825454252580444e-07 +2.036114933599784652e-07 +-8.046964350132957548e-07 +-1.000544261483735576e-06 +-3.019790716176576691e-07 +6.561857219740708865e-07 +1.007380514786610828e-06 +4.881467987234624526e-07 +-4.859015576518420529e-07 +-1.007351444534011420e-06 +-5.744946932721045515e-07 +3.941703882120694352e-07 +1.005078113667816480e-06 +7.328937513922214597e-07 +-2.005485968202532280e-07 +-9.952571063988636494e-07 +-8.048591533933414212e-07 +1.001409216825303091e-07 +9.232767563823591207e-07 +9.294286160761272853e-07 +1.030357163397571565e-07 +-8.046964356567483566e-07 +-9.747664445262941850e-07 +-2.036006033225554009e-07 +7.335872414428896429e-07 +1.006599692269014033e-06 +3.971244920932981258e-07 +-5.734197848051192882e-07 +-1.007381331126183156e-06 +-4.881262836231832957e-07 +4.858809798281510061e-07 +1.007081504753755236e-06 +6.560708926547023947e-07 +-2.988410448544338677e-07 +-1.005080931508151039e-06 +-7.328674526781186881e-07 +2.005402038085742265e-07 +9.676358278649439926e-07 +8.711571999531510123e-07 +1.406631014538256826e-09 +-8.683180467752300057e-07 +-9.294055321418282562e-07 +-1.030291960466442633e-07 +8.046703991799883436e-07 +1.000548531789185217e-06 +3.019937172054097811e-07 +-6.562109184404555105e-07 +-1.006598506884153856e-06 +-3.971066766370785196e-07 +5.733964943905043969e-07 +1.007349754892116208e-06 +5.745175589400944665e-07 +-3.941875256656648562e-07 +-1.005080896632132925e-06 +-2.000164600401336329e-06 +9.222447130126748175e-07 +3.027257120328875046e-06 +2.459197049475813440e-06 +-3.152100988591829596e-07 +-2.818282960320236686e-06 +-2.662240623918497307e-06 +5.177599967053031403e-09 +2.656525704776692750e-06 +2.975242484345936160e-06 +6.122169816257862666e-07 +-2.249847544735278521e-06 +-3.055508536200097404e-06 +-9.128700923248491102e-07 +2.014347162471758928e-06 +3.087115942545467270e-06 +1.483209004082876766e-06 +-1.494393881401377758e-06 +-3.083622528824594606e-06 +-1.748729129417257092e-06 +1.213820143808149941e-06 +3.068911928744286609e-06 +2.237352580735936842e-06 +-6.218751953906710184e-07 +-2.943681922858387149e-06 +-2.459117958842387412e-06 +3.151955302699433135e-07 +2.818222717423963391e-06 +2.838920677095309822e-06 +3.050739631878120801e-07 +-2.465265907912817365e-06 +-3.055508255721094174e-06 +-6.121897148371430959e-07 +2.249771371000582817e-06 +3.081780049479383614e-06 +1.204066102358552120e-06 +-1.761878267187738988e-06 +-3.087022490170425038e-06 +-1.483149266023190730e-06 +1.494332629405549339e-06 +3.083619667655742292e-06 +2.000237981411015913e-06 +-9.222855630942051851e-07 +-3.027284067541193195e-06 +-2.237274972999872001e-06 +6.218469306796444209e-07 +2.943636432669303082e-06 +2.662316940973670670e-06 +-5.177918822022096318e-09 +-2.656595776902332376e-06 +-2.975193911082368820e-06 +-3.050601395245967083e-07 +2.465190717755353298e-06 +3.055529748302201587e-06 +9.129096337423613593e-07 +-2.014420931548350863e-06 +-3.087117304726884433e-06 +-1.204015689111762063e-06 +1.761809630570721700e-06 +3.087021664989664050e-06 +1.748796496204983334e-06 +-1.213872066081707796e-06 +-3.068920138318293335e-06 +-2.237274958057157526e-06 +9.222447132503667680e-07 +3.027256901262132608e-06 +2.459197096410802667e-06 +-3.152100983541052070e-07 +-2.818282959911827469e-06 +-2.838852490409840987e-06 +5.177600505655263596e-09 +2.656525683350705087e-06 +2.975242007791355705e-06 +6.122169799780032052e-07 +-2.249847545124946288e-06 +-3.081776648797771915e-06 +-9.128700894731423337e-07 +2.014347162091129003e-06 +3.087117174540557254e-06 +1.483209017073808584e-06 +-1.494393881331721580e-06 +-3.083623171896271279e-06 +-2.000164572751261632e-06 +1.213820143878340174e-06 +3.068910229705429056e-06 +2.237352623523504629e-06 +-6.218751950443644567e-07 +-2.943681900171466007e-06 +-2.662240627546797242e-06 +3.151955308460338202e-07 +2.818222699243139368e-06 +2.838920541928440347e-06 +3.050739647337500175e-07 +-2.465265907215793529e-06 +-3.055508336973947250e-06 +-9.128700910339584048e-07 +2.249771371083119825e-06 +3.081780839428713369e-06 +1.204066121114378954e-06 +-1.761878266983632000e-06 +-3.087024015440775005e-06 +-1.748729112813366588e-06 +1.494332629395895281e-06 +3.083622466934211245e-06 +2.000238032798529119e-06 +-9.222855628500002924e-07 +-3.027284110262618297e-06 +-2.459117896043975414e-06 +6.218469306425458601e-07 +2.943636644764209658e-06 +2.662317353357709279e-06 +-5.177917465497133916e-09 +-2.656595783455085982e-06 +-2.975194029825672211e-06 +-6.121897147847749663e-07 +2.465190724087207064e-06 +3.055529535098033792e-06 +9.129096465147600536e-07 +-2.014420931540297274e-06 +-3.087116887118177198e-06 +-1.483149255993005404e-06 +1.761809630260435109e-06 +3.087015480294009457e-06 +1.748796544150532207e-06 +-1.213872066043083517e-06 +-3.068920639760422104e-06 +-2.237274950582254820e-06 +6.218469304994584902e-07 +3.027257267425890604e-06 +2.459197135874444185e-06 +-3.152100980212754159e-07 +-2.818282959618253049e-06 +-2.838852514580857026e-06 +-3.050601398378343097e-07 +2.656525652810296052e-06 +2.975241609550027586e-06 +6.122169866280836498e-07 +-2.249847545214697475e-06 +-3.081776340579127877e-06 +-1.204015683978650500e-06 +2.014347161035999542e-06 +3.087120452268125444e-06 +1.483209031757313825e-06 +-1.494393881446237258e-06 +-3.083623429701967304e-06 +-2.000164558067580843e-06 +9.222447130660705040e-07 +3.068900199591174303e-06 +2.237352665296288405e-06 +-6.218751952323854422e-07 +-2.943681909310511898e-06 +-2.662240490951830218e-06 +5.177600786082367972e-09 +2.818222666889502149e-06 +2.838920728856203197e-06 +3.050739660439167046e-07 +-2.465265906177216363e-06 +-3.055508223238704983e-06 +-9.128700881822972623e-07 +2.014347160967820166e-06 +3.081785459456511538e-06 +1.204066128157341443e-06 +-1.761878267217374977e-06 +-3.087024740964965139e-06 +-1.748729098693308934e-06 +1.213820143620283751e-06 +3.083630587141290704e-06 +2.000238093724568858e-06 +-9.222855630673673697e-07 +-3.027284089181581838e-06 +-2.459117869304699151e-06 +3.151955308194736198e-07 +2.943636738334340566e-06 +2.662318416699685006e-06 +-5.177917600794576775e-09 +-2.656595791362026736e-06 +-2.975193987062474409e-06 +-6.121897144822575727e-07 +2.249771372731092304e-06 +3.055529111826160834e-06 +9.129096456882073042e-07 +-2.014420931653842464e-06 +-3.087116317992878315e-06 +-1.483149253582620687e-06 +1.494332629186293166e-06 +3.087023742968697569e-06 +1.748796567853142202e-06 +-1.213872066153680820e-06 +-3.068920516935931861e-06 +-2.237274936286490558e-06 +6.218469306368714806e-07 +2.943636290768439938e-06 +2.459196933198562618e-06 +-3.152100985427826960e-07 +-2.818282945172002372e-06 +-2.838852432368107675e-06 +-3.050601398823266211e-07 +2.465190714646088049e-06 +2.975241815909446647e-06 +6.122169807437946814e-07 +-2.249847545645638616e-06 +-3.081776131829732087e-06 +-1.204015683769788268e-06 +1.761809630582506893e-06 +3.087115586178412298e-06 +1.483209004765927734e-06 +-1.494393881444599944e-06 +-3.083626888847235096e-06 +-2.000164541566777214e-06 +9.222447132300103428e-07 +3.027256959525581029e-06 +2.237352540054578736e-06 +-6.218751952735721016e-07 +-2.943682137430753711e-06 +-2.662240279547135640e-06 +5.177600548371468480e-09 +2.656525708168374454e-06 +2.838920367487901057e-06 +3.050739615392967058e-07 +-2.465265910652078294e-06 +-3.055507693310671003e-06 +-9.128700888621563989e-07 +2.014347162791535465e-06 +3.087115334546631429e-06 +1.204066099747427159e-06 +-1.761878267186252869e-06 +-3.087019783906130566e-06 +-1.748729085519497409e-06 +1.213820143621742554e-06 +3.068915747898269982e-06 +2.000237948703548045e-06 +-9.222855631041150470e-07 +-3.027284153417684457e-06 +-2.459117771558093995e-06 +3.151955305534898161e-07 +2.818222755331747264e-06 +2.662316837887460871e-06 +-5.177919575905114790e-09 +-2.656595770028137389e-06 +-2.975193541927646431e-06 +-6.121897160717602145e-07 +2.249771369899464973e-06 +3.081778638098394829e-06 +9.129096343552982227e-07 +-2.014420931247520759e-06 +-3.087118143518693237e-06 +-1.483149252050013248e-06 +1.494332629271366614e-06 +3.083611191664898888e-06 +1.748796516735614396e-06 +-1.213872065890197670e-06 +-3.068916019074334916e-06 +-2.237274925003372191e-06 +6.218469303313082624e-07 +2.943636018290449366e-06 +2.662316743245618561e-06 +-3.152100982096967303e-07 +-2.818282924138291761e-06 +-2.838852449590507989e-06 +-3.050601415378084178e-07 +2.465190714300445667e-06 +3.055530744710881232e-06 +6.122169829998658776e-07 +-2.249847546790695871e-06 +-3.081778195477484829e-06 +-1.204015685519354515e-06 +1.761809630506516815e-06 +3.087026089947893743e-06 +1.483209029837644940e-06 +-1.494393881414008290e-06 +-3.083631170055253859e-06 +-2.000164564008749556e-06 +9.222447128838812344e-07 +3.027257137092730263e-06 +2.459197274791265313e-06 +-6.218751946993127742e-07 +-2.943682241840437243e-06 +-2.662240875883936313e-06 +5.177599411854798302e-09 +2.656525689580131394e-06 +2.975242719503508963e-06 +3.050739642304999503e-07 +-2.465265908141236740e-06 +-3.055507409966075692e-06 +-9.128700877040965533e-07 +2.014347162246525667e-06 +3.087117081367807194e-06 +1.483209031484586580e-06 +-1.761878267191295256e-06 +-3.087022424574749835e-06 +-1.748729111255308350e-06 +1.213820143539339800e-06 +3.068910735938296978e-06 +2.237352678661156682e-06 +-9.222855626564606755e-07 +-3.027283786468236064e-06 +-2.459117716458082392e-06 +3.151955302193813758e-07 +2.818222728519646496e-06 +2.838920826522776868e-06 +-5.177918327220845245e-09 +-2.656595788650424821e-06 +-2.975193465148680353e-06 +-6.121897128985342367e-07 +2.249771370986777450e-06 +3.081780883773516501e-06 +1.204066124213934665e-06 +-2.014420932157988851e-06 +-3.087115737115972223e-06 +-1.483149249589518697e-06 +1.494332629194715850e-06 +3.083617096360583382e-06 +2.000238057825104762e-06 +-1.213872065907873130e-06 +-3.068925187140496035e-06 +-2.237274883635692086e-06 +6.218469300457401039e-07 +2.943636246392468535e-06 +2.662317340547320204e-06 +-5.177918037145504550e-09 +-2.818282997454390640e-06 +-2.838852163745343939e-06 +-3.050601384198528222e-07 +2.465190716858414864e-06 +3.055530309727082865e-06 +9.129096466147376817e-07 +-2.249847543712664961e-06 +-3.081774497777610128e-06 +-1.204015679127434042e-06 +1.761809630415061185e-06 +3.087024364161945625e-06 +1.748796528563774324e-06 +-1.494393881236986451e-06 +-3.083614825898169062e-06 +-2.000164503180351116e-06 +9.222447130369182769e-07 +3.027257278447567593e-06 +2.459197181235523377e-06 +-3.152100984826847025e-07 +-2.943681534589918649e-06 +-2.662240108556591613e-06 +5.177600657344220044e-09 +2.656525686817303467e-06 +2.975242448115834870e-06 +6.122169843940998965e-07 +-2.465265904934462328e-06 +-3.055509188280463714e-06 +-9.128700870515184421e-07 +2.014347162001775497e-06 +3.087117696763499701e-06 +1.483209020055367523e-06 +-1.494393881406907401e-06 +-3.087026609318223618e-06 +-1.748729100122564589e-06 +1.213820143633766610e-06 +3.068906735535524230e-06 +2.237352645569658475e-06 +-6.218751955161471970e-07 +-3.027284318223303594e-06 +-2.459118034120911282e-06 +3.151955301458421871e-07 +2.818222688584428278e-06 +2.838920821735220019e-06 +3.050739648845807509e-07 +-2.656595781800089482e-06 +-2.975194602679749042e-06 +-6.121897170761729421e-07 +2.249771372717715959e-06 +3.081782304621676918e-06 +1.204066117044157791e-06 +-1.761878267186168377e-06 +-3.087117395411752190e-06 +-1.483149266865272135e-06 +1.494332629502537999e-06 +3.083625972580297810e-06 +2.000238039944751701e-06 +-9.222855631034597612e-07 +-3.068920055843501212e-06 +-2.237275003743279682e-06 +6.218469297725546143e-07 +2.943636630220438807e-06 +2.662317523990572892e-06 +-5.177918369192947401e-09 +-2.656595775959408522e-06 +-2.838852798037668533e-06 +-3.050601405035421198e-07 +2.465190717108607221e-06 +3.055529205919641637e-06 +9.129096418802539052e-07 +-2.014420931666647908e-06 +-3.081776861500625377e-06 +-1.204015688337056836e-06 +1.761809630563580789e-06 +3.087023012703732639e-06 +1.748796516783566625e-06 +-1.213872066246057387e-06 +-3.083622086087906214e-06 +-2.000164598443290499e-06 +9.222447127348415522e-07 +3.027256799326660267e-06 +2.459196978793427138e-06 +-3.152100987257418655e-07 +-2.818282966704568379e-06 +-2.662240676059719911e-06 +5.177600349215444931e-09 +2.656525696998041488e-06 +2.975241780752169337e-06 +6.122169823718578628e-07 +-2.249847544824541394e-06 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 diff --git a/tst/regression/mocmc_diffusion.gold b/tst/regression/mocmc_diffusion.gold index 8ac0793ae..be328867b 100644 --- a/tst/regression/mocmc_diffusion.gold +++ b/tst/regression/mocmc_diffusion.gold @@ -1,25 +1,25 @@ -2.578814481287529143e-07 -8.539726882839995124e-11 -9.881445129638462621e-11 -2.880460357023293774e-08 -2.198153555894489294e-03 -6.512936231879704607e-01 -1.589897079949703271e-04 -2.487714603862657345e-02 -2.197674042014613471e-02 -1.033579527379025267e-02 -2.531893588544336000e-03 -1.082843017130321120e-02 -4.744884338522541156e-02 -7.923109834475633752e-20 -2.826042354446315629e-19 --5.282457299947488459e-18 --3.598817619128907654e-19 -2.373795279650825908e-18 -1.611427945521856623e-19 --4.579214891635570918e-18 -4.115761985687671323e-18 --2.074785247678172627e-19 -2.203076930037841375e-18 -2.798240882969251344e-18 --8.643975586171317102e-19 +2.903293198869289909e-07 +8.917324339173791865e-11 +1.015753710095253432e-10 +2.902242925555535038e-08 +2.203269912692418925e-03 +6.667470045966119585e-01 +1.853892724151311666e-04 +2.337474611586995866e-03 +2.038951701080027150e-03 +1.008054500631459446e-03 +2.727958063904742988e-04 +9.915326561658027293e-04 +4.440062562521645412e-03 +-1.078541213857498782e-19 +-9.518559574664915933e-22 +-1.821776703100472804e-20 +-6.373869807731488544e-20 +-4.544586618454861709e-20 +2.605898796556116674e-19 +2.824314855367294156e-19 +-1.467053235248274521e-19 +-8.360076325690497809e-21 +9.421651747181250117e-20 +-5.419127957100820803e-19 +1.637435460668574465e-19 diff --git a/tst/regression/mocmc_equilibration.gold b/tst/regression/mocmc_equilibration.gold index 50c369f26..61d88ba5e 100644 --- a/tst/regression/mocmc_equilibration.gold +++ b/tst/regression/mocmc_equilibration.gold @@ -1,16 +1,16 @@ -7.782239480291789357e-01 -7.782239480291789357e-01 -7.782239480291788247e-01 -7.782239480291789357e-01 --3.319442647366402016e-17 --2.641239402025653325e-17 -1.803654437194548050e-17 -5.094346950324803810e-17 --1.430155146108605356e-18 --6.894023125021501989e-18 --5.658156072167891793e-18 -4.106285280551914311e-18 -1.742030982911749756e-19 -1.424440983663824178e-18 -5.480844936769992495e-18 -1.804352161149941056e-18 +7.899461039993100897e-01 +7.899461039993102007e-01 +7.899461039993099787e-01 +7.899461039993100897e-01 +-2.379322113936495115e-17 +-1.129620046446157642e-17 +9.319386201426945604e-19 +3.415748298368384591e-17 +-5.269569986218940562e-18 +-4.383374134615519398e-18 +4.868801217256894866e-18 +4.784142903577562012e-18 +5.173045656497453725e-18 +-3.052373860422978175e-18 +-5.992335522073705248e-18 +3.871663725999230853e-18 From 1404cb880fcc1ebbf370ff48d5e73a0f64048b43 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Tue, 28 Jan 2025 21:40:29 +0000 Subject: [PATCH 30/48] thread in correct radiation K Source with FMKS --- src/radiation/geodesics.hpp | 33 ++++++++++++- tst/regression/run/test_input.pin | 82 ------------------------------- 2 files changed, 32 insertions(+), 83 deletions(-) delete mode 100644 tst/regression/run/test_input.pin diff --git a/src/radiation/geodesics.hpp b/src/radiation/geodesics.hpp index f3706e65e..7c9fae78c 100644 --- a/src/radiation/geodesics.hpp +++ b/src/radiation/geodesics.hpp @@ -39,7 +39,38 @@ template KOKKOS_INLINE_FUNCTION void GetKSource(Real &X0, Real &X1, Real &X2, Real &X3, Real &Kcov0, Real &Kcov1, Real &Kcov2, Real &Kcov3, Real &Kcon0, Geom &geom, Real source[4]) { - SPACETIMELOOP(mu) { source[mu] = 0.; } + if constexpr (std::is_same::value) { + Real Kcov[NDFULL] = {Kcov0, Kcov1, Kcov2, Kcov3}; + Real Xm[NDFULL], Xp[NDFULL]; + Real Gconm[NDFULL][NDFULL], Gconp[NDFULL][NDFULL]; + Real dG; + + constexpr static Real DELTA = 1.0e-6; + + SPACETIMELOOP(mu) { + source[mu] = 0.; + Xp[0] = X0; + Xp[1] = X1; + Xp[2] = X2; + Xp[3] = X3; + SPACETIMELOOP(nu) { Xm[nu] = Xp[nu]; } + + Xm[mu] -= DELTA; + Xp[mu] += DELTA; + + geom.SpacetimeMetricInverse(Xm[0], Xm[1], Xm[2], Xm[3], Gconm); + geom.SpacetimeMetricInverse(Xp[0], Xp[1], Xp[2], Xp[3], Gconp); + + SPACETIMELOOP2(nu, kap) { + dG = (Gconp[nu][kap] - Gconm[nu][kap]) / (Xp[mu] - Xm[mu]); + source[mu] += Kcov[nu] * Kcov[kap] * dG; + } + + source[mu] *= -1.0 / (2.0 * Kcon0); + } + } else { + SPACETIMELOOP(mu) { source[mu] = 0.; } + } } template diff --git a/tst/regression/run/test_input.pin b/tst/regression/run/test_input.pin deleted file mode 100644 index 208b2e594..000000000 --- a/tst/regression/run/test_input.pin +++ /dev/null @@ -1,82 +0,0 @@ - -problem = linear_modes - -problem_id = hydro_modes # problem ID: basename of output filenames - -variables = p.density, & - c.density, & - p.velocity, & - p.energy, & - p.bfield, & - pressure, & - cs -file_type = hdf5 # Tabular data dump -dt = 0.5 # time increment between outputs - -nlim = -1 # cycle limit -tlim = 0.4 # time limit -integrator = rk2 # time integration algorithm -ncycle_out = 10 # interval for stdout summary info -dt_init_fact = 0.1 - -nghost = 4 -nx1 = 64 -x1min = 0 # minimum value of X1 -x1max = 1 # maximum value of X1 -ix1_bc = periodic # Inner-X1 boundary condition flag -ox1_bc = periodic # Outer-X1 boundary condition flag -nx2 = 64 -x2min = 0 # minimum value of X2 -x2max = 1 # maximum value of X2 -ix2_bc = periodic # Inner-X2 boundary condition flag -ox2_bc = periodic # Outer-X2 boundary condition flag -nx3 = 1 # Number of zones in X3-direction -x3min = -1 # minimum value of X3 -x3max = 1 # maximum value of X3 -ix3_bc = periodic # Inner-X3 boundary condition flag -ox3_bc = periodic # Outer-X3 boundary condition flfgag -num_threads = 1 # maximum number of OMP threads - - -field = c.c.bulk.rho -method = derivative_order_1 -max_level = 3 - -vx = 0. # 0.436436 -vy = 0. # 0.436436 - -kmult = 4 # Multiply k by this factor -a = 0.3 # amplitude of wiggle -alpha = 1. # 0.85 # lapse -vy = 0. # 0.524631 -do_fd_on_grid = false - -type = IdealGas -Gamma = 1.333333333333333333 -Cv = 1.0 - -hydro = true -he = false -3t = false -rad = false - -mhd = true -xorder = 2 -cfl = 0.4 -riemann = hll -c2p_max_iter = 100 -recon = linear -c2p_tol = 1.e-12 -c2p_method = robust - -physics = hydro -mode = sound -amplitude = 1.e-5 - -nx1 = 64 - -nx2 = 64 - -nx1 = 64 - -nx2 = 64 From d089b52e000e92d35c9391091750c2e288661cd6 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Tue, 28 Jan 2025 21:40:57 +0000 Subject: [PATCH 31/48] cleanup --- src/radiation/mocmc.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/radiation/mocmc.cpp b/src/radiation/mocmc.cpp index b7c130819..51fb36ffe 100644 --- a/src/radiation/mocmc.cpp +++ b/src/radiation/mocmc.cpp @@ -523,7 +523,6 @@ TaskStatus MOCMCFluidSource(T *rc_base, T *rc, const Real dt, const bool update_ // Assume particles are already sorted from MOCMCReconstruction call! auto *pmb = rc->GetParentPointer(); - // swarm pack StateDescriptor *rad = pmb->packages.Get("radiation").get(); // Meshblock geometry From 6bd24ce525a3282e326753f7b27387b53ba2acc9 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Tue, 28 Jan 2025 21:41:55 +0000 Subject: [PATCH 32/48] start moving moments to MeshData & packs --- src/radiation/moments.cpp | 144 ++++++++++++++++++++------------------ 1 file changed, 76 insertions(+), 68 deletions(-) diff --git a/src/radiation/moments.cpp b/src/radiation/moments.cpp index 368782489..e8bb16287 100644 --- a/src/radiation/moments.cpp +++ b/src/radiation/moments.cpp @@ -62,39 +62,24 @@ TaskStatus MomentCon2PrimImpl(T *rc) { namespace cr = radmoment_cons; namespace pr = radmoment_prim; namespace ir = radmoment_internal; + namespace pf = fluid_prim; Mesh *pmesh = rc->GetMeshPointer(); StateDescriptor *rad = pmesh->packages.Get("radiation").get(); + const int n_species = rad->Param("num_species"); IndexRange ib = rc->GetBoundsI(IndexDomain::entire); IndexRange jb = rc->GetBoundsJ(IndexDomain::entire); IndexRange kb = rc->GetBoundsK(IndexDomain::entire); + using parthenon::MakePackDescriptor; + auto &resolved_pkgs = pmesh->resolved_packages; - std::vector variables{cr::E::name(), - cr::F::name(), - pr::J::name(), - pr::H::name(), - fluid_prim::velocity::name(), - ir::xi::name(), - ir::phi::name(), - ir::c2pfail::name(), - ir::tilPi::name()}; - PackIndexMap imap; - auto v = rc->PackVariables(variables, imap); + static auto desc = + MakePackDescriptor(resolved_pkgs.get()); - auto cE = imap.GetFlatIdx(cr::E::name()); - auto pJ = imap.GetFlatIdx(pr::J::name()); - auto cF = imap.GetFlatIdx(cr::F::name()); - auto pH = imap.GetFlatIdx(pr::H::name()); - auto pv = imap.GetFlatIdx(fluid_prim::velocity::name()); - auto iTilPi = imap.GetFlatIdx(ir::tilPi::name(), false); - auto specB = cE.GetBounds(1); - auto dirB = pH.GetBounds(2); - - auto iXi = imap.GetFlatIdx(ir::xi::name()); - auto iPhi = imap.GetFlatIdx(ir::phi::name()); - - auto ifail = imap[ir::c2pfail::name()].first; + auto v = desc.GetPack(rc); + const int nblocks = v.GetNBlocks(); auto geom = Geometry::GetCoordinateSystem(rc); const Real pi = acos(-1); @@ -104,21 +89,22 @@ TaskStatus MomentCon2PrimImpl(T *rc) { parthenon::par_for( DEFAULT_LOOP_PATTERN, "RadMoments::Con2Prim", DevExecSpace(), 0, - v.GetDim(5) - 1, // Loop over meshblocks - specB.s, specB.e, // Loop over species - kb.s, kb.e, // z-loop - jb.s, jb.e, // y-loop - ib.s, ib.e, // x-loop + rc->NumBlocks() - 1, // Loop over meshblocks + 0, n_species - 1, // Loop over species + kb.s, kb.e, // z-loop + jb.s, jb.e, // y-loop + ib.s, ib.e, // x-loop KOKKOS_LAMBDA(const int b, const int ispec, const int k, const int j, const int i) { Tens2 cov_gamma; geom.Metric(CellLocation::Cent, b, k, j, i, cov_gamma.data); const Real isdetgam = 1.0 / geom.DetGamma(CellLocation::Cent, b, k, j, i); - const Real vp[3] = {v(b, pv(0), k, j, i), v(b, pv(1), k, j, i), - v(b, pv(2), k, j, i)}; + const Real vp[3] = {v(b, pf::velocity(0), k, j, i), + v(b, pf::velocity(1), k, j, i), + v(b, pf::velocity(2), k, j, i)}; const Real W = phoebus::GetLorentzFactor(vp, cov_gamma.data); - Vec con_v{{v(b, pv(0), k, j, i) / W, v(b, pv(1), k, j, i) / W, - v(b, pv(2), k, j, i) / W}}; + Vec con_v{{v(b, pf::velocity(0), k, j, i) / W, v(b, pf::velocity(1), k, j, i) / W, + v(b, pf::velocity(2), k, j, i) / W}}; typename CLOSURE::LocalGeometryType g(geom, CellLocation::Cent, b, k, j, i); CLOSURE c(con_v, &g, closure_runtime_params); @@ -126,44 +112,46 @@ TaskStatus MomentCon2PrimImpl(T *rc) { Real J; Vec covH; Tens2 conTilPi; - Real E = v(b, cE(ispec), k, j, i) * isdetgam; - Vec covF = {{v(b, cF(ispec, 0), k, j, i) * isdetgam, - v(b, cF(ispec, 1), k, j, i) * isdetgam, - v(b, cF(ispec, 2), k, j, i) * isdetgam}}; - - if (iTilPi.IsValid()) { - SPACELOOP2(ii, jj) { conTilPi(ii, jj) = v(b, iTilPi(ispec, ii, jj), k, j, i); } + Real E = v(b, cr::E(ispec), k, j, i) * isdetgam; + Vec covF = {{v(b, cr::F(ispec, 0), k, j, i) * isdetgam, + v(b, cr::F(ispec, 1), k, j, i) * isdetgam, + v(b, cr::F(ispec, 2), k, j, i) * isdetgam}}; + + if (v.Contains(b, ir::tilPi())) { + SPACELOOP2(ii, jj) { + conTilPi(ii, jj) = v(b, ir::tilPi(ispec, ii, jj), k, j, i); + } } else { Real xi = 0.0; Real phi = pi; // TODO(BRR) Remove STORE_GUESS parameter and instead check if closure type is // M1? if (STORE_GUESS) { - xi = v(b, iXi(ispec), k, j, i); - phi = 1.0001 * v(b, iPhi(ispec), k, j, i); + xi = v(b, ir::xi(ispec), k, j, i); + phi = 1.0001 * v(b, ir::phi(ispec), k, j, i); } c.GetConTilPiFromCon(E, covF, xi, phi, &conTilPi); if (STORE_GUESS) { - v(b, iXi(ispec), k, j, i) = xi; - v(b, iPhi(ispec), k, j, i) = phi; + v(b, ir::xi(ispec), k, j, i) = xi; + v(b, ir::phi(ispec), k, j, i) = phi; } } auto status = c.Con2Prim(E, covF, conTilPi, &J, &covH); if (status == ClosureStatus::modified) { c.Prim2Con(J, covH, conTilPi, &E, &covF); - v(b, cE(ispec), k, j, i) = E / isdetgam; - SPACELOOP(ii) { v(b, cF(ispec, ii), k, j, i) = covF(ii) / isdetgam; } + v(b, cr::E(ispec), k, j, i) = E / isdetgam; + SPACELOOP(ii) { v(b, cr::F(ispec, ii), k, j, i) = covF(ii) / isdetgam; } status = ClosureStatus::success; } - v(b, pJ(ispec), k, j, i) = J; - for (int idir = dirB.s; idir <= dirB.e; ++idir) { // Loop over directions + v(b, pr::J(ispec), k, j, i) = J; + for (int idir = 0; idir <= 2; ++idir) { // Loop over directions // Use the scaled value of the rest frame flux for reconstruction - v(b, pH(ispec, idir), k, j, i) = robust::ratio(covH(idir), J); + v(b, pr::H(ispec, idir), k, j, i) = robust::ratio(covH(idir), J); } - v(b, ifail, k, j, i) = + v(b, ir::c2pfail(), k, j, i) = (status == ClosureStatus::success ? FailFlags::success : FailFlags::fail); }); @@ -199,16 +187,30 @@ TaskStatus MomentPrim2ConImpl(T *rc, IndexDomain domain) { namespace cr = radmoment_cons; namespace pr = radmoment_prim; namespace ir = radmoment_internal; + namespace pf = fluid_prim; + Mesh *pmesh = rc->GetMeshPointer(); + StateDescriptor *rad = pmesh->packages.Get("radiation").get(); IndexRange ib = rc->GetBoundsI(domain); IndexRange jb = rc->GetBoundsJ(domain); IndexRange kb = rc->GetBoundsK(domain); + auto &resolved_pkgs = pmesh->resolved_packages; + static auto desc = + MakePackDescriptor( + resolved_pkgs.get()); + + auto v = desc.GetPack(rc); + + /* std::vector variables{cr::E::name(), cr::F::name(), pr::J::name(), pr::H::name(), fluid_prim::velocity::name()}; if (programming::is_specialization_of::value) { variables.push_back(ir::tilPi::name()); } + + + PackIndexMap imap; auto v = rc->PackVariables(variables, imap); @@ -218,19 +220,21 @@ TaskStatus MomentPrim2ConImpl(T *rc, IndexDomain domain) { auto pH = imap.GetFlatIdx(pr::H::name()); auto pv = imap.GetFlatIdx(fluid_prim::velocity::name()); auto iTilPi = imap.GetFlatIdx(ir::tilPi::name(), false); + */ - auto specB = cE.GetBounds(1); - auto dirB = pH.GetBounds(2); + // auto specB = cE.GetBounds(1); + // auto dirB = pH.GetBounds(2); auto geom = Geometry::GetCoordinateSystem(rc); + const int n_species = rad->Param("num_species"); parthenon::par_for( DEFAULT_LOOP_PATTERN, "RadMoments::Prim2Con", DevExecSpace(), 0, - v.GetDim(5) - 1, // Loop over meshblocks - specB.s, specB.e, // Loop over species - kb.s, kb.e, // z-loop - jb.s, jb.e, // y-loop - ib.s, ib.e, // x-loop + rc->NumBlocks() - 1, // Loop over meshblocks + 0, n_species - 1, // Loop over species + kb.s, kb.e, // z-loop + jb.s, jb.e, // y-loop + ib.s, ib.e, // x-loop KOKKOS_LAMBDA(const int b, const int ispec, const int k, const int j, const int i) { // Set up the background const Real sdetgam = geom.DetGamma(CellLocation::Cent, b, k, j, i); @@ -238,8 +242,9 @@ TaskStatus MomentPrim2ConImpl(T *rc, IndexDomain domain) { geom.Metric(CellLocation::Cent, b, k, j, i, cov_gamma.data); typename CLOSURE::LocalGeometryType g(geom, CellLocation::Cent, b, k, j, i); - const Real con_vp[3] = {v(b, pv(0), k, j, i), v(b, pv(1), k, j, i), - v(b, pv(2), k, j, i)}; + const Real con_vp[3] = {v(b, pf::velocity(0), k, j, i), + v(b, pf::velocity(1), k, j, i), + v(b, pf::velocity(2), k, j, i)}; const Real W = phoebus::GetLorentzFactor(con_vp, cov_gamma.data); Vec con_v{{con_vp[0] / W, con_vp[1] / W, con_vp[2] / W}}; @@ -248,12 +253,15 @@ TaskStatus MomentPrim2ConImpl(T *rc, IndexDomain domain) { Real E; Vec covF; Tens2 conTilPi; - Real J = v(b, pJ(ispec), k, j, i); - Vec covH = {{v(b, pH(ispec, 0), k, j, i) * J, v(b, pH(ispec, 1), k, j, i) * J, - v(b, pH(ispec, 2), k, j, i) * J}}; - - if (iTilPi.IsValid()) { - SPACELOOP2(ii, jj) { conTilPi(ii, jj) = v(b, iTilPi(ispec, ii, jj), k, j, i); } + Real J = v(b, pr::J(ispec), k, j, i); + Vec covH = {{v(b, pr::H(ispec, 0), k, j, i) * J, + v(b, pr::H(ispec, 1), k, j, i) * J, + v(b, pr::H(ispec, 2), k, j, i) * J}}; + + if (v.Contains(b, ir::tilPi())) { + SPACELOOP2(ii, jj) { + conTilPi(ii, jj) = v(b, ir::tilPi(ispec, ii, jj), k, j, i); + } } else { c.GetConTilPiFromPrim(J, covH, &conTilPi); } @@ -262,9 +270,9 @@ TaskStatus MomentPrim2ConImpl(T *rc, IndexDomain domain) { c.Prim2Con(J, covH, conTilPi, &E, &covF); - v(b, cE(ispec), k, j, i) = sdetgam * E; - for (int idir = dirB.s; idir <= dirB.e; ++idir) { - v(b, cF(ispec, idir), k, j, i) = sdetgam * covF(idir); + v(b, cr::E(ispec), k, j, i) = sdetgam * E; + for (int idir = 0; idir <= 2; ++idir) { + v(b, cr::F(ispec, idir), k, j, i) = sdetgam * covF(idir); } }); From 1981869214364f853d36edc4bdfb6108241ec315 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Tue, 28 Jan 2025 21:43:34 +0000 Subject: [PATCH 33/48] monte carlo variables & packs refactor --- cmake/Transport.cmake | 22 +++- src/compile_constants.hpp.in | 7 +- src/phoebus_driver.cpp | 38 ++---- src/phoebus_utils/variables.hpp | 41 +++++- src/radiation/monte_carlo.cpp | 222 ++++++++++++++++---------------- src/radiation/radiation.cpp | 61 +++++---- 6 files changed, 215 insertions(+), 176 deletions(-) diff --git a/cmake/Transport.cmake b/cmake/Transport.cmake index 6e97fb6e5..33dd44a00 100644 --- a/cmake/Transport.cmake +++ b/cmake/Transport.cmake @@ -10,4 +10,24 @@ # prepare derivative works, distribute copies to the public, perform # publicly and display publicly, and to permit others to do so. -set(MOCMC_NUM_SPECIES 3 CACHE INTEGER "Number of radiation species for MOCMC.") +option(PHOEBUS_DO_NU_ELECTRON "Evolve electron neutrinos" ON) +option(PHOEBUS_DO_NU_ELECTRON_ANTI "Evolve electron antineutrinos" ON) +option(PHOEBUS_DO_NU_HEAVY "Evolve heavy composite type neutrinos" ON) + +if(PHOEBUS_DO_NU_ELECTRON) + set(DO_NU_ELECTRON 1 CACHE BOOL "Electron neutrinos enabled") +else() + set(DO_NU_ELECTRON 0 CACHE BOOL "Electron neutrinos disabled") +endif() + +if(PHOEBUS_DO_NU_ELECTRON_ANTI) + set(DO_NU_ELECTRON_ANTI 1 CACHE BOOL "Electron antineutrinos enabled") +else() + set(DO_NU_ELECTRON_ANTI 0 CACHE BOOL "Electron antineutrinos disabled") +endif() + +if(PHOEBUS_DO_NU_HEAVY) + set(DO_NU_HEAVY 1 CACHE BOOL "Heavy neutrinos enabled") +else() + set(DO_NU_HEAVY 0 CACHE BOOL "Heavy neutrinos disabled") +endif() diff --git a/src/compile_constants.hpp.in b/src/compile_constants.hpp.in index 44c69c8a7..b81b147bd 100644 --- a/src/compile_constants.hpp.in +++ b/src/compile_constants.hpp.in @@ -16,7 +16,10 @@ #define NCONS_MAX (@MAX_NUMBER_CONSERVED_VARS@) -#define MOCMC_NUM_SPECIES (@MOCMC_NUM_SPECIES@) +// NOTE(BLB) This will have to change if we support additional species +#define DO_NU_ELECTRON (@DO_NU_ELECTRON@) +#define DO_NU_ELECTRON_ANTI (@DO_NU_ELECTRON_ANTI@) +#define DO_NU_HEAVY (@DO_NU_HEAVY@) #define PHOEBUS_GEOMETRY Geometry::@PHOEBUS_GEOMETRY@ #define GEOMETRY_MESH @GEOMETRY_MESH@ @@ -28,4 +31,6 @@ #define CON2PRIM_STATISTICS (@CON2PRIM_STATISTICS@) +constexpr static int PHOEBUS_NUM_SPECIES = DO_NU_ELECTRON + DO_NU_ELECTRON_ANTI + DO_NU_HEAVY; + #endif //COMPILE_CONSTANTS_HPP_ diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 813b91bd6..e53b479bb 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -334,11 +334,7 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { tl.AddTask(none, &SwarmContainer::ResetCommunication, sc.get()); } } - } - // TODO(BLB) merge ifs - // if (rad_mocmc_active && stage == integrator->nstages) { - if (rad_mocmc_active) { TaskRegion &sync_region_mocmc = tc.AddRegion(num_partitions); for (int n = 0; n < num_partitions; n++) { auto &tl = sync_region_mocmc[n]; @@ -349,9 +345,6 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { auto samples_transport = tl.AddTask(none, radiation::MOCMCTransport, base.get(), beta * dt); } - } - - if (rad_mocmc_active) { TaskRegion &sync_region_mocmc_comm = tc.AddRegion(blocks.size()); for (int ib = 0; ib < blocks.size(); ib++) { auto &tl = sync_region_mocmc_comm[ib]; @@ -364,9 +357,7 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { auto receive = tl.AddTask(send, &SwarmContainer::Receive, sc.get(), BoundaryCommSubset::all); } - } - if (rad_mocmc_active) { TaskRegion &async_region_mocmc_2 = tc.AddRegion(num_partitions); for (int n = 0; n < num_partitions; n++) { auto &tl = async_region_mocmc_2[n]; @@ -600,32 +591,27 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { TaskRegion &sync_region_fixup = tc.AddRegion(num_partitions); TaskID gas_rad_int(0); - for (int ib = 0; ib < num_partitions; ib++) { - auto &tl = sync_region_fixup[ib]; + if (rad_mocmc_active) { + for (int ib = 0; ib < num_partitions; ib++) { + auto &tl = sync_region_fixup[ib]; - auto &base = pmesh->mesh_data.GetOrAdd("base", ib); - auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], ib); - using MDT = std::remove_pointer::type; - if (rad_mocmc_active) { + auto &base = pmesh->mesh_data.GetOrAdd("base", ib); + auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], ib); + using MDT = std::remove_pointer::type; auto impl_update = tl.AddTask(none, radiation::MOCMCFluidSource, base.get(), sc1.get(), beta * dt, fluid_active); auto impl_edd = tl.AddTask(impl_update, radiation::MOCMCEddington, base.get(), sc1.get()); gas_rad_int = gas_rad_int | impl_edd; - } else if (rad_moments_active) { - // auto impl_update = - // tl.AddTask(none, radiation::MomentFluidSource>, sc1.get(), - // beta * dt, fluid_active); - // gas_rad_int = gas_rad_int | impl_update; } } - TaskRegion &async_region_fixup2 = tc.AddRegion(num_independent_task_lists); - for (int ib = 0; ib < num_independent_task_lists; ib++) { - auto pmb = blocks[ib].get(); - auto &tl = async_region_fixup2[ib]; - auto &sc1 = pmb->meshblock_data.Get(stage_name[stage]); - if (rad_moments_active) { // TODO(PRE MERGE) move outside + if (rad_moments_active) { + TaskRegion &async_region_fixup2 = tc.AddRegion(num_independent_task_lists); + for (int ib = 0; ib < num_independent_task_lists; ib++) { + auto pmb = blocks[ib].get(); + auto &tl = async_region_fixup2[ib]; + auto &sc1 = pmb->meshblock_data.Get(stage_name[stage]); auto impl_update = tl.AddTask(none, radiation::MomentFluidSource>, sc1.get(), beta * dt, fluid_active); diff --git a/src/phoebus_utils/variables.hpp b/src/phoebus_utils/variables.hpp index f2b4f833c..0a71b9a9e 100644 --- a/src/phoebus_utils/variables.hpp +++ b/src/phoebus_utils/variables.hpp @@ -52,6 +52,15 @@ static std::string name() { return #ns "." #varname; } \ } +#define TENSOR_VARIABLE(ns, varname, ...) \ + struct varname : public parthenon::variable_names::base_t { \ + template \ + KOKKOS_INLINE_FUNCTION varname(Ts &&...args) \ + : parthenon::variable_names::base_t( \ + std::forward(args)...) {} \ + static std::string name() { return #ns "." #varname; } \ + } + using parthenon::variable_names::ANYDIM; namespace fluid_prim { @@ -77,26 +86,26 @@ VARIABLE(c, ye); namespace radmoment_prim { VARIABLE(r.p, J); -VARIABLE(r.p, H); +TENSOR_VARIABLE(r.p, H, ANYDIM, 3); // (num_species, 3) } // namespace radmoment_prim namespace radmoment_cons { VARIABLE(r.c, E); -VARIABLE(r.c, F); +TENSOR_VARIABLE(r.c, F, ANYDIM, 3); // (num_species, 3) } // namespace radmoment_cons namespace radmoment_internal { VARIABLE(r.i, xi); VARIABLE(r.i, phi); -VARIABLE(r.i, ql); -VARIABLE(r.i, qr); +TENSOR_VARIABLE(r.i, ql, ANYDIM, 3); // (num_species, nrecon) +TENSOR_VARIABLE(r.i, qr, ANYDIM, 3); // (num_species, nrecon) VARIABLE(r.i, ql_v); VARIABLE(r.i, qr_v); VARIABLE(r.i, dJ); VARIABLE(r.i, kappaJ); VARIABLE(r.i, kappaH); VARIABLE(r.i, JBB); -VARIABLE(r.i, tilPi); +TENSOR_VARIABLE(r.i, tilPi, PHOEBUS_NUM_SPECIES, 3, 3); VARIABLE(r.i, kappaH_mean); VARIABLE(r.i, c2pfail); VARIABLE(r.i, srcfail); @@ -116,7 +125,7 @@ SWARM_VARIABLE(Real, mocmc.c, mu_hi); SWARM_VARIABLE(Real, mocmc.c, phi_lo); SWARM_VARIABLE(Real, mocmc.c, phi_hi); SWARM_VARIABLE(Real, mocmc.c, ncov); -TENSOR_SWARM(Real, mocmc.c, Inuinv, ANYDIM, MOCMC_NUM_SPECIES); +TENSOR_SWARM(Real, mocmc.c, Inuinv, ANYDIM, PHOEBUS_NUM_SPECIES); } // namespace mocmc_core namespace internal_variables { @@ -141,6 +150,24 @@ VARIABLE_CUSTOM(cell_coords, g.c.coord); VARIABLE_CUSTOM(node_coords, g.n.coord); } // namespace geometric_variables +namespace monte_carlo_internal { +VARIABLE(monte_carlo.i, dNdlnu_max); +VARIABLE(monte_carlo.i, dN); +VARIABLE(monte_carlo.i, Ns); +TENSOR_VARIABLE(monte_carlo.i, dNdlnu, ANYDIM, PHOEBUS_NUM_SPECIES); +} // namespace monte_carlo_internal + +// TODO(BLB) Can probably move k0..k3 to k(mu) +namespace monte_carlo_core { +SWARM_VARIABLE(Real, monte_carlo.c, t); +SWARM_VARIABLE(Real, monte_carlo.c, k0); +SWARM_VARIABLE(Real, monte_carlo.c, k1); +SWARM_VARIABLE(Real, monte_carlo.c, k2); +SWARM_VARIABLE(Real, monte_carlo.c, k3); +SWARM_VARIABLE(Real, monte_carlo.c, weight); +SWARM_VARIABLE(int, monte_carlo.c, species); +} // namespace monte_carlo_core + namespace tracer_variables { SWARM_VARIABLE(Real, tr, rho); SWARM_VARIABLE(Real, tr, temperature); @@ -162,7 +189,7 @@ SWARM_VARIABLE(Real, tr, bernoulli); SWARM_VARIABLE(Real, tr, B_x); SWARM_VARIABLE(Real, tr, B_y); SWARM_VARIABLE(Real, tr, B_z); -TENSOR_SWARM(Real, tr, test, ANYDIM, MOCMC_NUM_SPECIES); +TENSOR_SWARM(Real, tr, test, ANYDIM, PHOEBUS_NUM_SPECIES); } // namespace tracer_variables namespace diagnostic_variables { diff --git a/src/radiation/monte_carlo.cpp b/src/radiation/monte_carlo.cpp index c9bd47b9f..4f1b143ce 100644 --- a/src/radiation/monte_carlo.cpp +++ b/src/radiation/monte_carlo.cpp @@ -19,6 +19,7 @@ using Geometry::CoordSysMeshBlock; using Geometry::NDFULL; +using parthenon::MakePackDescriptor; namespace radiation { @@ -88,14 +89,14 @@ void SetWeight(Mesh *pmesh) { auto &phoebus_pkg = pmesh->packages.Get("phoebus"); auto &unit_conv = phoebus_pkg->Param("unit_conv"); auto rad = pmesh->packages.Get("radiation"); - const auto nusamp = rad->Param>("nusamp"); + const auto nusamp0 = rad->Param("nu_min"); auto &code_constants = phoebus_pkg->Param("code_constants"); const Real sim_vol = PhoebusUtils::GetRegionVolume(pmesh->mesh_size); const Real h_code = code_constants.h; const Real dNtot = rad->Param("tune_emission") / (std::pow(sim_vol, 1. / 3.) * 1.0); // Note: may need a dt term here? - rad->UpdateParam("wgtC", rad->Param("Jtot") / (h_code * dNtot) * nusamp[0]); + rad->UpdateParam("wgtC", rad->Param("Jtot") / (h_code * dNtot) * nusamp0); } TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, @@ -103,6 +104,8 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, namespace p = fluid_prim; namespace c = fluid_cons; namespace iv = internal_variables; + namespace mci = monte_carlo_internal; + namespace mcc = monte_carlo_core; auto opac = pmb->packages.Get("opacity"); auto rad = pmb->packages.Get("radiation"); auto swarm = sc->Get("monte_carlo"); @@ -151,23 +154,13 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, const Real h_code = code_constants.h; const Real mp_code = code_constants.mp; - std::vector vars( - {p::density::name(), p::temperature::name(), p::ye::name(), p::velocity::name(), - "dNdlnu_max", "dNdlnu", "dN", "Ns", iv::Gcov::name(), iv::Gye::name()}); - PackIndexMap imap; - auto v = rc->PackVariables(vars, imap); - const int pye = imap[p::ye::name()].first; - const int pdens = imap[p::density::name()].first; - const int ptemp = imap[p::temperature::name()].first; - const int pvlo = imap[p::velocity::name()].first; - const int pvhi = imap[p::velocity::name()].second; - const int idNdlnu = imap["dNdlnu"].first; - const int idNdlnu_max = imap["dNdlnu_max"].first; - const int idN = imap["dN"].first; - const int iNs = imap["Ns"].first; - const int Gcov_lo = imap[iv::Gcov::name()].first; - const int Gcov_hi = imap[iv::Gcov::name()].second; - const int Gye = imap[iv::Gye::name()].first; + Mesh *pmesh = rc->GetMeshPointer(); + auto &resolved_pkgs = pmesh->resolved_packages; + static auto desc = + MakePackDescriptor( + resolved_pkgs.get()); + auto v = desc.GetPack(rc); // TODO(BRR) update this dynamically somewhere else. Get a reasonable starting value Real wgtC = rad->Param("wgtC"); @@ -175,10 +168,10 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, pmb->par_for( "MonteCarloZeroFiveForce", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { - for (int mu = Gcov_lo; mu <= Gcov_lo + 3; mu++) { - v(mu, k, j, i) = 0.; + for (int mu = 0; mu <= 3; mu++) { + v(0, iv::Gcov(mu), k, j, i) = 0.; } - v(Gye, k, j, i) = 0.; + v(0, iv::Gye(), k, j, i) = 0.; }); for (int sidx = 0; sidx < num_species; sidx++) { @@ -188,9 +181,9 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, KOKKOS_LAMBDA(const int k, const int j, const int i) { auto rng_gen = rng_pool.get_state(); Real detG = geom.DetG(CellLocation::Cent, k, j, i); - const Real &dens = v(pdens, k, j, i); - const Real &temp = v(ptemp, k, j, i); - const Real &ye = v(pye, k, j, i); + const Real &dens = v(0, p::density(), k, j, i); + const Real &temp = v(0, p::temperature(), k, j, i); + const Real &ye = v(0, p::ye(), k, j, i); Real dN = 0.; Real dNdlnu_max = 0.; @@ -204,15 +197,14 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, // Note that factors of nu in numerator and denominator cancel Real dNdlnu = Jnu * d3x * detG / (h_code * wgt); - // TODO(BRR) use FlatIdx - v(idNdlnu + sidx + n * num_species, k, j, i) = dNdlnu; + v(0, mci::dNdlnu(sidx + n * num_species), k, j, i) = dNdlnu; if (dNdlnu > dNdlnu_max) { dNdlnu_max = dNdlnu; } } for (int n = 0; n <= nu_bins; n++) { - v(idNdlnu + sidx + n * num_species, k, j, i) /= dNdlnu_max; + v(0, mci::dNdlnu(sidx + n * num_species), k, j, i) /= dNdlnu_max; } // Trapezoidal rule @@ -224,7 +216,7 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, (h_code * GetWeight(wgtC, nu1)) * dlnu; dN *= d3x * detG * dt; - v(idNdlnu_max + sidx, k, j, i) = dNdlnu_max; + v(0, mci::dNdlnu_max(sidx), k, j, i) = dNdlnu_max; int Ns = static_cast(dN); if (dN - Ns > rng_gen.drand()) { @@ -232,8 +224,8 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, } // TODO(BRR) Use a ParArrayND instead of these weird static_casts - v(idN + sidx, k, j, i) = dN; - v(iNs + sidx, k, j, i) = static_cast(Ns); + v(0, mci::dN(sidx), k, j, i) = dN; + v(0, mci::Ns(sidx), k, j, i) = static_cast(Ns); rng_pool.free_state(rng_gen); }); } @@ -244,7 +236,7 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, parthenon::loop_pattern_mdrange_tag, "MonteCarloReduceParticleCreation", DevExecSpace(), 0, num_species - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int sidx, const int k, const int j, const int i, Real &dNtot) { - dNtot += v(idN + sidx, k, j, i); + dNtot += v(0, mci::dN(sidx), k, j, i); }, Kokkos::Sum(dNtot)); @@ -260,14 +252,14 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, KOKKOS_LAMBDA(const int sidx, const int k, const int j, const int i) { auto rng_gen = rng_pool.get_state(); - Real dN_upd = wgtCfac * v(idN + sidx, k, j, i); + Real dN_upd = wgtCfac * v(0, mci::dN(sidx), k, j, i); int Ns = static_cast(dN_upd); if (dN_upd - Ns > rng_gen.drand()) { Ns++; } // TODO(BRR) Use a ParArrayND instead of these weird static_casts - v(iNs + sidx, k, j, i) = static_cast(Ns); + v(0, mci::Ns(sidx), k, j, i) = static_cast(Ns); rng_pool.free_state(rng_gen); }); int Nstot = 0; @@ -275,7 +267,7 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, parthenon::loop_pattern_mdrange_tag, "MonteCarloReduceParticleCreationNs", DevExecSpace(), 0, num_species - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int sidx, const int k, const int j, const int i, int &Nstot) { - Nstot += static_cast(v(iNs + sidx, k, j, i)); + Nstot += static_cast(v(0, mci::Ns(sidx), k, j, i)); }, Kokkos::Sum(Nstot)); @@ -284,22 +276,23 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, const auto new_particles_context = swarm->AddEmptyParticles(Nstot); - auto &t = swarm->Get("t").Get(); - auto &x = swarm->Get(swarm_position::x::name()).Get(); - auto &y = swarm->Get(swarm_position::y::name()).Get(); - auto &z = swarm->Get(swarm_position::z::name()).Get(); - auto &k0 = swarm->Get("k0").Get(); - auto &k1 = swarm->Get("k1").Get(); - auto &k2 = swarm->Get("k2").Get(); - auto &k3 = swarm->Get("k3").Get(); - auto &weight = swarm->Get("weight").Get(); - auto &swarm_species = swarm->Get("species").Get(); - auto swarm_d = swarm->GetDeviceContext(); + // monte carlo swarm pack + static constexpr auto swarm_name = "monte_carlo"; + static const auto desc_mc = + MakeSwarmPackDescriptor( + swarm_name); + auto pack_mc = desc_mc.GetPack(rc); + + // NOTE: Currently cannot pack Real and int valued swarms together, so species is + // separated + static const auto desc_species = MakeSwarmPackDescriptor(swarm_name); + auto pack_species = desc_species.GetPack(rc); // Calculate array of starting index for each zone to compute particles ParArrayND starting_index("Starting index", 3, nx_k, nx_j, nx_i); auto starting_index_h = starting_index.GetHostMirror(); - auto dN = rc->Get("Ns").data; + auto dN = rc->Get(mci::Ns::name()).data; auto dN_h = dN.GetHostMirrorAndCopy(); int index = 0; for (int sidx = 0; sidx < num_species; sidx++) { @@ -314,7 +307,7 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, } starting_index.DeepCopy(starting_index_h); - auto dNdlnu = rc->Get("dNdlnu").data; + auto dNdlnu = rc->Get(mci::dNdlnu::name()).data; // auto dNdlnu_max = rc->Get("dNdlnu_max").data; // Loop over zones and generate appropriate number of particles in each zone @@ -324,15 +317,17 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, pmb->par_for( "MonteCarloSourceParticles", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { + const auto swarm_d = pack_mc.GetContext(0); // Create tetrad transformation once per zone Real Gcov[4][4]; geom.SpacetimeMetric(CellLocation::Cent, k, j, i, Gcov); Real Ucon[4]; - Real vel[3] = {v(pvlo, k, j, i), v(pvlo + 1, k, j, i), v(pvlo + 2, k, j, i)}; + Real vel[3] = {v(0, p::velocity(0), k, j, i), v(0, p::velocity(1), k, j, i), + v(0, p::velocity(2), k, j, i)}; GetFourVelocity(vel, geom, CellLocation::Cent, k, j, i, Ucon); Geometry::Tetrads Tetrads(Ucon, Gcov); Real detG = geom.DetG(CellLocation::Cent, k, j, i); - int dNs = v(iNs + sidx, k, j, i); + int dNs = v(0, mci::Ns(sidx), k, j, i); auto rng_gen = rng_pool.get_state(); // Loop over particles to create in this zone @@ -341,15 +336,15 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, starting_index(sidx, k - kb.s, j - jb.s, i - ib.s) + n); // Set particle species - swarm_species(m) = static_cast(s); + pack_species(0, mcc::species(), m) = static_cast(s); // Create particles at initial time - t(m) = t0; + pack_mc(0, mcc::t(), m) = t0; // Create particles at zone centers - x(m) = minx_i + (i - ib.s + 0.5) * dx_i; - y(m) = minx_j + (j - jb.s + 0.5) * dx_j; - z(m) = minx_k + (k - kb.s + 0.5) * dx_k; + pack_mc(0, swarm_position::x(), m) = minx_i + (i - ib.s + 0.5) * dx_i; + pack_mc(0, swarm_position::y(), m) = minx_j + (j - jb.s + 0.5) * dx_j; + pack_mc(0, swarm_position::z(), m) = minx_k + (k - kb.s + 0.5) * dx_k; // Sample energy and set weight Real lnu; @@ -368,7 +363,7 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, } while (rng_gen.drand() > prob); Real nu = exp(lnu); - weight(m) = GetWeight(wgtC / wgtCfac, nu); + pack_mc(0, mcc::weight(), m) = GetWeight(wgtC / wgtCfac, nu); // Encode frequency and randomly sample direction Real E = nu * h_code; @@ -379,16 +374,18 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, Real K_coord[4]; Tetrads.TetradToCoordCov(K_tetrad, K_coord); - k0(m) = K_coord[0]; - k1(m) = K_coord[1]; - k2(m) = K_coord[2]; - k3(m) = K_coord[3]; + pack_mc(0, mcc::k0(), m) = K_coord[0]; + pack_mc(0, mcc::k1(), m) = K_coord[1]; + pack_mc(0, mcc::k2(), m) = K_coord[2]; + pack_mc(0, mcc::k3(), m) = K_coord[3]; - for (int mu = Gcov_lo; mu <= Gcov_lo + 3; mu++) { + for (int mu = 0; mu <= 3; mu++) { // detG is in both numerator and denominator - v(mu, k, j, i) -= 1. / (d3x * dt) * weight(m) * K_coord[mu - Gcov_lo]; + v(0, iv::Gcov(mu), k, j, i) -= + 1. / (d3x * dt) * pack_mc(0, mcc::weight(), m) * K_coord[mu]; } - v(Gye, k, j, i) -= LeptonSign(s) / (d3x * dt) * Ucon[0] * weight(m) * mp_code; + v(0, iv::Gye(), k, j, i) -= LeptonSign(s) / (d3x * dt) * Ucon[0] * + pack_mc(0, mcc::weight(), m) * mp_code; } // for n rng_pool.free_state(rng_gen); @@ -399,7 +396,8 @@ TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, pmb->par_for( "MonteCarloRemoveEmittedParticles", 0, num_species - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int sidx, const int k, const int j, const int i) { - int dNs = v(iNs + sidx, k, j, i); + const auto swarm_d = pack_mc.GetContext(0); + int dNs = v(0, mci::Ns(sidx), k, j, i); // Loop over particles to create in this zone for (int n = 0; n < static_cast(dNs); n++) { const int m = new_particles_context.GetNewParticleIndex( @@ -418,6 +416,7 @@ TaskStatus MonteCarloTransport(MeshBlock *pmb, MeshBlockData *rc, namespace p = fluid_prim; namespace c = fluid_cons; namespace iv = internal_variables; + namespace mcc = monte_carlo_core; auto rad = pmb->packages.Get("radiation"); auto swarm = sc->Get("monte_carlo"); auto opac = pmb->packages.Get("opacity"); @@ -437,17 +436,19 @@ TaskStatus MonteCarloTransport(MeshBlock *pmb, MeshBlockData *rc, const Real &dx_k = pmb->coords.Dxf<3>(pmb->cellbounds.ks(IndexDomain::interior)); const Real d4x = dx_i * dx_j * dx_k * dt; auto geom = Geometry::GetCoordinateSystem(rc); - auto &t = swarm->Get("t").Get(); - auto &x = swarm->Get(swarm_position::x::name()).Get(); - auto &y = swarm->Get(swarm_position::y::name()).Get(); - auto &z = swarm->Get(swarm_position::z::name()).Get(); - auto &k0 = swarm->Get("k0").Get(); - auto &k1 = swarm->Get("k1").Get(); - auto &k2 = swarm->Get("k2").Get(); - auto &k3 = swarm->Get("k3").Get(); - auto &weight = swarm->Get("weight").Get(); - auto &swarm_species = swarm->Get("species").Get(); - auto swarm_d = swarm->GetDeviceContext(); + + // monte carlo swarm pack + static constexpr auto swarm_name = "monte_carlo"; + static const auto desc_mc = + MakeSwarmPackDescriptor( + swarm_name); + auto pack_mc = desc_mc.GetPack(rc); + + // NOTE: Currently cannot pack Real and int valued swarms together, so species is + // separated + static const auto desc_species = MakeSwarmPackDescriptor(swarm_name); + auto pack_species = desc_species.GetPack(rc); auto phoebus_pkg = pmb->packages.Get("phoebus"); auto &unit_conv = phoebus_pkg->Param("unit_conv"); @@ -456,46 +457,52 @@ TaskStatus MonteCarloTransport(MeshBlock *pmb, MeshBlockData *rc, const Real h_code = code_constants.h; const Real mp_code = code_constants.mp; - std::vector vars({p::density::name(), p::ye::name(), p::velocity::name(), - p::temperature::name(), iv::Gcov::name(), - iv::Gye::name()}); - PackIndexMap imap; - auto v = rc->PackVariables(vars, imap); - const int prho = imap[p::density::name()].first; - const int iye = imap[p::ye::name()].first; - const int ivlo = imap[p::velocity::name()].first; - const int ivhi = imap[p::velocity::name()].second; - const int itemp = imap[p::temperature::name()].first; - const int iGcov_lo = imap[iv::Gcov::name()].first; - const int iGcov_hi = imap[iv::Gcov::name()].second; - const int iGye = imap[iv::Gye::name()].first; + Mesh *pmesh = rc->GetMeshPointer(); + auto &resolved_pkgs = pmesh->resolved_packages; + static auto desc = MakePackDescriptor(resolved_pkgs.get()); + auto v = desc.GetPack(rc); ParArray1D num_interactions("Number interactions", 2); pmb->par_for( - "MonteCarloTransport", 0, swarm->GetMaxActiveIndex(), KOKKOS_LAMBDA(const int n) { + "MonteCarloTransport", 0, pack_mc.GetMaxFlatIndex(), KOKKOS_LAMBDA(const int idx) { + const auto [b, n] = pack_mc.GetBlockParticleIndices(idx); + const auto swarm_d = pack_mc.GetContext(b); if (swarm_d.IsActive(n)) { + Real &x = pack_mc(b, swarm_position::x(), n); + Real &y = pack_mc(b, swarm_position::y(), n); + Real &z = pack_mc(b, swarm_position::z(), n); + Real &t = pack_mc(b, mcc::t(), n); + Real &k0 = pack_mc(b, mcc::k0(), n); + Real &k1 = pack_mc(b, mcc::k1(), n); + Real &k2 = pack_mc(b, mcc::k2(), n); + Real &k3 = pack_mc(b, mcc::k3(), n); + Real &weight = pack_mc(b, mcc::weight(), n); + auto rng_gen = rng_pool.get_state(); - auto s = static_cast(swarm_species(n)); + auto s = static_cast(pack_species(b, mcc::species(), n)); // TODO(BRR) Get u^mu, evaluate -k.u - const Real nu = -k0(n) / h_code; + const Real nu = -k0 / h_code; // TODO(BRR) Get K^0 via metric - Real Kcon0 = -k0(n); + Real Kcon0 = -k0; // Real dlam = dt / Kcon0; int k, j, i; - swarm_d.Xtoijk(x(n), y(n), z(n), i, j, k); + swarm_d.Xtoijk(x, y, z, i, j, k); - Real alphanu = opacities.AbsorptionCoefficient( - v(prho, k, j, i), v(itemp, k, j, i), v(iye, k, j, i), s, nu); + Real alphanu = opacities.AbsorptionCoefficient(v(b, p::density(), k, j, i), + v(b, p::temperature(), k, j, i), + v(b, p::ye(), k, j, i), s, nu); Real dtau_abs = alphanu * dt; // c = 1 in code units - Real vel[3] = {v(ivlo, k, j, i), v(ivlo + 1, k, j, i), v(ivlo + 2, k, j, i)}; - Real W = GetLorentzFactor(vel, geom, CellLocation::Cent, k, j, i); - Real alpha = geom.Lapse(CellLocation::Cent, k, j, i); + Real vel[3] = {v(b, p::velocity(0), k, j, i), v(b, p::velocity(1), k, j, i), + v(b, p::velocity(2), k, j, i)}; + Real W = GetLorentzFactor(vel, geom, CellLocation::Cent, b, k, j, i); + Real alpha = geom.Lapse(CellLocation::Cent, b, k, j, i); Real Ucon0 = robust::ratio(W, std::abs(alpha)); bool absorbed = false; @@ -506,15 +513,12 @@ TaskStatus MonteCarloTransport(MeshBlock *pmb, MeshBlockData *rc, Real xabs = -log(rng_gen.drand()); if (xabs <= dtau_abs) { // Deposit energy-momentum and lepton number in fluid - Kokkos::atomic_add(&(v(iGcov_lo, k, j, i)), 1. / d4x * weight(n) * k0(n)); - Kokkos::atomic_add(&(v(iGcov_lo + 1, k, j, i)), - 1. / d4x * weight(n) * k1(n)); - Kokkos::atomic_add(&(v(iGcov_lo + 2, k, j, i)), - 1. / d4x * weight(n) * k2(n)); - Kokkos::atomic_add(&(v(iGcov_lo + 3, k, j, i)), - 1. / d4x * weight(n) * k3(n)); - Kokkos::atomic_add(&(v(iGye, k, j, i)), - LeptonSign(s) / d4x * weight(n) * mp_code * Ucon0); + Kokkos::atomic_add(&(v(b, iv::Gcov(0), k, j, i)), 1. / d4x * weight * k0); + Kokkos::atomic_add(&(v(b, iv::Gcov(1), k, j, i)), 1. / d4x * weight * k1); + Kokkos::atomic_add(&(v(b, iv::Gcov(2), k, j, i)), 1. / d4x * weight * k2); + Kokkos::atomic_add(&(v(b, iv::Gcov(3), k, j, i)), 1. / d4x * weight * k3); + Kokkos::atomic_add(&(v(b, iv::Gye(), k, j, i)), + LeptonSign(s) / d4x * weight * mp_code * Ucon0); absorbed = true; Kokkos::atomic_add(&(num_interactions[0]), 1.); @@ -523,10 +527,10 @@ TaskStatus MonteCarloTransport(MeshBlock *pmb, MeshBlockData *rc, } if (absorbed == false) { - PushParticle(t(n), x(n), y(n), z(n), k0(n), k1(n), k2(n), k3(n), dt, geom); + PushParticle(t, x, y, z, k0, k1, k2, k3, dt, geom); bool on_current_mesh_block = true; - swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block); + swarm_d.GetNeighborBlockIndex(n, x, y, z, on_current_mesh_block); } rng_pool.free_state(rng_gen); diff --git a/src/radiation/radiation.cpp b/src/radiation/radiation.cpp index ac1d1b405..27bb9b8ff 100644 --- a/src/radiation/radiation.cpp +++ b/src/radiation/radiation.cpp @@ -62,12 +62,11 @@ std::shared_ptr Initialize(ParameterInput *pin) { bool is_frequency_dependent = (method == "monte_carlo" || method == "mocmc"); // Set which neutrino species to include in simulation - bool do_nu_electron = pin->GetOrAddBoolean("radiation", "do_nu_electron", true); + bool do_nu_electron = DO_NU_ELECTRON; params.Add("do_nu_electron", do_nu_electron); - bool do_nu_electron_anti = - pin->GetOrAddBoolean("radiation", "do_nu_electron_anti", true); + bool do_nu_electron_anti = DO_NU_ELECTRON_ANTI; params.Add("do_nu_electron_anti", do_nu_electron_anti); - bool do_nu_heavy = pin->GetOrAddBoolean("radiation", "do_nu_heavy", true); + bool do_nu_heavy = DO_NU_HEAVY; params.Add("do_nu_heavy", do_nu_heavy); // Boundary conditions @@ -96,22 +95,18 @@ std::shared_ptr Initialize(ParameterInput *pin) { params.Add("dlnu", dlnu); } - // NOTE(BLB): This will have to change if we ever support more than 3 species. - PARTHENON_REQUIRE(do_nu_electron + do_nu_electron_anti + do_nu_heavy == MOCMC_NUM_SPECIES, - "Requested species must be equal to compile time requested number of species."); // TODO(BLB) With compile time num species this can be a static array. std::vector species; - if (do_nu_electron) { + if (DO_NU_ELECTRON) { species.push_back(RadiationType::NU_ELECTRON); } - if (do_nu_electron_anti) { + if (DO_NU_ELECTRON_ANTI) { species.push_back(RadiationType::NU_ELECTRON_ANTI); } - if (do_nu_heavy) { + if (DO_NU_HEAVY) { species.push_back(RadiationType::NU_HEAVY); } - //const int num_species = species.size(); - static constexpr int num_species = MOCMC_NUM_SPECIES; + static constexpr int num_species = PHOEBUS_NUM_SPECIES; params.Add("num_species", num_species); params.Add("species", species); @@ -235,28 +230,31 @@ std::shared_ptr Initialize(ParameterInput *pin) { } if (method == "monte_carlo") { - std::string swarm_name = "monte_carlo"; + namespace mci = monte_carlo_internal; + namespace mcc = monte_carlo_core; + + constexpr static auto swarm_name = "monte_carlo"; Metadata swarm_metadata({Metadata::Provides}); physics->AddSwarm(swarm_name, swarm_metadata); Metadata real_swarmvalue_metadata({Metadata::Real}); - physics->AddSwarmValue("t", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("k0", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("k1", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("k2", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("k3", swarm_name, real_swarmvalue_metadata); - physics->AddSwarmValue("weight", swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(mcc::t::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(mcc::k0::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(mcc::k1::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(mcc::k2::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(mcc::k3::name(), swarm_name, real_swarmvalue_metadata); + physics->AddSwarmValue(mcc::weight::name(), swarm_name, real_swarmvalue_metadata); Metadata int_swarmvalue_metadata({Metadata::Integer}); - physics->AddSwarmValue("species", swarm_name, int_swarmvalue_metadata); + physics->AddSwarmValue(mcc::species::name(), swarm_name, int_swarmvalue_metadata); Metadata mspecies_scalar = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector{num_species}); - physics->AddField("dNdlnu_max", mspecies_scalar); - physics->AddField("dN", mspecies_scalar); - physics->AddField("Ns", mspecies_scalar); + physics->AddField(mci::dNdlnu_max::name(), mspecies_scalar); + physics->AddField(mci::dN::name(), mspecies_scalar); + physics->AddField(mci::Ns::name(), mspecies_scalar); std::vector dNdlnu_size{num_species, params.Get("nu_bins") + 1}; Metadata mdNdlnu = Metadata({Metadata::Cell, Metadata::OneCopy}, dNdlnu_size); - physics->AddField("dNdlnu", mdNdlnu); + physics->AddField(mci::dNdlnu::name(), mdNdlnu); // Parameters controlling automatic sampling resolution. // This system targets 1 scattering per light crossing time. @@ -562,7 +560,6 @@ TaskStatus ApplyRadiationFourForce(MeshData *rc, const double dt) { MakePackDescriptor( resolved_pkgs.get()); - PackIndexMap imap; auto v = desc.GetPack(rc); const int nblocks = v.GetNBlocks(); @@ -591,6 +588,7 @@ Real EstimateTimestepBlock(MeshBlockData *rc) { // Note that this is still used for the cooling function even though that option // contains no transport. This is useful for consistency between methods. + Mesh *pmesh = rc->GetMeshPointer(); auto pmb = rc->GetBlockPointer(); IndexRange ib = rc->GetBoundsI(IndexDomain::interior); IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); @@ -603,11 +601,9 @@ Real EstimateTimestepBlock(MeshBlockData *rc) { auto geom = Geometry::GetCoordinateSystem(rc); - PackIndexMap imap; - std::vector vars{ir::kappaH::name(), p::velocity::name()}; - auto v = rc->PackVariables(vars, imap); - auto idx_v = imap.GetFlatIdx(p::velocity::name()); - auto idx_kappaH = imap.GetFlatIdx(ir::kappaH::name(), false); + auto &resolved_pkgs = pmesh->resolved_packages; + static auto desc = MakePackDescriptor(resolved_pkgs.get()); + auto v = desc.GetPack(rc); auto num_species = rad->Param("num_species"); @@ -626,14 +622,15 @@ Real EstimateTimestepBlock(MeshBlockData *rc) { for (int ispec = 0; ispec < num_species; ispec++) { - const Real kappaH = idx_kappaH.IsValid() ? v(idx_kappaH(ispec), k, j, i) : 0.; + const Real kappaH = + v.Contains(0, ir::kappaH()) ? v(0, ir::kappaH(ispec), k, j, i) : 0.; for (int d = 0; d < ndim; d++) { // Signal speeds (assume (i.e. somewhat overestimate, esp. for large opt. // depth) cs_rad = 1) const Real sigp = alpha * std::sqrt(con_gamma(d, d)) - con_beta(d); const Real sigm = -alpha * std::sqrt(con_gamma(d, d)) - con_beta(d); - const Real asym_sigl = alpha * v(idx_v(d), k, j, i) - con_beta(d); + const Real asym_sigl = alpha * v(0, p::velocity(d), k, j, i) - con_beta(d); const Real rad_speed = std::max(std::fabs(sigm), std::fabs(sigp)); const Real asym_speed = std::fabs(asym_sigl); From 3f799b2bdf24b4100743668b6358f5e6cfefa26f Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Wed, 29 Jan 2025 19:54:07 +0000 Subject: [PATCH 34/48] BCs and input decks --- inputs/blandford_mckee.pin | 14 ++++++--- inputs/blast_wave.pin | 10 ++++--- inputs/blast_wave_2d.pin | 4 +-- inputs/bondi.pin | 14 +++++---- inputs/homogeneous_sphere.pin | 29 +++++++++---------- inputs/homologous.pin | 8 ++--- inputs/linear_modes.pin | 12 ++++---- inputs/mhd_shocktube.pin | 16 +++++----- inputs/progenitor.pin | 8 ++--- inputs/radiation_advection.pin | 3 -- inputs/radiation_equilibration.pin | 3 -- inputs/rhs_tester.pin | 16 +++++----- inputs/shocktube.pin | 8 ++--- inputs/shocktube_nonrel.pin | 16 +++++----- inputs/shocktube_stellarcollapse.pin | 16 +++++----- .../shocktube_stellarcollapse_faketable.pin | 14 ++++----- inputs/spherical_shock_tube.pin | 10 +++---- inputs/torus.pin | 16 +++++----- inputs/torus_cutout.pin | 16 +++++----- inputs/torus_rad.pin | 16 +++++----- inputs/tov-static.pin | 6 ++-- inputs/tov.pin | 10 +++---- inputs/tov_cart.pin | 24 +++++++-------- src/fixup/fixup.cpp | 12 ++++---- src/phoebus_boundaries/phoebus_boundaries.cpp | 6 ++-- 25 files changed, 154 insertions(+), 153 deletions(-) diff --git a/inputs/blandford_mckee.pin b/inputs/blandford_mckee.pin index 06d92a41e..4c431ba32 100644 --- a/inputs/blandford_mckee.pin +++ b/inputs/blandford_mckee.pin @@ -15,6 +15,12 @@ problem = blandford_mckee +ix1_bc = reflect +ox1_bc = gr_outflow +ix2_bc = reflect +ox2_bc = reflect +ix3_bc = periodic +ox3_bc = periodic problem_id = blast # problem ID: basename of output filenames @@ -50,14 +56,14 @@ nghost = 4 nx1 = 512 # Number of zones in X1-direction x1min = 0 # minimum value of X1 x1max = 1 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 3.14159265359 # maximum value of X2. Pi -ix2_bc = reflecting # Inner-X2 boundary condition flag -ox2_bc = reflecting # Outer-X2 boundary condition flag +ix2_bc = reflect # Inner-X2 boundary condition flag +ox2_bc = reflect # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/blast_wave.pin b/inputs/blast_wave.pin index b504a28e8..8ca004559 100644 --- a/inputs/blast_wave.pin +++ b/inputs/blast_wave.pin @@ -18,6 +18,8 @@ problem = sedov +ix1_bc = reflect +ox1_bc = gr_outflow ix2_bc = reflect ox2_bc = reflect @@ -55,14 +57,14 @@ nghost = 4 nx1 = 512 # Number of zones in X1-direction x1min = 0.0 # minimum value of X1 x1max = 0.3 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 3.14159265359 # maximum value of X2. Pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = reflect # Inner-X2 boundary condition flag +ox2_bc = reflect # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/blast_wave_2d.pin b/inputs/blast_wave_2d.pin index 8b8a2ed48..0c8080dd9 100644 --- a/inputs/blast_wave_2d.pin +++ b/inputs/blast_wave_2d.pin @@ -74,8 +74,8 @@ num_threads = 1 # maximum number of OMP threads bc_vars = conserved -nx1 = 32 -nx2 = 32 +nx1 = 64 +nx2 = 64 nx3 = 1 diff --git a/inputs/bondi.pin b/inputs/bondi.pin index bd8e8b081..d19e098e6 100644 --- a/inputs/bondi.pin +++ b/inputs/bondi.pin @@ -12,10 +12,12 @@ problem = bondi -ix1_bc = outflow -ox1_bc = outflow +ix1_bc = gr_outflow +ox1_bc = gr_outflow ix2_bc = reflect ox2_bc = reflect +ix3_bc = periodic +ox3_bc = periodic problem_id = bondi # problem ID: basename of output filenames @@ -51,14 +53,14 @@ nghost = 4 nx1 = 256 # Number of zones in X1-direction x1min = 0.59 # minimum value of X1 x1max = 5 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 1 #3.14159265359 #1 # maximum value of X2. Pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = reflect # Inner-X2 boundary condition flag +ox2_bc = reflect # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/homogeneous_sphere.pin b/inputs/homogeneous_sphere.pin index 8f9bbd4fe..38f0e5c91 100644 --- a/inputs/homogeneous_sphere.pin +++ b/inputs/homogeneous_sphere.pin @@ -13,7 +13,7 @@ problem = homogeneous_sphere ix1_bc = reflect -ox1_bc = outflow +ox1_bc = gr_outflow problem_id = hom_sph # problem ID: basename of output filenames @@ -39,17 +39,17 @@ nghost = 4 #refinement = adaptive #numlevel = 3 -nx1 = 128 # Number of zones in X1-direction -x1min = 0 # minimum value of X1 -x1max = 5 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +nx1 = 128 # Number of zones in X1-direction +x1min = 0 # minimum value of X1 +x1max = 5 # maximum value of X1 +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag -nx2 = 1 # Number of zones in X2-direction -x2min = 0 # minimum value of X2 -x2max = 3.14159265359 # maximum value of X2 -ix2_bc = periodic # Inner-X2 boundary condition flag -ox2_bc = periodic # Outer-X2 boundary condition flag +nx2 = 1 # Number of zones in X2-direction +x2min = 0 # minimum value of X2 +x2max = 3.14159265359 # maximum value of X2 +ix2_bc = periodic # Inner-X2 boundary condition flag +ox2_bc = periodic # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = -1 # minimum value of X3 @@ -98,10 +98,7 @@ nu_min = 1.e15 nu_max = 1.e22 nu_bins = 200 absorption = true -do_nu_electron = true -do_nu_electron_anti = false -do_nu_heavy = false - +:x B_fake = 1.0 use_B_fake = true @@ -110,7 +107,7 @@ type = gray gray_kappa = 1.e1 -J = 0.01 +J = 0.1 Hx = 0.01 Hy = 0.0 Hz = 0.0 diff --git a/inputs/homologous.pin b/inputs/homologous.pin index ca1327244..175244d81 100644 --- a/inputs/homologous.pin +++ b/inputs/homologous.pin @@ -64,14 +64,14 @@ numlevel = 3 nx1 = 10000 # Number of zones in X1-direction x1min = 0 # minimum value of X1 x1max = 0.10086475522209183 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = reflect # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 3.14159265359 # maximum value of X2. Pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = reflect # Inner-X2 boundary condition flag +ox2_bc = reflect # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/linear_modes.pin b/inputs/linear_modes.pin index 9ff2b0081..428a07433 100644 --- a/inputs/linear_modes.pin +++ b/inputs/linear_modes.pin @@ -42,13 +42,13 @@ nghost = 4 #refinement = adaptive #numlevel = 3 -nx1 = 128 # Number of zones in X1-direction +nx1 = 64 # Number of zones in X1-direction x1min = 0 # minimum value of X1 x1max = 1 # maximum value of X1 ix1_bc = periodic # Inner-X1 boundary condition flag ox1_bc = periodic # Outer-X1 boundary condition flag -nx2 = 128 # Number of zones in X2-direction +nx2 = 64 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 1 # maximum value of X2 ix2_bc = periodic # Inner-X2 boundary condition flag @@ -66,8 +66,8 @@ num_threads = 1 # maximum number of OMP threads #bc_vars = primitive -nx1 = 64 -nx2 = 64 +nx1 = 32 +nx2 = 32 # nx3 = 1 @@ -118,6 +118,6 @@ c2p_tol = 1.e-12 c2p_method = robust -physics = hydro -mode = sound +physics = mhd +mode = fast amplitude = 1.e-5 diff --git a/inputs/mhd_shocktube.pin b/inputs/mhd_shocktube.pin index c7aab1cec..e6599a5ee 100644 --- a/inputs/mhd_shocktube.pin +++ b/inputs/mhd_shocktube.pin @@ -12,8 +12,8 @@ problem = shock_tube -ix1_bc = outflow -ox1_bc = outflow +ix1_bc = gr_outflow +ox1_bc = gr_outflow problem_id = sod # problem ID: basename of output filenames @@ -43,13 +43,13 @@ dt_init_fact = 1.e-6 #numlevel = 3 nghost = 4 -nx1 = 256 # Number of zones in X1-direction -x1min = 0 # minimum value of X1 -x1max = 1 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +nx1 = 256 # Number of zones in X1-direction +x1min = 0 # minimum value of X1 +x1max = 1 # maximum value of X1 +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag -nx2 = 1 # Number of zones in X2-direction +nx2 = 1 # Number of zones in X2-direction x2min = -1 # minimum value of X2 x2max = 1 # maximum value of X2 ix2_bc = periodic # Inner-X2 boundary condition flag diff --git a/inputs/progenitor.pin b/inputs/progenitor.pin index 39ad7fe7b..67c4d8930 100644 --- a/inputs/progenitor.pin +++ b/inputs/progenitor.pin @@ -73,14 +73,14 @@ bc_vars = conserved nx1 = 8000 # Number of zones in X1-direction x1min = 0 # minimum value of X1 x1max = 1. # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = reflect # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 3.14159265359 # maximum value of X2. Pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = reflect # Inner-X2 boundary condition flag +ox2_bc = reflect # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/radiation_advection.pin b/inputs/radiation_advection.pin index 155f07845..6ec4c4176 100644 --- a/inputs/radiation_advection.pin +++ b/inputs/radiation_advection.pin @@ -108,9 +108,6 @@ nu_min = 1.e-2 nu_max = 1.e2 nu_bins = 10 absorption = true -do_nu_electron = true -do_nu_electron_anti = false -do_nu_heavy = false src_solver = oned diff --git a/inputs/radiation_equilibration.pin b/inputs/radiation_equilibration.pin index cfbb5c7ff..b6b2b9260 100644 --- a/inputs/radiation_equilibration.pin +++ b/inputs/radiation_equilibration.pin @@ -101,9 +101,6 @@ nu_min = 1.e-2 nu_max = 1.e2 nu_bins = 10 absorption = true -do_nu_electron = true -do_nu_electron_anti = false -do_nu_heavy = false type = gray diff --git a/inputs/rhs_tester.pin b/inputs/rhs_tester.pin index 43d68120a..fa237fb91 100644 --- a/inputs/rhs_tester.pin +++ b/inputs/rhs_tester.pin @@ -15,7 +15,7 @@ problem = rhs_tester ix1_bc = reflect -ox1_bc = outflow +ox1_bc = gr_outflow ix2_bc = reflect ox2_bc = reflect @@ -50,17 +50,17 @@ nghost = 4 #refinement = adaptive #numlevel = 3 -nx1 = 64 # Number of zones in X1-direction -x1min = 0 # minimum value of X1 -x1max = 1 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +nx1 = 64 # Number of zones in X1-direction +x1min = 0 # minimum value of X1 +x1max = 1 # maximum value of X1 +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 3.14159265359 # maximum value of X2. Pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = reflect # Inner-X2 boundary condition flag +ox2_bc = reflect # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/shocktube.pin b/inputs/shocktube.pin index dbb49ad15..ec775c20e 100644 --- a/inputs/shocktube.pin +++ b/inputs/shocktube.pin @@ -12,8 +12,8 @@ problem = shock_tube -ix1_bc = outflow -ox1_bc = outflow +ix1_bc = gr_outflow +ox1_bc = gr_outflow problem_id = sod # problem ID: basename of output filenames @@ -45,8 +45,8 @@ nghost = 4 nx1 = 512 # Number of zones in X1-direction x1min = 0 # minimum value of X1 x1max = 1 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = -1 # minimum value of X2 diff --git a/inputs/shocktube_nonrel.pin b/inputs/shocktube_nonrel.pin index af43eff6d..b0d6cd4e6 100644 --- a/inputs/shocktube_nonrel.pin +++ b/inputs/shocktube_nonrel.pin @@ -12,8 +12,8 @@ problem = shock_tube -ix1_bc = outflow -ox1_bc = outflow +ix1_bc = gr_outflow +ox1_bc = gr_outflow problem_id = sod # problem ID: basename of output filenames @@ -42,13 +42,13 @@ nghost = 4 #refinement = adaptive #numlevel = 3 -nx1 = 512 # Number of zones in X1-direction -x1min = 0 # minimum value of X1 -x1max = 1 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +nx1 = 512 # Number of zones in X1-direction +x1min = 0 # minimum value of X1 +x1max = 1 # maximum value of X1 +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag -nx2 = 1 # Number of zones in X2-direction +nx2 = 1 # Number of zones in X2-direction x2min = -1 # minimum value of X2 x2max = 1 # maximum value of X2 ix2_bc = periodic # Inner-X2 boundary condition flag diff --git a/inputs/shocktube_stellarcollapse.pin b/inputs/shocktube_stellarcollapse.pin index 4a96c2c48..f6cbefc97 100644 --- a/inputs/shocktube_stellarcollapse.pin +++ b/inputs/shocktube_stellarcollapse.pin @@ -15,8 +15,8 @@ problem = shock_tube -ix1_bc = outflow -ox1_bc = outflow +ix1_bc = gr_outflow +ox1_bc = gr_outflow problem_id = sod # problem ID: basename of output filenames @@ -45,13 +45,13 @@ nghost = 4 #refinement = adaptive #numlevel = 3 -nx1 = 512 # Number of zones in X1-direction -x1min = 0 # minimum value of X1 -x1max = 1 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +nx1 = 512 # Number of zones in X1-direction +x1min = 0 # minimum value of X1 +x1max = 1 # maximum value of X1 +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag -nx2 = 1 # Number of zones in X2-direction +nx2 = 1 # Number of zones in X2-direction x2min = -1 # minimum value of X2 x2max = 1 # maximum value of X2 ix2_bc = periodic # Inner-X2 boundary condition flag diff --git a/inputs/shocktube_stellarcollapse_faketable.pin b/inputs/shocktube_stellarcollapse_faketable.pin index a36eb05bd..8ab22a110 100644 --- a/inputs/shocktube_stellarcollapse_faketable.pin +++ b/inputs/shocktube_stellarcollapse_faketable.pin @@ -19,8 +19,8 @@ problem = shock_tube -ix1_bc = outflow -ox1_bc = outflow +ix1_bc = gr_outflow +ox1_bc = gr_outflow problem_id = sod # problem ID: basename of output filenames @@ -49,11 +49,11 @@ nghost = 4 #refinement = adaptive #numlevel = 3 -nx1 = 512 # Number of zones in X1-direction -x1min = 0 # minimum value of X1 -x1max = 1 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +nx1 = 512 # Number of zones in X1-direction +x1min = 0 # minimum value of X1 +x1max = 1 # maximum value of X1 +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = -1 # minimum value of X2 diff --git a/inputs/spherical_shock_tube.pin b/inputs/spherical_shock_tube.pin index 9865d978f..a5d40eb9a 100644 --- a/inputs/spherical_shock_tube.pin +++ b/inputs/spherical_shock_tube.pin @@ -25,7 +25,7 @@ problem = shock_tube ix1_bc = reflect -ox1_bc = outflow +ox1_bc = gr_outflow ix2_bc = reflect ox2_bc = reflect @@ -79,14 +79,14 @@ nghost = 4 nx1 = 512 # Number of zones in X1-direction x1min = 0 # minimum value of X1 x1max = 1 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 3.14159265359 # maximum value of X2. Pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = reflect # Inner-X2 boundary condition flag +ox2_bc = reflect # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/torus.pin b/inputs/torus.pin index 0341d22c4..87a040d3b 100644 --- a/inputs/torus.pin +++ b/inputs/torus.pin @@ -1,10 +1,10 @@ problem = torus -ix1_bc = outflow -ox1_bc = outflow -ix2_bc = outflow -ox2_bc = outflow +ix1_bc = gr_outflow +ox1_bc = gr_outflow +ix2_bc = gr_outflow +ox2_bc = gr_outflow bc_vars = primitive @@ -59,14 +59,14 @@ nghost = 4 nx1 = 256 # Number of zones in X1-direction x1min = 0.59 # minimum value of X1; overwritten for the torus problem x1max = 3.69 # maximum value of X1; set by /r_outer for the torus problem -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 256 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 1 # maximum value of X2. X2 = 1 -> theta = pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = gr_outflow # Inner-X2 boundary condition flag +ox2_bc = gr_outflow # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/torus_cutout.pin b/inputs/torus_cutout.pin index f651812ee..892f85e4e 100644 --- a/inputs/torus_cutout.pin +++ b/inputs/torus_cutout.pin @@ -1,10 +1,10 @@ problem = torus -ix1_bc = outflow -ox1_bc = outflow -ix2_bc = outflow -ox2_bc = outflow +ix1_bc = gr_outflow +ox1_bc = gr_outflow +ix2_bc = gr_outflow +ox2_bc = gr_outflow problem_id = torus # problem ID: basename of output filenames @@ -49,14 +49,14 @@ nghost = 4 nx1 = 128 # Number of zones in X1-direction x1min = 0.59 # minimum value of X1; overwritten for the torus problem x1max = 3.69 # maximum value of X1; set by /r_outer for the torus problem -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 128 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 1 # maximum value of X2. X2 = 1 -> theta = pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = gr_outflow # Inner-X2 boundary condition flag +ox2_bc = gr_outflow # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/torus_rad.pin b/inputs/torus_rad.pin index 484eed76a..efa327355 100644 --- a/inputs/torus_rad.pin +++ b/inputs/torus_rad.pin @@ -1,10 +1,10 @@ problem = torus -ix1_bc = outflow -ox1_bc = outflow -ix2_bc = outflow -ox2_bc = outflow +ix1_bc = gr_outflow +ox1_bc = gr_outflow +ix2_bc = gr_outflow +ox2_bc = gr_outflow bc_vars = primitive @@ -58,15 +58,15 @@ nx1 = 256 # Number of zones in X1-direction #nx1 = 128 x1min = 0.59 # minimum value of X1; overwritten for the torus problem x1max = 3.69 # maximum value of X1; set by /r_outer for the torus problem -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 256 # Number of zones in X2-direction #nx2 = 128 x2min = 0 # minimum value of X2 x2max = 1 # maximum value of X2. X2 = 1 -> theta = pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = gr_outflow # Inner-X2 boundary condition flag +ox2_bc = gr_outflow # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction #nx3 = 64 diff --git a/inputs/tov-static.pin b/inputs/tov-static.pin index 0b5118356..7938fd1b0 100644 --- a/inputs/tov-static.pin +++ b/inputs/tov-static.pin @@ -19,7 +19,7 @@ problem = tov ix1_bc = reflect -ox1_bc = outflow +ox1_bc = gr_outflow problem_id = tov # problem ID: basename of output filenames @@ -62,8 +62,8 @@ nghost = 4 nx1 = 1024 # Number of zones in X1-direction x1min = 0 # minimum value of X1 x1max = 250 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 diff --git a/inputs/tov.pin b/inputs/tov.pin index 10aae4b72..56686d663 100644 --- a/inputs/tov.pin +++ b/inputs/tov.pin @@ -15,7 +15,7 @@ problem = tov ix1_bc = reflect -ox1_bc = outflow +ox1_bc = gr_outflow ix2_bc = reflect ox2_bc = reflect @@ -64,14 +64,14 @@ nghost = 4 nx1 = 1024 # Number of zones in X1-direction x1min = 0 # minimum value of X1 x1max = 250 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = reflect # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 1 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 3.14159265359 # maximum value of X2. Pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = reflect # Inner-X2 boundary condition flag +ox2_bc = reflect # Outer-X2 boundary condition flag nx3 = 1 # Number of zones in X3-direction x3min = 0 # minimum value of X3 diff --git a/inputs/tov_cart.pin b/inputs/tov_cart.pin index 1b6e43f18..04e8e4b61 100644 --- a/inputs/tov_cart.pin +++ b/inputs/tov_cart.pin @@ -19,14 +19,14 @@ problem = tov # change inner BCs to reflect # and change domain to [0,xmax] -ix1_bc = outflow -ox1_bc = outflow +ix1_bc = gr_outflow +ox1_bc = gr_outflow -ix2_bc = outflow -ox2_bc = outflow +ix2_bc = gr_outflow +ox2_bc = gr_outflow -ix3_bc = outflow -ox3_bc = outflow +ix3_bc = gr_outflow +ox3_bc = gr_outflow @@ -76,20 +76,20 @@ nghost = 4 nx1 = 200 # Number of zones in X1-direction x1min = -100 # minimum value of X1 x1max = 100 # maximum value of X1 -ix1_bc = user # Inner-X1 boundary condition flag -ox1_bc = user # Outer-X1 boundary condition flag +ix1_bc = gr_outflow # Inner-X1 boundary condition flag +ox1_bc = gr_outflow # Outer-X1 boundary condition flag nx2 = 200 # Number of zones in X2-direction x2min = -100 # minimum value of X2 x2max = 100 # maximum value of X2. Pi -ix2_bc = user # Inner-X2 boundary condition flag -ox2_bc = user # Outer-X2 boundary condition flag +ix2_bc = gr_outflow # Inner-X2 boundary condition flag +ox2_bc = gr_outflow # Outer-X2 boundary condition flag nx3 = 200 # Number of zones in X3-direction x3min = -100 # minimum value of X3 x3max = 100 # maximum value of X3. 2*pi -ix3_bc = user # Inner-X3 boundary condition flag -ox3_bc = user # Outer-X3 boundary condition flfgag +ix3_bc = gr_outflow # Inner-X3 boundary condition flag +ox3_bc = gr_outflow # Outer-X3 boundary condition flfgag num_threads = 1 # maximum number of OMP threads diff --git a/src/fixup/fixup.cpp b/src/fixup/fixup.cpp index 6787e7cdf..ca95eebc9 100644 --- a/src/fixup/fixup.cpp +++ b/src/fixup/fixup.cpp @@ -751,7 +751,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { // x1-direction if (pmb->boundary_flag[BoundaryFace::inner_x1] == BoundaryFlag::user) { - if (ix1_bc == "outflow") { + if (ix1_bc == "gr_outflow") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes(std::vector({c::density::name()}), std::vector({c::density::name()}), @@ -787,7 +787,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { } } if (pmb->boundary_flag[BoundaryFace::outer_x1] == BoundaryFlag::user) { - if (ox1_bc == "outflow") { + if (ox1_bc == "gr_outflow") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes(std::vector({c::density::name()}), std::vector({c::density::name()}), @@ -826,7 +826,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { // x2-direction if (pmb->boundary_flag[BoundaryFace::inner_x2] == BoundaryFlag::user) { - if (ix2_bc == "outflow") { + if (ix2_bc == "gr_outflow") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes(std::vector({c::density::name()}), std::vector({c::density::name()}), @@ -908,7 +908,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { } } if (pmb->boundary_flag[BoundaryFace::outer_x2] == BoundaryFlag::user) { - if (ox2_bc == "outflow") { + if (ox2_bc == "gr_outflow") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes(std::vector({c::density::name()}), std::vector({c::density::name()}), @@ -994,7 +994,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { if (ndim == 2) return TaskStatus::complete; // x3-direction - if (ix3_bc == "outflow") { + if (ix3_bc == "gr_outflow") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes(std::vector({c::density::name()}), @@ -1026,7 +1026,7 @@ TaskStatus FixFluxes(MeshBlockData *rc) { } }); } - if (ox3_bc == "outflow") { + if (ox3_bc == "gr_outflow") { PackIndexMap imap; auto v = rc->PackVariablesAndFluxes(std::vector({c::density::name()}), diff --git a/src/phoebus_boundaries/phoebus_boundaries.cpp b/src/phoebus_boundaries/phoebus_boundaries.cpp index 82ec620ce..3255eb81d 100644 --- a/src/phoebus_boundaries/phoebus_boundaries.cpp +++ b/src/phoebus_boundaries/phoebus_boundaries.cpp @@ -476,9 +476,9 @@ void ProcessBoundaryConditions(parthenon::ParthenonManager &pman) { pman.pinput->GetOrAddBoolean("coordinates", "derefine_poles", true); if (derefine_poles) { const std::string ix2_bc = - pman.pinput->GetOrAddString("phoebus", "ix2_bc", "outflow"); + pman.pinput->GetOrAddString("phoebus", "ix2_bc", "gr_outflow"); const std::string ox2_bc = - pman.pinput->GetOrAddString("phoebus", "ox2_bc", "outflow"); + pman.pinput->GetOrAddString("phoebus", "ox2_bc", "gr_outflow"); PARTHENON_REQUIRE( ix2_bc != "reflect" && ix2_bc != "polar" && ox2_bc != "reflect" && ox2_bc != "polar", @@ -500,7 +500,7 @@ void ProcessBoundaryConditions(parthenon::ParthenonManager &pman) { // parthenon/mesh/" + // name); - const std::string bc = pman.pinput->GetOrAddString("phoebus", name, "outflow"); + const std::string bc = pman.pinput->GetOrAddString("phoebus", name, "gr_outflow"); if (bc == "reflect") { pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "reflect", reflect[d - 1][outer]); From 862de2d632382d17ac95d1eeb2572e608d5b0b28 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Wed, 29 Jan 2025 19:57:30 +0000 Subject: [PATCH 35/48] bump external --- external/parthenon | 2 +- external/singularity-eos | 2 +- external/singularity-opac | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/external/parthenon b/external/parthenon index d65f3e6b7..9b045aec7 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit d65f3e6b7f2f22150e18726be99c3422abcfcb43 +Subproject commit 9b045aec7ed88be4aab1337ed0a9839f50974a3c diff --git a/external/singularity-eos b/external/singularity-eos index b6bf01a01..e3738bc8b 160000 --- a/external/singularity-eos +++ b/external/singularity-eos @@ -1 +1 @@ -Subproject commit b6bf01a012a9ece27f15e49d33d060d2c678be1a +Subproject commit e3738bc8bd8d26e3b49adb302a4ae9477cdad33a diff --git a/external/singularity-opac b/external/singularity-opac index 0e1d38840..b05c6e12f 160000 --- a/external/singularity-opac +++ b/external/singularity-opac @@ -1 +1 @@ -Subproject commit 0e1d38840ae18b1ebec038c888545edbddc63c13 +Subproject commit b05c6e12f003edae4cea38f6aa279c395aa11098 From dc5326602851f6a10cd4e74b2f3b1efe1f581f76 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Wed, 29 Jan 2025 20:37:42 +0000 Subject: [PATCH 36/48] thread compile time neutrino options into actions --- .github/workflows/tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 67a558c2d..64572664f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,22 +32,22 @@ jobs: cd tst/regression mkdir build_minkowski_release cd build_minkowski_release - cmake -DCMAKE_BUILD_TYPE=Release -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=Minkowski -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON ../../../ + cmake -DCMAKE_BUILD_TYPE=Release -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=Minkowski -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON -DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off ../../../ cmake --build . --parallel 8 cd .. mkdir build_fmks_release cd build_fmks_release - cmake -DCMAKE_BUILD_TYPE=Release -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=FMKS -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON ../../../ + cmake -DCMAKE_BUILD_TYPE=Release -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=FMKS -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON .-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off ../../../ cmake --build . --parallel 8 cd .. mkdir build_minkowski_debug cd build_minkowski_debug - cmake -DCMAKE_BUILD_TYPE=Debug -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=Minkowski -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON ../../../ + cmake -DCMAKE_BUILD_TYPE=Debug -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=Minkowski -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON .-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off ../../../ cmake --build . --parallel 8 cd .. mkdir build_fmks_debug cd build_fmks_debug - cmake -DCMAKE_BUILD_TYPE=Debug -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=FMKS -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON ../../../ + cmake -DCMAKE_BUILD_TYPE=Debug -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=FMKS -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON .-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off ../../../ cmake --build . --parallel 8 cd .. # is there a run directory that shouldn't be here? @@ -79,7 +79,7 @@ jobs: run: | mkdir build cd build - cmake -DCMAKE_BUILD_TYPE=Release -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=Minkowski -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON ../ + cmake -DCMAKE_BUILD_TYPE=Release -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=Minkowski -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON .-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off ../ cmake --build . --parallel 4 - name: Run unit tests run: | From 482c992bcda2b2715c6e5dcd087f8e06df296657 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Wed, 29 Jan 2025 21:22:40 +0000 Subject: [PATCH 37/48] get lienar modes test and pin back in order --- inputs/linear_modes.pin | 12 +- tst/regression/linear_modes.gold | 2458 +++++++++++++++--------------- 2 files changed, 1235 insertions(+), 1235 deletions(-) diff --git a/inputs/linear_modes.pin b/inputs/linear_modes.pin index 428a07433..9ff2b0081 100644 --- a/inputs/linear_modes.pin +++ b/inputs/linear_modes.pin @@ -42,13 +42,13 @@ nghost = 4 #refinement = adaptive #numlevel = 3 -nx1 = 64 # Number of zones in X1-direction +nx1 = 128 # Number of zones in X1-direction x1min = 0 # minimum value of X1 x1max = 1 # maximum value of X1 ix1_bc = periodic # Inner-X1 boundary condition flag ox1_bc = periodic # Outer-X1 boundary condition flag -nx2 = 64 # Number of zones in X2-direction +nx2 = 128 # Number of zones in X2-direction x2min = 0 # minimum value of X2 x2max = 1 # maximum value of X2 ix2_bc = periodic # Inner-X2 boundary condition flag @@ -66,8 +66,8 @@ num_threads = 1 # maximum number of OMP threads #bc_vars = primitive -nx1 = 32 -nx2 = 32 +nx1 = 64 +nx2 = 64 # nx3 = 1 @@ -118,6 +118,6 @@ c2p_tol = 1.e-12 c2p_method = robust -physics = mhd -mode = fast +physics = hydro +mode = sound amplitude = 1.e-5 diff --git a/tst/regression/linear_modes.gold b/tst/regression/linear_modes.gold index 56b52b9c9..9d0566086 100644 --- a/tst/regression/linear_modes.gold +++ b/tst/regression/linear_modes.gold @@ -1,1232 +1,1232 @@ -1.000004666400361453e+00 -1.000002248011152339e+00 -9.999977404516279256e-01 -9.999953361638420635e-01 -9.999969712837875058e-01 -1.000001393321431120e+00 -1.000004592398310077e+00 -1.000003386213018208e+00 -9.999990615324131360e-01 -9.999955337770192187e-01 -9.999959715599420207e-01 -1.000000004744709070e+00 -1.000004023217759386e+00 -1.000004296599104503e+00 -1.000000465061607891e+00 -9.999962680697185302e-01 -9.999953748349462712e-01 -9.999986145838325458e-01 -1.000003047856125704e+00 -1.000004666399947117e+00 -1.000001825924000443e+00 -9.999973351656267218e-01 -9.999953333260460653e-01 -9.999973507013483198e-01 -1.000001834803263323e+00 -1.000004647713942374e+00 -1.000003028598724919e+00 -9.999986067435152615e-01 -9.999954076462023167e-01 -9.999962788503311550e-01 -1.000000474205735124e+00 -1.000004271376126797e+00 -1.000004028316689553e+00 -9.999999952551462679e-01 -9.999959768960008377e-01 -9.999954956180908683e-01 -9.999990698133630618e-01 -1.000003405033339510e+00 -1.000004659995234135e+00 -1.000001385351878991e+00 -9.999969522622452756e-01 -9.999953335994599124e-01 -9.999977518926730369e-01 -1.000002259646641800e+00 -1.000004663823589768e+00 -1.000002649190411930e+00 -9.999981652797611531e-01 -9.999953523026023916e-01 -9.999966136635332070e-01 -1.000000938512173088e+00 -1.000004466299231121e+00 -1.000004028316591409e+00 -9.999995258168943302e-01 -9.999957287239070736e-01 -9.999957032905217869e-01 -9.999995349155391677e-01 -1.000003732051085725e+00 -1.000004625133037894e+00 -1.000000930142094901e+00 -9.999965950888028976e-01 -9.999953399974228496e-01 -9.999981739944652226e-01 -1.000002664944579678e+00 -1.000004666686853616e+00 -1.000002248011124806e+00 -9.999977404516252610e-01 -9.999953361786149131e-01 -9.999969712837798452e-01 -1.000001393321430010e+00 -1.000004592398679337e+00 -1.000003721023358105e+00 -9.999990615324095833e-01 -9.999955337781045728e-01 -9.999959715597189769e-01 -1.000000004744709070e+00 -1.000004023217781368e+00 -1.000004504305695097e+00 -1.000000465061601895e+00 -9.999962680697194184e-01 -9.999953748341793291e-01 -9.999986145838234419e-01 -1.000003047856125260e+00 -1.000004666399358477e+00 -1.000002248011129691e+00 -9.999973351656280540e-01 -9.999953333195393812e-01 -9.999973507012623886e-01 -1.000001834803264211e+00 -1.000004647713058414e+00 -1.000003386212947820e+00 -9.999986067435119308e-01 -9.999954076461353702e-01 -9.999962788501154387e-01 -1.000000474205734013e+00 -1.000004271376119247e+00 -1.000004296598814957e+00 -9.999999952551494875e-01 -9.999959768959527651e-01 -9.999954956156673624e-01 -9.999990698133554012e-01 -1.000003405033344395e+00 -1.000004659995257006e+00 -1.000001825924003329e+00 -9.999969522622470519e-01 -9.999953336000279025e-01 -9.999977518926045361e-01 -1.000002259646642022e+00 -1.000004663827931184e+00 -1.000003028598684063e+00 -9.999981652797619303e-01 -9.999953522974022180e-01 -9.999966136631759372e-01 -1.000000938512173088e+00 -1.000004466299570849e+00 -1.000004028316538118e+00 -9.999999952551463789e-01 -9.999957287238498971e-01 -9.999957032903722398e-01 -9.999995349155340607e-01 -1.000003732051085059e+00 -1.000004625132793201e+00 -1.000001385351878769e+00 -9.999965950888151101e-01 -9.999953399980278101e-01 -9.999981739944291403e-01 -1.000002664944579234e+00 -1.000004666684458199e+00 -1.000002649190382398e+00 -9.999977404516281476e-01 -9.999953362018981773e-01 -9.999969712836818125e-01 -1.000001393321429566e+00 -1.000004592398526126e+00 -1.000003721023295711e+00 -9.999995258168982160e-01 -9.999955337806277766e-01 -9.999959715596077325e-01 -1.000000004744707960e+00 -1.000004023217778704e+00 -1.000004504304911279e+00 -1.000000930142098010e+00 -9.999962680697201955e-01 -9.999953748321477320e-01 -9.999986145838194451e-01 -1.000003047856125704e+00 -1.000004666399210818e+00 -1.000002248011114814e+00 -9.999977404516300350e-01 -9.999953332918282145e-01 -9.999973507012395180e-01 -1.000001834803263767e+00 -1.000004647712783967e+00 -1.000003386212862333e+00 -9.999990615324134691e-01 -9.999954076448548390e-01 -9.999962788498980570e-01 -1.000000474205733569e+00 -1.000004271376105924e+00 -1.000004296598671960e+00 -1.000000465061604116e+00 -9.999959768957805695e-01 -9.999954956105355786e-01 -9.999990698133564004e-01 -1.000003405033346837e+00 -1.000004659995051615e+00 -1.000001825923991783e+00 -9.999973351656272769e-01 -9.999953336051309316e-01 -9.999977518925733388e-01 -1.000002259646642244e+00 -1.000004663832038121e+00 -1.000003028598672739e+00 -9.999986067435135961e-01 -9.999953523000170152e-01 -9.999966136629715452e-01 -1.000000938512174642e+00 -1.000004466299900585e+00 -1.000004028316468618e+00 -9.999999952551471560e-01 -9.999959768958928130e-01 -9.999957032910258281e-01 -9.999995349155321733e-01 -1.000003732051085281e+00 -1.000004625132317360e+00 -1.000001385351880101e+00 -9.999969522622410567e-01 -9.999953399996251990e-01 -9.999981739943998305e-01 -1.000002664944579678e+00 -1.000004666682248633e+00 -1.000002649190380399e+00 -9.999981652797613751e-01 -9.999953361678434183e-01 -9.999969712837857294e-01 -1.000001393321430232e+00 -1.000004592398119119e+00 -1.000003721023209113e+00 -9.999995258168984380e-01 -9.999957287239132908e-01 -9.999959715603238264e-01 -1.000000004744704629e+00 -1.000004023217817339e+00 -1.000004504303755981e+00 -1.000000930142097788e+00 -9.999965950887820254e-01 -9.999953748355584482e-01 -9.999986145838228868e-01 -1.000003047856125926e+00 -1.000004666397982467e+00 -1.000002248011110373e+00 -9.999977404516288138e-01 -9.999953361605120605e-01 -9.999973507014099372e-01 -1.000001834803264211e+00 -1.000004647717442907e+00 -1.000003386212769074e+00 -9.999990615324144683e-01 -9.999955337764854235e-01 -9.999962788506364664e-01 -1.000000474205735790e+00 -1.000004271376145004e+00 -1.000004296598153486e+00 -1.000000465061604338e+00 -9.999962680697163098e-01 -9.999954956181416055e-01 -9.999990698133610634e-01 -1.000003405033335069e+00 -1.000004659994018219e+00 -1.000001825923998888e+00 -9.999973351656312737e-01 -9.999953333334294925e-01 -9.999977518926795872e-01 -1.000002259646640690e+00 -1.000004663816347117e+00 -1.000003028598642763e+00 -9.999986067435127080e-01 -9.999954076461047281e-01 -9.999966136634478309e-01 -1.000000938512173976e+00 -1.000004466298461736e+00 -1.000004028316343385e+00 -9.999999952551524851e-01 -9.999959768961173001e-01 -9.999954956188110700e-01 -9.999995349155367252e-01 -1.000003732051084615e+00 -1.000004625133427805e+00 -1.000001385351886984e+00 -9.999969522622440543e-01 -9.999953335954714362e-01 -9.999981739944518999e-01 -1.000002664944580122e+00 -1.000004666695629485e+00 -1.000002649190358417e+00 -9.999981652797603759e-01 -9.999953523091763552e-01 -9.999969712837353253e-01 -1.000001393321430454e+00 -1.000004592398613390e+00 -1.000003721023265291e+00 -9.999995258168957735e-01 -9.999957287239312764e-01 -9.999957032895571141e-01 -1.000000004744705739e+00 -1.000004023217897720e+00 -1.000004504306811981e+00 -1.000000930142101563e+00 -9.999965950887911292e-01 -9.999953399951342359e-01 -9.999986145838121177e-01 -1.000003047856125704e+00 -1.000004666395742259e+00 -1.000002248011120587e+00 -9.999977404516278146e-01 -9.999953361715600009e-01 -9.999969712837485369e-01 -1.000001834803264211e+00 -1.000004647717475992e+00 -1.000003386212974021e+00 -9.999990615324142462e-01 -9.999955337777335362e-01 -9.999959715592217080e-01 -1.000000474205733125e+00 -1.000004271376128795e+00 -1.000004296597990283e+00 -1.000000465061608335e+00 -9.999962680697170869e-01 -9.999953748338852311e-01 -9.999990698133466305e-01 -1.000003405033344395e+00 -1.000004659993012135e+00 -1.000001825924012655e+00 -9.999973351656316067e-01 -9.999953333197989513e-01 -9.999973507012055451e-01 -1.000002259646641578e+00 -1.000004663830784013e+00 -1.000003028598643873e+00 -9.999986067435108206e-01 -9.999954076455773722e-01 -9.999962788498614197e-01 -1.000000938512174198e+00 -1.000004466300216555e+00 -1.000004028316062055e+00 -9.999999952551527072e-01 -9.999959768960148265e-01 -9.999954956159711195e-01 -9.999990698133480738e-01 -1.000003732051082173e+00 -1.000004625131559965e+00 -1.000001385351883876e+00 -9.999969522622428331e-01 -9.999953335986523362e-01 -9.999977518925946551e-01 -1.000002664944577235e+00 -1.000004666674098930e+00 -1.000002649190306903e+00 -9.999981652797607090e-01 -9.999953523057741878e-01 -9.999966136632807423e-01 -1.000001393321430010e+00 -1.000004592398435976e+00 -1.000003721022906022e+00 -9.999995258168977719e-01 -9.999957287239165105e-01 -9.999957032900425036e-01 -9.999995349155347268e-01 -1.000004023217651250e+00 -1.000004504303320774e+00 -1.000000930142100897e+00 -9.999965950887975685e-01 -9.999953399963393830e-01 -9.999981739944453496e-01 -1.000003047856124594e+00 -1.000004666403346176e+00 -1.000002248011097494e+00 -9.999977404516284807e-01 -9.999953361761605430e-01 -9.999969712837305513e-01 -1.000001393321430676e+00 -1.000004647707787520e+00 -1.000003386212811485e+00 -9.999990615324095833e-01 -9.999955337783269504e-01 -9.999959715595423404e-01 -1.000000004744708182e+00 -1.000004271376105480e+00 -1.000004296599532827e+00 -1.000000465061610777e+00 -9.999962680697176420e-01 -9.999953748335558279e-01 -9.999986145838278828e-01 -1.000003405033346837e+00 -1.000004659997403511e+00 -1.000001825924002663e+00 -9.999973351656301634e-01 -9.999953333120960020e-01 -9.999973507012649421e-01 -1.000001834803264211e+00 -1.000004663829241025e+00 -1.000003028598694277e+00 -9.999986067435129300e-01 -9.999954076457778784e-01 -9.999962788499922040e-01 -1.000000474205734680e+00 -1.000004466299469152e+00 -1.000004028317039495e+00 -9.999999952551497095e-01 -9.999959768958892603e-01 -9.999954956152694585e-01 -9.999990698133587319e-01 -1.000003405033342174e+00 -1.000004625133476432e+00 -1.000001385351894312e+00 -9.999969522622431661e-01 -9.999953336026468076e-01 -9.999977518926256304e-01 -1.000002259646642688e+00 -1.000004666687664301e+00 -1.000002649190435244e+00 -9.999981652797605980e-01 -9.999953523010406409e-01 -9.999966136633081648e-01 -1.000000938512173532e+00 -1.000004592398902270e+00 -1.000003721023604131e+00 -9.999995258168995482e-01 -9.999957287239137349e-01 -9.999957032910183896e-01 -9.999995349155389457e-01 -1.000003732051084615e+00 -1.000004504305699982e+00 -1.000000930142109112e+00 -9.999965950887942379e-01 -9.999953399989767178e-01 -9.999981739944453496e-01 -1.000002664944578346e+00 -1.000004666399634923e+00 -1.000002248011125916e+00 -9.999977404516260382e-01 -9.999953361699313037e-01 -9.999969712837786240e-01 -1.000001393321428900e+00 -1.000004592398641368e+00 -1.000003386212862111e+00 -9.999990615324135801e-01 -9.999955337772126196e-01 -9.999959715601252075e-01 -1.000000004744708626e+00 -1.000004023217752946e+00 --9.294055605829665049e-07 --1.030291957496450776e-07 -8.046703979068646837e-07 -1.000549553077626191e-06 -3.019937159876685764e-07 --6.562109180205328918e-07 --1.006598455220162532e-06 --3.971066828603020044e-07 -5.733964941424576805e-07 -1.007349474896367298e-06 -5.745175580737134868e-07 --3.941875256883406693e-07 --1.005080896372813576e-06 --6.560460356071897101e-07 -2.988280090240379087e-07 -9.952472322291836494e-07 -8.048859651250844561e-07 --1.001443213029248955e-07 --9.232992404455057782e-07 --8.711308617708640842e-07 --1.404958832198173164e-09 -8.682932468611122998e-07 -9.747824518831289863e-07 -2.036114893984304378e-07 --7.336134015618541773e-07 --1.006598655095421562e-06 --3.019790656009647984e-07 -6.561857221984301149e-07 -1.007380094339941091e-06 -4.881467828113725560e-07 --4.859015582911342406e-07 --1.007082731718890411e-06 --5.744946763633820308e-07 -3.941703885595222443e-07 -1.005077803376744554e-06 -7.328936611983004574e-07 --2.005485973645621850e-07 --9.676535229571514956e-07 --8.048589854925684981e-07 -1.001409230956606669e-07 -9.232768255384944365e-07 -9.294280201533293177e-07 -1.030357151124987955e-07 --8.046964356243842865e-07 --1.000543221413674344e-06 --2.036006000799612629e-07 -7.335872418718312566e-07 -1.006598822150295398e-06 -3.971244869449107467e-07 --5.734197853180035611e-07 --1.007351720028704540e-06 --4.881262794805896256e-07 -4.858809806757326190e-07 -1.007081825336202509e-06 -6.560708920619997719e-07 --2.988410452247087619e-07 --9.952571243814315649e-07 --8.048589497438283035e-07 -2.005402049912768811e-07 -9.676356082333004592e-07 -8.711572815316465394e-07 -1.406631894765357807e-09 --8.683180606476139550e-07 --9.747665153267913961e-07 --1.030291939310566975e-07 -8.046704067551571979e-07 -1.000548532352946841e-06 -3.019937204881005265e-07 --6.562109176569878331e-07 --1.007381271481910094e-06 --3.971066810608646289e-07 -5.733964953263467370e-07 -1.007350482720288475e-06 -5.745175809304583819e-07 --3.941875258143732426e-07 --1.005080964527917202e-06 --7.328675160914734301e-07 -2.988280095462563788e-07 -9.952476620641745090e-07 -8.048859608725118163e-07 --1.001443209425435050e-07 --9.232992367045987427e-07 --9.294055980330326193e-07 --1.404959517640701816e-09 -8.682932773388525434e-07 -9.747826957941115791e-07 -2.036114952335989597e-07 --7.336134023099747463e-07 --1.006598701388902975e-06 --3.971066847334067353e-07 -6.561857226187680973e-07 -1.007380188439154189e-06 -4.881468076402621446e-07 --4.859015583827707117e-07 --1.007082837263389934e-06 --6.560460353188880705e-07 -3.941703882469833392e-07 -1.005078004821177435e-06 -7.328938038143404117e-07 --2.005485965280355433e-07 --9.676536526718752940e-07 --8.711308358705835906e-07 -1.001409227611797836e-07 -9.232768064746382478e-07 -9.294284726415213611e-07 -1.030357178216759599e-07 --8.046964334422135385e-07 --1.000543486380220574e-06 --3.019790657874176612e-07 -7.335872359921279128e-07 -1.006599374420541780e-06 -3.971244997196466696e-07 --5.734197851770628903e-07 --1.007351405861018618e-06 --5.744946709092609681e-07 -4.858809805013788277e-07 -1.007081014439225773e-06 -6.560709161344415048e-07 --2.988410443891016538e-07 --9.952569826471431678e-07 --8.048589436051752351e-07 -1.001409222746548451e-07 -9.676348249515108273e-07 -8.711572942806880884e-07 -1.406631817966765624e-09 --8.683180537623393543e-07 --9.747664592654431377e-07 --2.036006005066277991e-07 -8.046704244787415075e-07 -1.000547438048910773e-06 -3.019937219917811019e-07 --6.562109182992073667e-07 --1.007381223111862613e-06 --4.881262767194982221e-07 -5.733964955760497205e-07 -1.007352598026047830e-06 -5.745175763699886305e-07 --3.941875254487432124e-07 --1.005080934378242984e-06 --7.328674761556064905e-07 -2.005402049373490197e-07 -9.952490497271137940e-07 -8.048858539997143893e-07 --1.001443202768902755e-07 --9.232992286496215109e-07 --9.294054590202618228e-07 --1.030291953709523303e-07 -8.682933990494582503e-07 -9.747827016462158366e-07 -2.036114941945643957e-07 --7.336134030057371382e-07 --1.006598732942832460e-06 --3.971066803716042206e-07 -5.733964963353193606e-07 -1.007381093907491412e-06 -4.881468083266627049e-07 --4.859015574202071292e-07 --1.007082892318401641e-06 --6.560460319610472384e-07 -2.988280080283115568e-07 -1.005078475712284694e-06 -7.328939380294382444e-07 --2.005485965233618537e-07 --9.676537729434291982e-07 --8.711308463507602519e-07 --1.404958758512861952e-09 -9.232767224797481987e-07 -9.294295603992337567e-07 -1.030357181236277737e-07 --8.046964303023859640e-07 --1.000543566924423863e-06 --3.019790665356147808e-07 -6.561857218223741678e-07 -1.006600252622787633e-06 -3.971245013732760583e-07 --5.734197844482541431e-07 --1.007351112727576302e-06 --5.744946719570341593e-07 -3.941703888336498463e-07 -1.007081254426845351e-06 -6.560708973246380255e-07 --2.988410449280466424e-07 --9.952568583960853288e-07 --8.048589446686297591e-07 -1.001409223708378053e-07 -9.232768318617425597e-07 -8.711570877529253580e-07 -1.406631382096227332e-09 --8.683180591714414629e-07 --9.747664209953362632e-07 --2.036006006470662853e-07 -7.335872463039276591e-07 -1.000548577095291758e-06 -3.019937124766091721e-07 --6.562109174690844793e-07 --1.007381201659390136e-06 --4.881262780308053861e-07 -4.858809794166910936e-07 -1.007349541517549313e-06 -5.745175345504756350e-07 --3.941875254938029300e-07 --1.005081017746366662e-06 --7.328674576769726707e-07 -2.005402047787423386e-07 -9.676361943290861294e-07 -8.048857393434741411e-07 --1.001443217229112800e-07 --9.232991899817397058e-07 --9.294053279097822050e-07 --1.030291956228264048e-07 -8.046703946814131955e-07 -9.747825369079970173e-07 -2.036114859916464274e-07 --7.336133983415780604e-07 --1.006599312755879993e-06 --3.971066810618459166e-07 -5.733964940734734114e-07 -1.007349051460694473e-06 -4.881467767607113878e-07 --4.859015582501338225e-07 --1.007082136642869904e-06 --6.560460377401635139e-07 -2.988280091995979807e-07 -9.952468567128696503e-07 -7.328937091375577813e-07 --2.005485978997841197e-07 --9.676532399336291053e-07 --8.711308755560668499e-07 --1.404959259262984853e-09 -8.682931914722783436e-07 -9.294278784563814152e-07 -1.030357146174803797e-07 --8.046964394762864135e-07 --1.000542599308030562e-06 --3.019790680866183458e-07 -6.561857225577792899e-07 -1.007379813889062744e-06 -3.971244933289400484e-07 --5.734197853806958577e-07 --1.007352364527693350e-06 --5.744946714740483495e-07 -3.941703884418093874e-07 -1.005077455138730845e-06 -6.560709097079438769e-07 --2.988410445870692943e-07 --9.952575760200046695e-07 --8.048588314377094271e-07 -1.001409232286070540e-07 -9.232769242132966126e-07 -9.294277527649889421e-07 -1.406632600900779972e-09 --8.683181184598553366e-07 --9.747664873476702331e-07 --2.036006022580282950e-07 -7.335872480031750659e-07 -1.006597553859811437e-06 -3.019937247961385248e-07 --6.562109174122158519e-07 --1.007381570085975475e-06 --4.881262837657042124e-07 -4.858809799792420963e-07 -1.007082840755948180e-06 -5.745175990700788054e-07 --3.941875254435710704e-07 --1.005081271373953455e-06 --7.328675606231233482e-07 -2.005402042525645453e-07 -9.676359117096829464e-07 -8.711573206991186470e-07 --1.001443197585308346e-07 --9.232991430965190641e-07 --9.294059393448317663e-07 --1.030291967012917025e-07 -8.046704058239257876e-07 -1.000549640675806998e-06 -2.036114966551241155e-07 --7.336133990798837410e-07 --1.006599892045066235e-06 --3.971066878600229781e-07 -5.733964951241903598e-07 -1.007350114803998982e-06 -5.745175943436896992e-07 --4.859015579621007982e-07 --1.007082106238151100e-06 --6.560460369184712513e-07 -2.988280092777632399e-07 -9.952475812636651914e-07 -8.048861864825506316e-07 --2.005485963684203371e-07 --9.676536779881982506e-07 --8.711308052841336562e-07 --1.404959423938335499e-09 -8.682932579749920019e-07 -9.747826190325700268e-07 -1.030357164468299561e-07 --8.046964274718452703e-07 --1.000542868051341273e-06 --3.019790651361441396e-07 -6.561857231891525666e-07 -1.007380232515156921e-06 -4.881468087301507669e-07 --5.734197848813358422e-07 --1.007350950294399677e-06 --5.744946517783403105e-07 -3.941703877627450853e-07 -1.005077800626097384e-06 -7.328937381305914640e-07 --2.988410443342520883e-07 --9.952564819183330304e-07 --8.048587592066878117e-07 -1.001409222760839353e-07 -9.232768647194232507e-07 -9.294283447542539941e-07 -1.030357170321422826e-07 --8.683179910261983592e-07 --9.747664841496742591e-07 --2.036005970270483214e-07 -7.335872446343642043e-07 -1.006598370965810515e-06 -3.971244919329538360e-07 --6.562109187731836047e-07 --1.007380870163989117e-06 --4.881262716706710956e-07 -4.858809802198892388e-07 -1.007082267466381430e-06 -6.560708945169878679e-07 --3.941875255511650599e-07 --1.005080570409457157e-06 --7.328674796952083110e-07 -2.005402040854753321e-07 -9.676356970333445729e-07 -8.711572222901105394e-07 -1.406631048658445245e-09 --9.232993319839070370e-07 --9.294050849430872943e-07 --1.030291957977054491e-07 -8.046704085642496373e-07 -1.000549175795806904e-06 -3.019937190892757479e-07 --7.336134064288557290e-07 --1.006597490885001862e-06 --3.971066861696735465e-07 -5.733964954294078827e-07 -1.007350584890920456e-06 -5.745175783441651443e-07 --3.941875256004935589e-07 --1.007083653283044906e-06 --6.560460501217274421e-07 -2.988280085134882174e-07 -9.952479636465961965e-07 -8.048860351608119245e-07 --1.001443210646145978e-07 --9.676537756389315584e-07 --8.711309078986095250e-07 --1.404960017817508730e-09 -8.682933158663948593e-07 -9.747825454252580444e-07 -2.036114933599784652e-07 --8.046964350132957548e-07 --1.000544261483735576e-06 --3.019790716176576691e-07 -6.561857219740708865e-07 -1.007380514786610828e-06 -4.881467987234624526e-07 --4.859015576518420529e-07 --1.007351444534011420e-06 --5.744946932721045515e-07 -3.941703882120694352e-07 -1.005078113667816480e-06 -7.328937513922214597e-07 --2.005485968202532280e-07 --9.952571063988636494e-07 --8.048591533933414212e-07 -1.001409216825303091e-07 -9.232767563823591207e-07 -9.294286160761272853e-07 -1.030357163397571565e-07 --8.046964356567483566e-07 --9.747664445262941850e-07 --2.036006033225554009e-07 -7.335872414428896429e-07 -1.006599692269014033e-06 -3.971244920932981258e-07 --5.734197848051192882e-07 --1.007381331126183156e-06 --4.881262836231832957e-07 -4.858809798281510061e-07 -1.007081504753755236e-06 -6.560708926547023947e-07 --2.988410448544338677e-07 --1.005080931508151039e-06 --7.328674526781186881e-07 -2.005402038085742265e-07 -9.676358278649439926e-07 -8.711571999531510123e-07 -1.406631014538256826e-09 --8.683180467752300057e-07 --9.294055321418282562e-07 --1.030291960466442633e-07 -8.046703991799883436e-07 -1.000548531789185217e-06 -3.019937172054097811e-07 --6.562109184404555105e-07 --1.006598506884153856e-06 --3.971066766370785196e-07 -5.733964943905043969e-07 -1.007349754892116208e-06 -5.745175589400944665e-07 --3.941875256656648562e-07 --1.005080896632132925e-06 --2.000164600401336329e-06 -9.222447130126748175e-07 -3.027257120328875046e-06 -2.459197049475813440e-06 --3.152100988591829596e-07 --2.818282960320236686e-06 --2.662240623918497307e-06 -5.177599967053031403e-09 -2.656525704776692750e-06 -2.975242484345936160e-06 -6.122169816257862666e-07 --2.249847544735278521e-06 --3.055508536200097404e-06 --9.128700923248491102e-07 -2.014347162471758928e-06 -3.087115942545467270e-06 -1.483209004082876766e-06 --1.494393881401377758e-06 --3.083622528824594606e-06 --1.748729129417257092e-06 -1.213820143808149941e-06 -3.068911928744286609e-06 -2.237352580735936842e-06 --6.218751953906710184e-07 --2.943681922858387149e-06 --2.459117958842387412e-06 -3.151955302699433135e-07 -2.818222717423963391e-06 -2.838920677095309822e-06 -3.050739631878120801e-07 --2.465265907912817365e-06 --3.055508255721094174e-06 --6.121897148371430959e-07 -2.249771371000582817e-06 -3.081780049479383614e-06 -1.204066102358552120e-06 --1.761878267187738988e-06 --3.087022490170425038e-06 --1.483149266023190730e-06 -1.494332629405549339e-06 -3.083619667655742292e-06 -2.000237981411015913e-06 --9.222855630942051851e-07 --3.027284067541193195e-06 --2.237274972999872001e-06 -6.218469306796444209e-07 -2.943636432669303082e-06 -2.662316940973670670e-06 --5.177918822022096318e-09 --2.656595776902332376e-06 --2.975193911082368820e-06 --3.050601395245967083e-07 -2.465190717755353298e-06 -3.055529748302201587e-06 -9.129096337423613593e-07 --2.014420931548350863e-06 --3.087117304726884433e-06 --1.204015689111762063e-06 -1.761809630570721700e-06 -3.087021664989664050e-06 -1.748796496204983334e-06 --1.213872066081707796e-06 --3.068920138318293335e-06 --2.237274958057157526e-06 -9.222447132503667680e-07 -3.027256901262132608e-06 -2.459197096410802667e-06 --3.152100983541052070e-07 --2.818282959911827469e-06 --2.838852490409840987e-06 -5.177600505655263596e-09 -2.656525683350705087e-06 -2.975242007791355705e-06 -6.122169799780032052e-07 --2.249847545124946288e-06 --3.081776648797771915e-06 --9.128700894731423337e-07 -2.014347162091129003e-06 -3.087117174540557254e-06 -1.483209017073808584e-06 --1.494393881331721580e-06 --3.083623171896271279e-06 --2.000164572751261632e-06 -1.213820143878340174e-06 -3.068910229705429056e-06 -2.237352623523504629e-06 --6.218751950443644567e-07 --2.943681900171466007e-06 --2.662240627546797242e-06 -3.151955308460338202e-07 -2.818222699243139368e-06 -2.838920541928440347e-06 -3.050739647337500175e-07 --2.465265907215793529e-06 --3.055508336973947250e-06 --9.128700910339584048e-07 -2.249771371083119825e-06 -3.081780839428713369e-06 -1.204066121114378954e-06 --1.761878266983632000e-06 --3.087024015440775005e-06 --1.748729112813366588e-06 -1.494332629395895281e-06 -3.083622466934211245e-06 -2.000238032798529119e-06 --9.222855628500002924e-07 --3.027284110262618297e-06 --2.459117896043975414e-06 -6.218469306425458601e-07 -2.943636644764209658e-06 -2.662317353357709279e-06 --5.177917465497133916e-09 --2.656595783455085982e-06 --2.975194029825672211e-06 --6.121897147847749663e-07 -2.465190724087207064e-06 -3.055529535098033792e-06 -9.129096465147600536e-07 --2.014420931540297274e-06 --3.087116887118177198e-06 --1.483149255993005404e-06 -1.761809630260435109e-06 -3.087015480294009457e-06 -1.748796544150532207e-06 --1.213872066043083517e-06 --3.068920639760422104e-06 --2.237274950582254820e-06 -6.218469304994584902e-07 -3.027257267425890604e-06 -2.459197135874444185e-06 --3.152100980212754159e-07 --2.818282959618253049e-06 --2.838852514580857026e-06 --3.050601398378343097e-07 -2.656525652810296052e-06 -2.975241609550027586e-06 -6.122169866280836498e-07 --2.249847545214697475e-06 --3.081776340579127877e-06 --1.204015683978650500e-06 -2.014347161035999542e-06 -3.087120452268125444e-06 -1.483209031757313825e-06 --1.494393881446237258e-06 --3.083623429701967304e-06 --2.000164558067580843e-06 -9.222447130660705040e-07 -3.068900199591174303e-06 -2.237352665296288405e-06 --6.218751952323854422e-07 --2.943681909310511898e-06 --2.662240490951830218e-06 -5.177600786082367972e-09 -2.818222666889502149e-06 -2.838920728856203197e-06 -3.050739660439167046e-07 --2.465265906177216363e-06 --3.055508223238704983e-06 --9.128700881822972623e-07 -2.014347160967820166e-06 -3.081785459456511538e-06 -1.204066128157341443e-06 --1.761878267217374977e-06 --3.087024740964965139e-06 --1.748729098693308934e-06 -1.213820143620283751e-06 -3.083630587141290704e-06 -2.000238093724568858e-06 --9.222855630673673697e-07 --3.027284089181581838e-06 --2.459117869304699151e-06 -3.151955308194736198e-07 -2.943636738334340566e-06 -2.662318416699685006e-06 --5.177917600794576775e-09 --2.656595791362026736e-06 --2.975193987062474409e-06 --6.121897144822575727e-07 -2.249771372731092304e-06 -3.055529111826160834e-06 -9.129096456882073042e-07 --2.014420931653842464e-06 --3.087116317992878315e-06 --1.483149253582620687e-06 -1.494332629186293166e-06 -3.087023742968697569e-06 -1.748796567853142202e-06 --1.213872066153680820e-06 --3.068920516935931861e-06 --2.237274936286490558e-06 -6.218469306368714806e-07 -2.943636290768439938e-06 -2.459196933198562618e-06 --3.152100985427826960e-07 --2.818282945172002372e-06 --2.838852432368107675e-06 --3.050601398823266211e-07 -2.465190714646088049e-06 -2.975241815909446647e-06 -6.122169807437946814e-07 --2.249847545645638616e-06 --3.081776131829732087e-06 --1.204015683769788268e-06 -1.761809630582506893e-06 -3.087115586178412298e-06 -1.483209004765927734e-06 --1.494393881444599944e-06 --3.083626888847235096e-06 --2.000164541566777214e-06 -9.222447132300103428e-07 -3.027256959525581029e-06 -2.237352540054578736e-06 --6.218751952735721016e-07 --2.943682137430753711e-06 --2.662240279547135640e-06 -5.177600548371468480e-09 -2.656525708168374454e-06 -2.838920367487901057e-06 -3.050739615392967058e-07 --2.465265910652078294e-06 --3.055507693310671003e-06 --9.128700888621563989e-07 -2.014347162791535465e-06 -3.087115334546631429e-06 -1.204066099747427159e-06 --1.761878267186252869e-06 --3.087019783906130566e-06 --1.748729085519497409e-06 -1.213820143621742554e-06 -3.068915747898269982e-06 -2.000237948703548045e-06 --9.222855631041150470e-07 --3.027284153417684457e-06 --2.459117771558093995e-06 -3.151955305534898161e-07 -2.818222755331747264e-06 -2.662316837887460871e-06 --5.177919575905114790e-09 --2.656595770028137389e-06 --2.975193541927646431e-06 --6.121897160717602145e-07 -2.249771369899464973e-06 -3.081778638098394829e-06 -9.129096343552982227e-07 --2.014420931247520759e-06 --3.087118143518693237e-06 --1.483149252050013248e-06 -1.494332629271366614e-06 -3.083611191664898888e-06 -1.748796516735614396e-06 --1.213872065890197670e-06 --3.068916019074334916e-06 --2.237274925003372191e-06 -6.218469303313082624e-07 -2.943636018290449366e-06 -2.662316743245618561e-06 --3.152100982096967303e-07 --2.818282924138291761e-06 --2.838852449590507989e-06 --3.050601415378084178e-07 -2.465190714300445667e-06 -3.055530744710881232e-06 -6.122169829998658776e-07 --2.249847546790695871e-06 --3.081778195477484829e-06 --1.204015685519354515e-06 -1.761809630506516815e-06 -3.087026089947893743e-06 -1.483209029837644940e-06 --1.494393881414008290e-06 --3.083631170055253859e-06 --2.000164564008749556e-06 -9.222447128838812344e-07 -3.027257137092730263e-06 -2.459197274791265313e-06 --6.218751946993127742e-07 --2.943682241840437243e-06 --2.662240875883936313e-06 -5.177599411854798302e-09 -2.656525689580131394e-06 -2.975242719503508963e-06 -3.050739642304999503e-07 --2.465265908141236740e-06 --3.055507409966075692e-06 --9.128700877040965533e-07 -2.014347162246525667e-06 -3.087117081367807194e-06 -1.483209031484586580e-06 --1.761878267191295256e-06 --3.087022424574749835e-06 --1.748729111255308350e-06 -1.213820143539339800e-06 -3.068910735938296978e-06 -2.237352678661156682e-06 --9.222855626564606755e-07 --3.027283786468236064e-06 --2.459117716458082392e-06 -3.151955302193813758e-07 -2.818222728519646496e-06 -2.838920826522776868e-06 --5.177918327220845245e-09 --2.656595788650424821e-06 --2.975193465148680353e-06 --6.121897128985342367e-07 -2.249771370986777450e-06 -3.081780883773516501e-06 -1.204066124213934665e-06 --2.014420932157988851e-06 --3.087115737115972223e-06 --1.483149249589518697e-06 -1.494332629194715850e-06 -3.083617096360583382e-06 -2.000238057825104762e-06 --1.213872065907873130e-06 --3.068925187140496035e-06 --2.237274883635692086e-06 -6.218469300457401039e-07 -2.943636246392468535e-06 -2.662317340547320204e-06 --5.177918037145504550e-09 --2.818282997454390640e-06 --2.838852163745343939e-06 --3.050601384198528222e-07 -2.465190716858414864e-06 -3.055530309727082865e-06 -9.129096466147376817e-07 --2.249847543712664961e-06 --3.081774497777610128e-06 --1.204015679127434042e-06 -1.761809630415061185e-06 -3.087024364161945625e-06 -1.748796528563774324e-06 --1.494393881236986451e-06 --3.083614825898169062e-06 --2.000164503180351116e-06 -9.222447130369182769e-07 -3.027257278447567593e-06 -2.459197181235523377e-06 --3.152100984826847025e-07 --2.943681534589918649e-06 --2.662240108556591613e-06 -5.177600657344220044e-09 -2.656525686817303467e-06 -2.975242448115834870e-06 -6.122169843940998965e-07 --2.465265904934462328e-06 --3.055509188280463714e-06 --9.128700870515184421e-07 -2.014347162001775497e-06 -3.087117696763499701e-06 -1.483209020055367523e-06 --1.494393881406907401e-06 --3.087026609318223618e-06 --1.748729100122564589e-06 -1.213820143633766610e-06 -3.068906735535524230e-06 -2.237352645569658475e-06 --6.218751955161471970e-07 --3.027284318223303594e-06 --2.459118034120911282e-06 -3.151955301458421871e-07 -2.818222688584428278e-06 -2.838920821735220019e-06 -3.050739648845807509e-07 --2.656595781800089482e-06 --2.975194602679749042e-06 --6.121897170761729421e-07 -2.249771372717715959e-06 -3.081782304621676918e-06 -1.204066117044157791e-06 --1.761878267186168377e-06 --3.087117395411752190e-06 --1.483149266865272135e-06 -1.494332629502537999e-06 -3.083625972580297810e-06 -2.000238039944751701e-06 --9.222855631034597612e-07 --3.068920055843501212e-06 --2.237275003743279682e-06 -6.218469297725546143e-07 -2.943636630220438807e-06 -2.662317523990572892e-06 --5.177918369192947401e-09 --2.656595775959408522e-06 --2.838852798037668533e-06 --3.050601405035421198e-07 -2.465190717108607221e-06 -3.055529205919641637e-06 -9.129096418802539052e-07 --2.014420931666647908e-06 --3.081776861500625377e-06 --1.204015688337056836e-06 -1.761809630563580789e-06 -3.087023012703732639e-06 -1.748796516783566625e-06 --1.213872066246057387e-06 --3.083622086087906214e-06 --2.000164598443290499e-06 -9.222447127348415522e-07 -3.027256799326660267e-06 -2.459196978793427138e-06 --3.152100987257418655e-07 --2.818282966704568379e-06 --2.662240676059719911e-06 -5.177600349215444931e-09 -2.656525696998041488e-06 -2.975241780752169337e-06 -6.122169823718578628e-07 --2.249847544824541394e-06 +1.000005625775184148e+00 +1.000002760802206403e+00 +9.999972997227719596e-01 +9.999943554832938508e-01 +9.999962801838924165e-01 +1.000001649259240066e+00 +1.000005621297554326e+00 +1.000004150765653632e+00 +9.999989029896425574e-01 +9.999945035842714436e-01 +9.999951247068956661e-01 +9.999999666912248175e-01 +1.000004883287164414e+00 +1.000005153055575136e+00 +1.000000599704507387e+00 +9.999955002429915529e-01 +9.999944883058548006e-01 +9.999982906363023005e-01 +1.000003652471175730e+00 +1.000005626023197314e+00 +1.000002244380173044e+00 +9.999968100129270621e-01 +9.999943613489549188e-01 +9.999967457294383077e-01 +1.000002185343074723e+00 +1.000005642183421362e+00 +1.000003720006108709e+00 +9.999983506243874665e-01 +9.999943787300924480e-01 +9.999954614235876393e-01 +1.000000534103657968e+00 +1.000005228284200687e+00 +1.000004875461331100e+00 +1.000000033314989656e+00 +9.999951161265646649e-01 +9.999946341258477700e-01 +9.999988401549567696e-01 +1.000004088591221585e+00 +1.000005594241463847e+00 +1.000001709471020517e+00 +9.999963472620381211e-01 +9.999943759118674613e-01 +9.999972393585931929e-01 +1.000002700094356101e+00 +1.000005642935601902e+00 +1.000003254449998158e+00 +9.999978145306619437e-01 +9.999943568920527381e-01 +9.999958494270475562e-01 +1.000001096927151556e+00 +1.000005496844230901e+00 +1.000004875462914722e+00 +9.999994658636355283e-01 +9.999947710426326797e-01 +9.999948470257585820e-01 +9.999994003354164551e-01 +1.000004499377124434e+00 +1.000005512032049904e+00 +1.000001159920290927e+00 +9.999959111855447613e-01 +9.999944086350009975e-01 +9.999977557558935581e-01 +1.000003189749025134e+00 +1.000005641981852156e+00 +1.000002760802472412e+00 +9.999972997496957561e-01 +9.999943555922112814e-01 +9.999962801838032656e-01 +1.000001649258354997e+00 +1.000005621006992085e+00 +1.000004538759016182e+00 +9.999989029995278722e-01 +9.999945035047999031e-01 +9.999951247343861205e-01 +9.999999666911814078e-01 +1.000004883310042558e+00 +1.000005365866759366e+00 +1.000000599704985893e+00 +9.999955004505431955e-01 +9.999944883353297786e-01 +9.999982906363189539e-01 +1.000003652478812066e+00 +1.000005627607422731e+00 +1.000002760802608082e+00 +9.999968101024463429e-01 +9.999943588459464516e-01 +9.999967457289983264e-01 +1.000002185341319683e+00 +1.000005642529046224e+00 +1.000004150765845923e+00 +9.999983506354790386e-01 +9.999943795359398369e-01 +9.999954614296149291e-01 +1.000000534103560934e+00 +1.000005228588552564e+00 +1.000005153052095697e+00 +1.000000033314421888e+00 +9.999951167751868208e-01 +9.999946341935053162e-01 +9.999988401550065076e-01 +1.000004088574629524e+00 +1.000005594116402996e+00 +1.000002244380725491e+00 +9.999963473590743890e-01 +9.999943705630350843e-01 +9.999972393585445651e-01 +1.000002700091033647e+00 +1.000005644504989410e+00 +1.000003720006078733e+00 +9.999978145306807065e-01 +9.999943582381595197e-01 +9.999958494249607810e-01 +1.000001096926319555e+00 +1.000005496920314485e+00 +1.000004875456607101e+00 +1.000000033315262993e+00 +9.999947715979419227e-01 +9.999948470606095929e-01 +9.999994003352935534e-01 +1.000004499306196948e+00 +1.000005511892942289e+00 +1.000001709471179279e+00 +9.999959112259785288e-01 +9.999944044212771832e-01 +9.999977557558298313e-01 +1.000003189745025889e+00 +1.000005641977540716e+00 +1.000003254450036350e+00 +9.999972997569587241e-01 +9.999943568080577050e-01 +9.999962801835043935e-01 +1.000001649257916236e+00 +1.000005619863243211e+00 +1.000004538758572092e+00 +9.999994658666658820e-01 +9.999945033589370258e-01 +9.999951246920915082e-01 +9.999999666911157936e-01 +1.000004883392187294e+00 +1.000005365901549537e+00 +1.000001159920163030e+00 +9.999955004701366335e-01 +9.999944875786248577e-01 +9.999982906362654411e-01 +1.000003652477882365e+00 +1.000005631199297751e+00 +1.000002760802340518e+00 +9.999972997609573033e-01 +9.999943572468015240e-01 +9.999967457287104455e-01 +1.000002185341415606e+00 +1.000005642579079534e+00 +1.000004150766196531e+00 +9.999989030082178099e-01 +9.999943789194916111e-01 +9.999954614455011104e-01 +1.000000534103923533e+00 +1.000005228595872486e+00 +1.000005153046126027e+00 +1.000000599705319626e+00 +9.999951162555039685e-01 +9.999946339857431754e-01 +9.999988401548633998e-01 +1.000004088569844463e+00 +1.000005592790779829e+00 +1.000002244380366445e+00 +9.999968100716229991e-01 +9.999943732042494426e-01 +9.999972393582763353e-01 +1.000002700095898867e+00 +1.000005644744252020e+00 +1.000003720006130914e+00 +9.999983506448507642e-01 +9.999943571126526098e-01 +9.999958494295454470e-01 +1.000001096927333633e+00 +1.000005496689988060e+00 +1.000004875457697340e+00 +1.000000033313640513e+00 +9.999951162463665000e-01 +9.999948471061756994e-01 +9.999994003352387084e-01 +1.000004499371139888e+00 +1.000005511919947576e+00 +1.000001709470925038e+00 +9.999963473308361994e-01 +9.999944077223968941e-01 +9.999977557559259767e-01 +1.000003189760898525e+00 +1.000005639429340842e+00 +1.000003254450063883e+00 +9.999978145314553091e-01 +9.999943553535456386e-01 +9.999962801834312298e-01 +1.000001649258108749e+00 +1.000005620825985098e+00 +1.000004538758234807e+00 +9.999994658652128221e-01 +9.999947711403059936e-01 +9.999951247163477719e-01 +9.999999666910157625e-01 +1.000004883516108389e+00 +1.000005365933064549e+00 +1.000001159920125282e+00 +9.999959112128709027e-01 +9.999944880828373073e-01 +9.999982906362719914e-01 +1.000003652479208194e+00 +1.000005627782454054e+00 +1.000002760802314539e+00 +9.999972997549056997e-01 +9.999943551254746321e-01 +9.999967457295289019e-01 +1.000002185338942251e+00 +1.000005642668040151e+00 +1.000004150765919197e+00 +9.999989030074172280e-01 +9.999945030108718935e-01 +9.999954614257452468e-01 +1.000000534103545835e+00 +1.000005228497013121e+00 +1.000005153049556617e+00 +1.000000599705282101e+00 +9.999955004138553205e-01 +9.999946341890698642e-01 +9.999988401548611794e-01 +1.000004088552002957e+00 +1.000005592403996113e+00 +1.000002244380352012e+00 +9.999968100815259664e-01 +9.999943596037572657e-01 +9.999972393583629326e-01 +1.000002700092436525e+00 +1.000005643971312974e+00 +1.000003720006111152e+00 +9.999983506437940539e-01 +9.999943789095604441e-01 +9.999958494266143472e-01 +1.000001096926856681e+00 +1.000005496713704201e+00 +1.000004875451790065e+00 +1.000000033313640069e+00 +9.999951164564397921e-01 +9.999946342331085258e-01 +9.999994003351478922e-01 +1.000004499329271157e+00 +1.000005512015281539e+00 +1.000001709470949018e+00 +9.999963473360324873e-01 +9.999943723320516842e-01 +9.999977557557845342e-01 +1.000003189753668975e+00 +1.000005640383372141e+00 +1.000003254450023027e+00 +9.999978145321564149e-01 +9.999943574830255644e-01 +9.999962801844133331e-01 +1.000001649257747260e+00 +1.000005620446916765e+00 +1.000004538757943040e+00 +9.999994658651829571e-01 +9.999947713501622371e-01 +9.999948470901370845e-01 +9.999999666909942242e-01 +1.000004883426925506e+00 +1.000005365967129745e+00 +1.000001159920161253e+00 +9.999959112302192477e-01 +9.999944062202245076e-01 +9.999982906362955282e-01 +1.000003652477207572e+00 +1.000005629411929498e+00 +1.000002760802297219e+00 +9.999972997554668064e-01 +9.999943565203065488e-01 +9.999962801842935400e-01 +1.000002185338930705e+00 +1.000005642454813604e+00 +1.000004150766473643e+00 +9.999989030074023511e-01 +9.999945030714696426e-01 +9.999951247120826281e-01 +1.000000534103416827e+00 +1.000005228508070276e+00 +1.000005153049992712e+00 +1.000000599705265225e+00 +9.999955004410504555e-01 +9.999944880857016827e-01 +9.999988401550021777e-01 +1.000004088538783531e+00 +1.000005592644174213e+00 +1.000002244380342908e+00 +9.999968100779922375e-01 +9.999943592858060448e-01 +9.999967457292057160e-01 +1.000002700092129659e+00 +1.000005644155826490e+00 +1.000003720006242380e+00 +9.999983506435720093e-01 +9.999943791736130017e-01 +9.999954614274593201e-01 +1.000001096926721900e+00 +1.000005496500181890e+00 +1.000004875453517794e+00 +1.000000033313782621e+00 +9.999951163849680746e-01 +9.999946341765165725e-01 +9.999988401550004014e-01 +1.000004499336313302e+00 +1.000005511922090973e+00 +1.000001709471145972e+00 +9.999963473387932789e-01 +9.999943715944688005e-01 +9.999972393582987618e-01 +1.000003189748610799e+00 +1.000005639112068634e+00 +1.000003254450053669e+00 +9.999978145336728685e-01 +9.999943575200217483e-01 +9.999958494266166786e-01 +1.000001649259649295e+00 +1.000005620341926749e+00 +1.000004538758855643e+00 +9.999994658653188484e-01 +9.999947714096492080e-01 +9.999948470907105147e-01 +9.999994003356920125e-01 +1.000004883371763631e+00 +1.000005365994392381e+00 +1.000001159920419491e+00 +9.999959112441260123e-01 +9.999944071746371677e-01 +9.999977557557465646e-01 +1.000003652483939520e+00 +1.000005630204431117e+00 +1.000002760802239488e+00 +9.999972997536744623e-01 +9.999943565228189835e-01 +9.999962801842512405e-01 +1.000001649259425696e+00 +1.000005642391002203e+00 +1.000004150766384603e+00 +9.999989030061567918e-01 +9.999945031291599395e-01 +9.999951247132606857e-01 +9.999999666913534924e-01 +1.000005228529977197e+00 +1.000005153051072071e+00 +1.000000599705023197e+00 +9.999955004362547362e-01 +9.999944880546194348e-01 +9.999982906363016344e-01 +1.000004088605031205e+00 +1.000005593202574428e+00 +1.000002244380393979e+00 +9.999968100713279018e-01 +9.999943592824395155e-01 +9.999967457292401329e-01 +1.000002185345483907e+00 +1.000005643946614509e+00 +1.000003720006287233e+00 +9.999983506357719154e-01 +9.999943799145210033e-01 +9.999954614274424447e-01 +1.000000534103291816e+00 +1.000005496729241994e+00 +1.000004875454678421e+00 +1.000000033313517278e+00 +9.999951165950072829e-01 +9.999946341601548827e-01 +9.999988401550966577e-01 +1.000004088597379326e+00 +1.000005511932688718e+00 +1.000001709471213696e+00 +9.999963473085962118e-01 +9.999943690904292692e-01 +9.999972393582837737e-01 +1.000002700093403751e+00 +1.000005640071133683e+00 +1.000003254450146484e+00 +9.999978145134313934e-01 +9.999943579543062455e-01 +9.999958494264483688e-01 +1.000001096926753208e+00 +1.000005621367910047e+00 +1.000004538757343298e+00 +9.999994658650096513e-01 +9.999947714452580572e-01 +9.999948471014514784e-01 +9.999994003356373895e-01 +1.000004499431749405e+00 +1.000005365881810881e+00 +1.000001159920228755e+00 +9.999959111303704518e-01 +9.999944046160988975e-01 +9.999977557556637420e-01 +1.000003189742899812e+00 +1.000005625765048034e+00 +1.000002760802242818e+00 +9.999972997503764338e-01 +9.999943553682689723e-01 +9.999962801842160465e-01 +1.000001649259704806e+00 +1.000005621363527553e+00 +1.000004150765572586e+00 +9.999989030014744262e-01 +9.999945031670292028e-01 +9.999951247189768910e-01 +9.999999666911644214e-01 +1.000004883320198656e+00 +1.590258709880481725e-06 +1.850727402495279277e-07 +-1.388541769673926653e-06 +-1.701017291696231477e-06 +-5.275213154298756335e-07 +1.127180152538241350e-06 +1.726491620999807195e-06 +6.926305838952891814e-07 +-9.844231305223494377e-07 +-1.740702235800200361e-06 +-1.004289952315689854e-06 +6.744114282080159465e-07 +1.741052633384862507e-06 +1.148018577632170975e-06 +-5.089969905231052128e-07 +-1.734358073696038867e-06 +-1.400633232471084825e-06 +1.648280955158712544e-07 +1.613497050054945738e-06 +1.504603245167010947e-06 +1.028046073127142543e-08 +-1.507032766629912559e-06 +-1.655943718395572940e-06 +-3.579362193198742469e-07 +1.261768751916517784e-06 +1.726444650652859829e-06 +5.275540717886982975e-07 +-1.127233184103640488e-06 +-1.738629705526233234e-06 +-8.519526867277922594e-07 +8.332669303254228888e-07 +1.742314501460785511e-06 +1.004344893216444626e-06 +-6.744446347904300477e-07 +-1.741190486442828142e-06 +-1.280895671953873192e-06 +3.385187000265915793e-07 +1.696500812349482576e-06 +1.400691858939751180e-06 +-1.648380796717641011e-07 +-1.613539989580305443e-06 +-1.590222811317883300e-06 +-1.850607679087758196e-07 +1.388528183416090092e-06 +1.701097407026286314e-06 +3.579591081825361336e-07 +-1.261811969936731450e-06 +-1.727216086362100305e-06 +-6.925890060411777431e-07 +9.843755628953675314e-07 +1.741468567047642194e-06 +8.520019343554935675e-07 +-8.333134720201472977e-07 +-1.742154442407720301e-06 +-1.147960835105777450e-06 +5.089716510225123269e-07 +1.734636259046536423e-06 +1.400691742656256031e-06 +-3.385365853475210078e-07 +-1.696322441869284449e-06 +-1.504545098921229713e-06 +-1.027950600037699394e-08 +1.507017642893446091e-06 +1.655912282090512858e-06 +1.850736573253833457e-07 +-1.388593103670551729e-06 +-1.701123011159464797e-06 +-5.275213203113701307e-07 +1.127175887512785350e-06 +1.737084630554120061e-06 +6.926305374906872101e-07 +-9.844220556220729739e-07 +-1.741418280039016872e-06 +-1.004290181794073400e-06 +6.744100915122527316e-07 +1.741136234302182269e-06 +1.280953800339244119e-06 +-5.089968161688048876e-07 +-1.734469996461215453e-06 +-1.400635145554576073e-06 +1.648280393050493682e-07 +1.613482434497394589e-06 +1.590260350272604001e-06 +1.028156819874885824e-08 +-1.507095212807075582e-06 +-1.655887238029578999e-06 +-3.579362205533576648e-07 +1.261753103397525227e-06 +1.726157707051833215e-06 +6.926304404517859172e-07 +-1.127228675567379859e-06 +-1.736257195298326751e-06 +-8.519527011095154370e-07 +8.332680875729218544e-07 +1.741496280575035026e-06 +1.148018540650626313e-06 +-6.744480534802965719e-07 +-1.741171439352269567e-06 +-1.280895278023926046e-06 +3.385191498947034055e-07 +1.696326457884578856e-06 +1.504599592364469134e-06 +-1.648375014590240681e-07 +-1.613480874528436288e-06 +-1.590224424181760898e-06 +-1.850607843687100034e-07 +1.388532291010664415e-06 +1.701034487815878602e-06 +5.275537985538963861e-07 +-1.261853076819824106e-06 +-1.726152893555389757e-06 +-6.925890073988103037e-07 +9.843836909720386743e-07 +1.740201062440504268e-06 +1.004344883696872571e-06 +-8.333172255183000351e-07 +-1.742519564130635849e-06 +-1.147960182234560617e-06 +5.089719730719171151e-07 +1.734569191161493850e-06 +1.400691149879991598e-06 +-1.648380427839468633e-07 +-1.696560750486992819e-06 +-1.504551627464877021e-06 +-1.027944716728731548e-08 +1.507085960251261140e-06 +1.655975652851829364e-06 +3.579588946777138358e-07 +-1.388629294146524730e-06 +-1.701182326728532360e-06 +-5.275213253407199463e-07 +1.127181229005059510e-06 +1.736714200870366039e-06 +8.520018482203611131e-07 +-9.844321538280139964e-07 +-1.741225550081222717e-06 +-1.004290007461292582e-06 +6.744106687903739636e-07 +1.741462204149202398e-06 +1.280954222530781399e-06 +-3.385401668286348946e-07 +-1.734649093637919336e-06 +-1.400628663622346324e-06 +1.648279882689770145e-07 +1.613614075780983244e-06 +1.590271522813106817e-06 +1.850730311209705578e-07 +-1.507153830918298451e-06 +-1.655961476946026026e-06 +-3.579361763399364898e-07 +1.261759921644646834e-06 +1.725893974381015557e-06 +6.926305516229005278e-07 +-9.844361182042738464e-07 +-1.736686570992468948e-06 +-8.519527346364049901e-07 +8.332672720311000855e-07 +1.741577450178477170e-06 +1.148018643972756175e-06 +-5.090028878685917445e-07 +-1.741071664343919371e-06 +-1.280893007492375202e-06 +3.385190283198168349e-07 +1.696273442558272825e-06 +1.504596047420710591e-06 +1.028067565849080836e-08 +-1.613678524575431118e-06 +-1.590228735450712356e-06 +-1.850607004991338731e-07 +1.388526752923310147e-06 +1.700983221527749291e-06 +5.275539677781154082e-07 +-1.127237941922898376e-06 +-1.726101689114446397e-06 +-6.925889769606349690e-07 +9.843780847074831729e-07 +1.740988709107291496e-06 +1.004344852365307235e-06 +-6.744507678687679026e-07 +-1.741609546519275544e-06 +-1.147960506446668160e-06 +5.089723585953228279e-07 +1.734442162620808360e-06 +1.400690583567017859e-06 +-1.648377356755172739e-07 +-1.613716912519909083e-06 +-1.504541679001261497e-06 +-1.027937814965926895e-08 +1.507029924103069356e-06 +1.655965816649125593e-06 +3.579589664754058362e-07 +-1.261828778088528421e-06 +-1.700922393705801968e-06 +-5.275213071047663598e-07 +1.127183456889939933e-06 +1.737589558734849914e-06 +8.520018474006426491e-07 +-8.333136941412435331e-07 +-1.741108267578420554e-06 +-1.004289945897832174e-06 +6.744118128911313061e-07 +1.741426051359071790e-06 +1.280953975366231753e-06 +-3.385392001881804617e-07 +-1.696289274454028947e-06 +-1.400634676346342227e-06 +1.648280454427183166e-07 +1.613620484792240372e-06 +1.590266227185648271e-06 +1.850729201873577123e-07 +-1.388598740879103959e-06 +-1.655949969668479244e-06 +-3.579361874326070693e-07 +1.261769798818398850e-06 +1.726421594208738017e-06 +6.926305803879282936e-07 +-9.844292928256493959e-07 +-1.741103988185197408e-06 +-8.519527256967228089e-07 +8.332681204532097489e-07 +1.742137816808617435e-06 +1.148018603647752241e-06 +-5.090022385865434066e-07 +-1.734594009746377118e-06 +-1.280895065173918608e-06 +3.385189830825236208e-07 +1.696358701328580575e-06 +1.504598802422561717e-06 +1.028066519374389938e-08 +-1.507085425042355993e-06 +-1.590223265583818352e-06 +-1.850607303696900364e-07 +1.388530208700179477e-06 +1.701003410918592177e-06 +5.275539720348859433e-07 +-1.127236328972740943e-06 +-1.737072019608247472e-06 +-6.925890397434343505e-07 +9.843797688342706320e-07 +1.741151865984534068e-06 +1.004344847574267386e-06 +-6.744500214686274675e-07 +-1.741180623148688443e-06 +-1.147960241359076898e-06 +5.089720195309898694e-07 +1.734542070068430320e-06 +1.400691576856052290e-06 +-1.648377264877983024e-07 +-1.613567264205507283e-06 +-1.590220997882724452e-06 +-1.027945266177097877e-08 +1.507054779731643882e-06 +1.655948986141254314e-06 +3.579589405865119535e-07 +-1.261822669735039415e-06 +-1.726248378931100790e-06 +-5.275213256528804924e-07 +1.127180212960296698e-06 +1.737145793846374856e-06 +8.520018437027837338e-07 +-8.333135839839537744e-07 +-1.741826166559895173e-06 +-1.004290002577691937e-06 +6.744107524456056007e-07 +1.741434120495588456e-06 +1.280954066336914685e-06 +-3.385393446547938588e-07 +-1.696408346730930952e-06 +-1.504547546145360437e-06 +1.648279238310219440e-07 +1.613593358049607904e-06 +1.590265266622223248e-06 +1.850728906943091782e-07 +-1.388592312864333375e-06 +-1.701033181207115733e-06 +-3.579361994878603425e-07 +1.261762142237637242e-06 +1.726154469385124064e-06 +6.926305729210776948e-07 +-9.844306732773516617e-07 +-1.740935075061179071e-06 +-1.004290005992815914e-06 +8.332668107363248065e-07 +1.742047849474825750e-06 +1.148018594455816554e-06 +-5.090024883766607723e-07 +-1.734576814070674148e-06 +-1.400634242297095216e-06 +3.385188554784090977e-07 +1.696263364638979683e-06 +1.504596670866742508e-06 +1.028062377219007074e-08 +-1.507102089845137072e-06 +-1.655924495095443164e-06 +-1.850607415461862521e-07 +1.388521422721493471e-06 +1.700993753069471308e-06 +5.275539553648519073e-07 +-1.127238335727100728e-06 +-1.737259124443960916e-06 +-8.519527452616847480e-07 +9.843784977436696009e-07 +1.741509171049854340e-06 +1.004344852353452585e-06 +-6.744502569944053872e-07 +-1.741384013451149539e-06 +-1.280894988554857104e-06 +5.089720508299514252e-07 +1.734418632636085218e-06 +1.400690944410196418e-06 +-1.648377472793484936e-07 +-1.613640522525692076e-06 +-1.590222066274167874e-06 +-1.850607705281996391e-07 +1.507045512717325373e-06 +1.655953953555566205e-06 +3.579588754104873128e-07 +-1.261827371769381423e-06 +-1.726241646624968192e-06 +-6.925890056911455069e-07 +1.127179221072367417e-06 +1.737438349978383794e-06 +8.520018353432443708e-07 +-8.333134243790168780e-07 +-1.742091801623881296e-06 +-1.147960315055135911e-06 +6.744108640788155416e-07 +1.741383014368384549e-06 +1.280954068588190404e-06 +-3.385395013434910266e-07 +-1.696407975714907660e-06 +-1.504546496833054631e-06 +-1.027945156890405462e-08 +1.613558248167692884e-06 +1.590264887097033811e-06 +1.850727480043968336e-07 +-1.388582710117581190e-06 +-1.701007943560229509e-06 +-5.275213266703994667e-07 +1.261761576141305520e-06 +1.726000676928844671e-06 +6.926305956639591272e-07 +-9.844267280894752903e-07 +-1.741512944258879865e-06 +-1.004290016499870624e-06 +6.744106055143346973e-07 +1.741555952554223046e-06 +1.148018595474808413e-06 +-5.090019614378216274e-07 +-1.734591562866344000e-06 +-1.400634284221151453e-06 +1.648279524011102915e-07 +1.696295494532174206e-06 +1.504597371495823923e-06 +1.028054244384596495e-08 +-1.507071808459608073e-06 +-1.655908573310841154e-06 +-3.579362582855198438e-07 +1.388517896125253766e-06 +1.701017301470918208e-06 +5.275539556353993128e-07 +-1.127217618902127510e-06 +-1.736963978597491788e-06 +-8.519527402804675796e-07 +8.332656401994154939e-07 +1.741378217346416395e-06 +1.004344844050856600e-06 +-6.744444954404536350e-07 +-1.741344194946463995e-06 +-1.280894810056264701e-06 +3.385185005648651511e-07 +1.734500300970092121e-06 +1.400690527060552305e-06 +-1.648381025857745115e-07 +-1.613514314594990808e-06 +-1.590218367364895719e-06 +-1.850608069330568487e-07 +1.388516357169056198e-06 +1.655945546814741661e-06 +3.579588203429640820e-07 +-1.261782497877215767e-06 +-1.725857671376120261e-06 +-6.925889913294783754e-07 +9.843740187301974938e-07 +1.737679640457574131e-06 +8.520018294553982875e-07 +-8.333077363292762579e-07 +-1.741350671057119731e-06 +-1.147960198128919997e-06 +5.089719448722273021e-07 +1.741072341592975031e-06 +1.280953859019597922e-06 +-3.385398952397837006e-07 +-1.696051325085198054e-06 +-1.504543627573693006e-06 +-1.027935900639229383e-08 +1.506996708215537967e-06 +1.590261173452735001e-06 +1.850728075432969316e-07 +-1.388549659267483009e-06 +-1.701102775001740137e-06 +-5.275213000104285693e-07 +1.127181349487917218e-06 +1.726545434173700165e-06 +6.926306403836626415e-07 +-9.844364571400124864e-07 +-1.739552051894324131e-06 +-1.004289941535728116e-06 +6.744120903403956215e-07 +1.741029127265671066e-06 +1.148018530681769560e-06 +-5.089966504975827709e-07 +-1.734766272327423066e-06 +-1.400632526955695013e-06 +1.648280872786722481e-07 +1.613579841192132622e-06 +1.504596093198193721e-06 +1.028166291379170719e-08 +-1.507180347925612950e-06 +-1.655880508224637709e-06 +-3.579362120797978204e-07 +1.261744760914462781e-06 +1.701066777656902194e-06 +5.275540513833159856e-07 +-1.127243239365637466e-06 +-1.737121590860897213e-06 +-8.519526930896695320e-07 +8.332666001719421804e-07 +1.742288573428257718e-06 +1.004344838581993247e-06 +-6.744443069608969913e-07 +-1.741736801215740741e-06 +-1.280894858590050639e-06 +3.385191693555600721e-07 +1.696188634757403962e-06 +1.400692097674729520e-06 +-1.648376974355455552e-07 +-1.613589568463775903e-06 +-1.590217322025785994e-06 +-1.850606813297324558e-07 +1.388517346821858019e-06 +1.701060300065911117e-06 +3.579588228472856531e-07 +-1.261814090928261297e-06 +-1.725769989100899558e-06 +-6.925890047770793251e-07 +9.843808966886625604e-07 +1.739938131701850016e-06 +8.520018907075687934e-07 +-8.333163162997386263e-07 +-1.742726832424178203e-06 +-1.147960427891790358e-06 +5.089724213524740518e-07 +1.734550541368317171e-06 +1.280954140092062080e-06 +-3.385398077827267889e-07 +-1.696243955884063799e-06 +-1.504544917060379414e-06 +-1.027936504216994339e-08 +1.507069983893762695e-06 +1.655982354507205710e-06 +1.850729744934016058e-07 +-1.388623461001337052e-06 +-1.701187672885859959e-06 +-5.275213246265532225e-07 +1.127182163690784350e-06 +1.737875380626029099e-06 +6.926305334885291674e-07 +-9.844297310867681608e-07 +-1.740987370623198207e-06 +-1.004290109880700681e-06 +6.744115823117965700e-07 +1.741332681612490186e-06 +1.280954073280507716e-06 +-5.089958973099097255e-07 +-1.734428316721273204e-06 +-1.400630363082169511e-06 +1.648280114153578379e-07 +1.613587251137130100e-06 +1.590257712069878907e-06 +1.028162319877566815e-08 +-1.507123472649657072e-06 +-1.656026673134441481e-06 +-3.579362138230268306e-07 +1.261765932968881193e-06 +1.725849496633837332e-06 +5.275538640396327477e-07 +-1.127226714463011616e-06 +-1.735888205023552666e-06 +-8.519527164467000378e-07 +8.332671510606626846e-07 +1.742076328180530369e-06 +1.148018497632888122e-06 +-6.744419986192260213e-07 +-1.741087924878268738e-06 +-1.280895072535686088e-06 +3.385186950226027242e-07 +1.696444750421543404e-06 +1.504597086469492552e-06 +-1.648375210356707209e-07 +-1.613646454703819000e-06 +-1.590229964066098528e-06 +-1.850607901780342002e-07 +1.388525104051798746e-06 +1.701031427843256524e-06 +5.275538641904303409e-07 +-1.261802586419445627e-06 +-1.726519944635073156e-06 +-6.925889748452972382e-07 +9.843743424279099744e-07 +1.740619921443903349e-06 +1.004344848179437275e-06 +-8.333060441746281620e-07 +-1.741997207858713043e-06 +-1.147960334818376722e-06 +5.089717784803551655e-07 +1.734438698613764786e-06 +1.400690994529669114e-06 +-3.385392492815753392e-07 +-1.696032358417492168e-06 +-1.504546007199993032e-06 +-1.027937479700997961e-08 +1.507000651863114781e-06 +1.655949143046259821e-06 +3.579588447697392843e-07 +-1.388596859089842529e-06 +-1.700945379316332144e-06 +-5.275213139274402495e-07 +1.127175366846241379e-06 +1.737338985482136595e-06 +8.520018480164420604e-07 +-9.844242560294862260e-07 +-1.739567432785072124e-06 +-1.004289853599555752e-06 +6.744114505553351100e-07 +1.741105823579371588e-06 +1.280954036891148774e-06 +-3.385393510609883186e-07 +-1.734659917651862858e-06 +-1.400636506470185294e-06 +1.648280844023104601e-07 +1.613517913534661131e-06 +1.590270511347760083e-06 +1.850729527781549711e-07 +-1.507126222466079221e-06 +-1.656024543609069937e-06 +-3.579361632341287926e-07 +1.261766949185430122e-06 +1.726403622829481694e-06 +6.926305417498411900e-07 +-1.127229822279195776e-06 +-1.738721891103089493e-06 +-8.519527005897984670e-07 +8.332677691466162183e-07 +1.741387876275864048e-06 +1.148018684700642225e-06 +-5.090018832493002919e-07 +-1.741396064626738821e-06 +-1.280893680614266850e-06 +3.385189161820171912e-07 +1.696365357801187906e-06 +1.504601305946100156e-06 +1.028070636010076308e-08 +-1.613504046691272219e-06 +-1.590228768881209665e-06 +-1.850607021379894719e-07 +1.388533158037275724e-06 +1.700993704208600421e-06 +5.275539425489083079e-07 +-1.127227302984051667e-06 +-1.725761071736155471e-06 +-6.925889864461445462e-07 +9.843777984914883103e-07 +1.741707439146850073e-06 +1.004344866413877849e-06 +-6.744499271448763110e-07 +-1.742301470748352831e-06 +-1.147960505668042653e-06 +5.089721092365632374e-07 +1.734692526625950660e-06 +1.400690738996414468e-06 +-1.648377664950101919e-07 +-1.696481721756107406e-06 +-1.504535658134828716e-06 +-1.027945642029108999e-08 +1.507038462891205193e-06 +1.655931791920991752e-06 +3.579589416177502146e-07 +-1.261825244286295888e-06 +-1.701096132573522042e-06 +-5.275213078853165381e-07 +1.127178254342393248e-06 +1.736675458571481429e-06 +8.520018446102271767e-07 +-8.333132946683656141e-07 +-1.741916864233069986e-06 +-1.004290180398935809e-06 +6.744108342309223272e-07 +1.741463583973312753e-06 +1.280953857998383449e-06 +-3.385394450034097421e-07 +-1.696418732256577872e-06 +-1.400632910854172307e-06 +1.648278809057894656e-07 +1.613455641596044816e-06 +1.590268176716472360e-06 +1.850729003681355142e-07 +-1.388590622007244039e-06 +-1.655880814542978364e-06 +-3.579361863642931702e-07 +1.261757307338035565e-06 +1.726175047362156238e-06 +6.926305767162723425e-07 +-9.844320923223519404e-07 +-1.735997048522879651e-06 +-8.519526792089144618e-07 +8.332669767177802576e-07 +1.741753592211998155e-06 +1.148018598796977087e-06 +-5.090024483558281695e-07 +-1.734742097728113351e-06 +-1.280895070223601450e-06 +3.385189691855595693e-07 +1.696209839631811581e-06 +1.504598674809486538e-06 +1.028063672740347746e-08 +-1.507118270675432684e-06 +-1.590225360960248558e-06 +-1.850606841024569039e-07 +1.388536125426654509e-06 +1.701073599894859555e-06 +5.275539714795897778e-07 +-1.127238326380262197e-06 +-1.736386034132772162e-06 +-6.925890310144362625e-07 +9.843783720537162368e-07 +1.741184655186639981e-06 +1.004344859441287492e-06 +-6.744501205690743933e-07 +-1.741347003965438300e-06 +-1.147960188013552591e-06 +5.089723477368453705e-07 +1.734683605293342141e-06 +1.400691216655022114e-06 +-1.648377232827669521e-07 +-1.613639073771378785e-06 +-1.504546951810305930e-06 +-1.027935849609450054e-08 +1.507031901604741637e-06 +1.655970651008055552e-06 +3.579589442097077203e-07 +-1.261822109719881423e-06 +-1.726100093657448231e-06 +-5.275213162748761252e-07 +1.127179837933315962e-06 +1.736693167282370843e-06 +8.520018476712604643e-07 +-8.333134571715394537e-07 +-1.742135981851321251e-06 +-1.004289964789669630e-06 +6.744117110257168481e-07 +1.741323323509899397e-06 +1.280954131873073330e-06 +-3.385392883702650713e-07 +-1.696409550551136906e-06 +-1.504546208740563941e-06 +1.648279394542165631e-07 +1.613506799932028768e-06 +1.590266190346411073e-06 +1.850729048899052037e-07 +-1.388594471725811615e-06 +-1.701019203045762002e-06 +-3.579361805888667291e-07 +1.261769824495432481e-06 +1.726288055965731559e-06 +6.926305783625514968e-07 +-9.844301214014016578e-07 +-1.740464699078939085e-06 +-8.519527518181598992e-07 +8.332674829461268232e-07 +1.741571421405706087e-06 +1.148018638352196632e-06 +-5.090023255024796197e-07 +-1.734609699043444968e-06 +-1.400634254259911360e-06 +3.385189465036579976e-07 +1.696291260974119887e-06 +1.504597220138932502e-06 +1.028066263148378901e-08 +-1.507099488988410566e-06 +-1.655929819017904328e-06 +-1.850607351979993349e-07 +1.388549290265500355e-06 +1.701031011994575546e-06 +5.275539831265354775e-07 +-1.127235640672913983e-06 +-1.737181060595688390e-06 +-8.519527483444261135e-07 +9.843785821019824182e-07 +1.741384825638160630e-06 +1.004344864225491785e-06 +-6.744497073654033995e-07 +-1.741376394832104721e-06 +-1.280894980975188916e-06 +5.089719669952205893e-07 +1.734757892297364502e-06 +1.400690477234687062e-06 +-1.648377135314778650e-07 +-1.613558531366298255e-06 +-1.590222270901464062e-06 +-1.027943369972034169e-08 +1.507045185363996357e-06 +1.655946250243168076e-06 +3.579588754105352231e-07 +-1.261820450975973593e-06 +-1.726149289851318984e-06 +-6.925890043500817579e-07 +1.127178811464622440e-06 +1.736408948979261214e-06 +8.520018235399207189e-07 +-8.333136147244381326e-07 +-1.741693107350031993e-06 +-1.147960319184804599e-06 +6.744107084741447340e-07 +1.741337794941990256e-06 +1.280953962835299388e-06 +-3.385396297787803991e-07 +-1.696308525721171551e-06 +-1.504546450550012052e-06 +-1.027941507945039562e-08 +1.613556261190885380e-06 +1.590266803856713060e-06 +1.850727478384819257e-07 +-1.388601975904503919e-06 +-1.701032983039951580e-06 +-5.275213016298772139e-07 +1.261764715639710775e-06 +1.726304122691646887e-06 +6.926305423912686419e-07 +-9.844325595067758946e-07 +-1.741047477913806350e-06 +-1.004290022246545538e-06 +8.332673411759573693e-07 +1.741457078219720895e-06 +1.148018614368116106e-06 +-5.090028821533999868e-07 +-1.734714497432303231e-06 +-1.400634117236224328e-06 +1.648282962189800269e-07 +1.696385721374679255e-06 +1.504597503501864806e-06 +1.028074892073632168e-08 +-1.507125115336429930e-06 +-1.655930948330665069e-06 +-3.579362270733610067e-07 +1.388532249601626233e-06 +1.701021720937121161e-06 +5.275540004170614873e-07 +-1.127240662429297996e-06 +-1.736866638400707936e-06 +-8.519527506836343347e-07 +8.332683080834365036e-07 +1.741372787339747066e-06 +1.004344826808355347e-06 +-6.744497853296252225e-07 +-1.741162167778092546e-06 +-1.280894975451796334e-06 +3.385194008389073970e-07 +1.734714508869997487e-06 +1.400690561398616406e-06 +-1.648378227492758156e-07 +-1.613741319337345870e-06 +-1.590220371641575078e-06 +-1.850607998008570295e-07 +1.507083042256146194e-06 +1.655942387161881569e-06 +3.579588108104002364e-07 +-1.261850015219649184e-06 +-1.726119544887293523e-06 +-6.925890080527540438e-07 +9.843862574081318839e-07 +1.736932390636651791e-06 +8.520017698944641626e-07 +-8.333131218680515216e-07 +-1.741623702504237426e-06 +-1.147960258204311864e-06 +5.089719878535787572e-07 +1.741455053460048243e-06 +1.280953931907779991e-06 +-3.385392209754343960e-07 +-1.696564441042693487e-06 +-1.504545697033834113e-06 +-1.027955911943850797e-08 +1.507093267830827907e-06 +1.590263201748633413e-06 +1.850728330872704220e-07 +-1.388602988230600451e-06 +-1.700975162906574283e-06 +-5.275213068780647678e-07 +1.127183628247477078e-06 +1.725905081414313856e-06 +6.926305493046929004e-07 +-9.844282250944378751e-07 +-1.740833565780563327e-06 +-1.004289960997508842e-06 +6.744095488001096150e-07 +1.742332148485337142e-06 +1.148018579738191767e-06 +-5.089996976389653168e-07 +-1.734546990192994622e-06 +-1.400633580966856378e-06 +1.648279621732273100e-07 +1.613567778279256706e-06 +1.504597234032535897e-06 +1.028137303124423731e-08 +-1.507049086268338031e-06 +-1.655913108463599139e-06 +-3.579362485832093780e-07 +1.261743205018147734e-06 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 From dbb7c707f130acd9b10a94958e6af7963808a599 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Wed, 29 Jan 2025 20:21:30 -0500 Subject: [PATCH 38/48] weird golds.. --- tst/regression/linear_modes.gold | 2458 +++++++++++++++--------------- 1 file changed, 1229 insertions(+), 1229 deletions(-) diff --git a/tst/regression/linear_modes.gold b/tst/regression/linear_modes.gold index 9d0566086..571fe701f 100644 --- a/tst/regression/linear_modes.gold +++ b/tst/regression/linear_modes.gold @@ -1,1232 +1,1232 @@ -1.000005625775184148e+00 -1.000002760802206403e+00 -9.999972997227719596e-01 -9.999943554832938508e-01 -9.999962801838924165e-01 -1.000001649259240066e+00 -1.000005621297554326e+00 -1.000004150765653632e+00 -9.999989029896425574e-01 -9.999945035842714436e-01 -9.999951247068956661e-01 -9.999999666912248175e-01 -1.000004883287164414e+00 -1.000005153055575136e+00 -1.000000599704507387e+00 -9.999955002429915529e-01 -9.999944883058548006e-01 -9.999982906363023005e-01 -1.000003652471175730e+00 -1.000005626023197314e+00 -1.000002244380173044e+00 -9.999968100129270621e-01 -9.999943613489549188e-01 -9.999967457294383077e-01 -1.000002185343074723e+00 -1.000005642183421362e+00 -1.000003720006108709e+00 -9.999983506243874665e-01 -9.999943787300924480e-01 -9.999954614235876393e-01 -1.000000534103657968e+00 -1.000005228284200687e+00 -1.000004875461331100e+00 -1.000000033314989656e+00 -9.999951161265646649e-01 -9.999946341258477700e-01 -9.999988401549567696e-01 -1.000004088591221585e+00 -1.000005594241463847e+00 -1.000001709471020517e+00 -9.999963472620381211e-01 -9.999943759118674613e-01 -9.999972393585931929e-01 -1.000002700094356101e+00 -1.000005642935601902e+00 -1.000003254449998158e+00 -9.999978145306619437e-01 -9.999943568920527381e-01 -9.999958494270475562e-01 -1.000001096927151556e+00 -1.000005496844230901e+00 -1.000004875462914722e+00 -9.999994658636355283e-01 -9.999947710426326797e-01 -9.999948470257585820e-01 -9.999994003354164551e-01 -1.000004499377124434e+00 -1.000005512032049904e+00 -1.000001159920290927e+00 -9.999959111855447613e-01 -9.999944086350009975e-01 -9.999977557558935581e-01 -1.000003189749025134e+00 -1.000005641981852156e+00 -1.000002760802472412e+00 -9.999972997496957561e-01 -9.999943555922112814e-01 -9.999962801838032656e-01 -1.000001649258354997e+00 -1.000005621006992085e+00 -1.000004538759016182e+00 -9.999989029995278722e-01 -9.999945035047999031e-01 -9.999951247343861205e-01 -9.999999666911814078e-01 -1.000004883310042558e+00 -1.000005365866759366e+00 -1.000000599704985893e+00 -9.999955004505431955e-01 -9.999944883353297786e-01 -9.999982906363189539e-01 -1.000003652478812066e+00 -1.000005627607422731e+00 -1.000002760802608082e+00 -9.999968101024463429e-01 -9.999943588459464516e-01 -9.999967457289983264e-01 -1.000002185341319683e+00 -1.000005642529046224e+00 -1.000004150765845923e+00 -9.999983506354790386e-01 -9.999943795359398369e-01 -9.999954614296149291e-01 -1.000000534103560934e+00 -1.000005228588552564e+00 -1.000005153052095697e+00 -1.000000033314421888e+00 -9.999951167751868208e-01 -9.999946341935053162e-01 -9.999988401550065076e-01 -1.000004088574629524e+00 -1.000005594116402996e+00 -1.000002244380725491e+00 -9.999963473590743890e-01 -9.999943705630350843e-01 -9.999972393585445651e-01 -1.000002700091033647e+00 -1.000005644504989410e+00 -1.000003720006078733e+00 -9.999978145306807065e-01 -9.999943582381595197e-01 -9.999958494249607810e-01 -1.000001096926319555e+00 -1.000005496920314485e+00 -1.000004875456607101e+00 -1.000000033315262993e+00 -9.999947715979419227e-01 -9.999948470606095929e-01 -9.999994003352935534e-01 -1.000004499306196948e+00 -1.000005511892942289e+00 -1.000001709471179279e+00 -9.999959112259785288e-01 -9.999944044212771832e-01 -9.999977557558298313e-01 -1.000003189745025889e+00 -1.000005641977540716e+00 -1.000003254450036350e+00 -9.999972997569587241e-01 -9.999943568080577050e-01 -9.999962801835043935e-01 -1.000001649257916236e+00 -1.000005619863243211e+00 -1.000004538758572092e+00 -9.999994658666658820e-01 -9.999945033589370258e-01 -9.999951246920915082e-01 -9.999999666911157936e-01 -1.000004883392187294e+00 -1.000005365901549537e+00 -1.000001159920163030e+00 -9.999955004701366335e-01 -9.999944875786248577e-01 -9.999982906362654411e-01 -1.000003652477882365e+00 -1.000005631199297751e+00 -1.000002760802340518e+00 -9.999972997609573033e-01 -9.999943572468015240e-01 -9.999967457287104455e-01 -1.000002185341415606e+00 -1.000005642579079534e+00 -1.000004150766196531e+00 -9.999989030082178099e-01 -9.999943789194916111e-01 -9.999954614455011104e-01 -1.000000534103923533e+00 -1.000005228595872486e+00 -1.000005153046126027e+00 -1.000000599705319626e+00 -9.999951162555039685e-01 -9.999946339857431754e-01 -9.999988401548633998e-01 -1.000004088569844463e+00 -1.000005592790779829e+00 -1.000002244380366445e+00 -9.999968100716229991e-01 -9.999943732042494426e-01 -9.999972393582763353e-01 -1.000002700095898867e+00 -1.000005644744252020e+00 -1.000003720006130914e+00 -9.999983506448507642e-01 -9.999943571126526098e-01 -9.999958494295454470e-01 -1.000001096927333633e+00 -1.000005496689988060e+00 -1.000004875457697340e+00 -1.000000033313640513e+00 -9.999951162463665000e-01 -9.999948471061756994e-01 -9.999994003352387084e-01 -1.000004499371139888e+00 -1.000005511919947576e+00 -1.000001709470925038e+00 -9.999963473308361994e-01 -9.999944077223968941e-01 -9.999977557559259767e-01 -1.000003189760898525e+00 -1.000005639429340842e+00 -1.000003254450063883e+00 -9.999978145314553091e-01 -9.999943553535456386e-01 -9.999962801834312298e-01 -1.000001649258108749e+00 -1.000005620825985098e+00 -1.000004538758234807e+00 -9.999994658652128221e-01 -9.999947711403059936e-01 -9.999951247163477719e-01 -9.999999666910157625e-01 -1.000004883516108389e+00 -1.000005365933064549e+00 -1.000001159920125282e+00 -9.999959112128709027e-01 -9.999944880828373073e-01 -9.999982906362719914e-01 -1.000003652479208194e+00 -1.000005627782454054e+00 -1.000002760802314539e+00 -9.999972997549056997e-01 -9.999943551254746321e-01 -9.999967457295289019e-01 -1.000002185338942251e+00 -1.000005642668040151e+00 -1.000004150765919197e+00 -9.999989030074172280e-01 -9.999945030108718935e-01 -9.999954614257452468e-01 -1.000000534103545835e+00 -1.000005228497013121e+00 -1.000005153049556617e+00 -1.000000599705282101e+00 -9.999955004138553205e-01 -9.999946341890698642e-01 -9.999988401548611794e-01 -1.000004088552002957e+00 -1.000005592403996113e+00 -1.000002244380352012e+00 -9.999968100815259664e-01 -9.999943596037572657e-01 -9.999972393583629326e-01 -1.000002700092436525e+00 -1.000005643971312974e+00 -1.000003720006111152e+00 -9.999983506437940539e-01 -9.999943789095604441e-01 -9.999958494266143472e-01 -1.000001096926856681e+00 -1.000005496713704201e+00 -1.000004875451790065e+00 -1.000000033313640069e+00 -9.999951164564397921e-01 -9.999946342331085258e-01 -9.999994003351478922e-01 -1.000004499329271157e+00 -1.000005512015281539e+00 -1.000001709470949018e+00 -9.999963473360324873e-01 -9.999943723320516842e-01 -9.999977557557845342e-01 -1.000003189753668975e+00 -1.000005640383372141e+00 -1.000003254450023027e+00 -9.999978145321564149e-01 -9.999943574830255644e-01 -9.999962801844133331e-01 -1.000001649257747260e+00 -1.000005620446916765e+00 -1.000004538757943040e+00 -9.999994658651829571e-01 -9.999947713501622371e-01 -9.999948470901370845e-01 -9.999999666909942242e-01 -1.000004883426925506e+00 -1.000005365967129745e+00 -1.000001159920161253e+00 -9.999959112302192477e-01 -9.999944062202245076e-01 -9.999982906362955282e-01 -1.000003652477207572e+00 -1.000005629411929498e+00 -1.000002760802297219e+00 -9.999972997554668064e-01 -9.999943565203065488e-01 -9.999962801842935400e-01 -1.000002185338930705e+00 -1.000005642454813604e+00 -1.000004150766473643e+00 -9.999989030074023511e-01 -9.999945030714696426e-01 -9.999951247120826281e-01 -1.000000534103416827e+00 -1.000005228508070276e+00 -1.000005153049992712e+00 -1.000000599705265225e+00 -9.999955004410504555e-01 -9.999944880857016827e-01 -9.999988401550021777e-01 -1.000004088538783531e+00 -1.000005592644174213e+00 -1.000002244380342908e+00 -9.999968100779922375e-01 -9.999943592858060448e-01 -9.999967457292057160e-01 -1.000002700092129659e+00 -1.000005644155826490e+00 -1.000003720006242380e+00 -9.999983506435720093e-01 -9.999943791736130017e-01 -9.999954614274593201e-01 -1.000001096926721900e+00 -1.000005496500181890e+00 -1.000004875453517794e+00 -1.000000033313782621e+00 -9.999951163849680746e-01 -9.999946341765165725e-01 -9.999988401550004014e-01 -1.000004499336313302e+00 -1.000005511922090973e+00 -1.000001709471145972e+00 -9.999963473387932789e-01 -9.999943715944688005e-01 -9.999972393582987618e-01 -1.000003189748610799e+00 -1.000005639112068634e+00 -1.000003254450053669e+00 -9.999978145336728685e-01 -9.999943575200217483e-01 -9.999958494266166786e-01 -1.000001649259649295e+00 -1.000005620341926749e+00 -1.000004538758855643e+00 -9.999994658653188484e-01 -9.999947714096492080e-01 -9.999948470907105147e-01 -9.999994003356920125e-01 -1.000004883371763631e+00 -1.000005365994392381e+00 -1.000001159920419491e+00 -9.999959112441260123e-01 -9.999944071746371677e-01 -9.999977557557465646e-01 -1.000003652483939520e+00 -1.000005630204431117e+00 -1.000002760802239488e+00 -9.999972997536744623e-01 -9.999943565228189835e-01 -9.999962801842512405e-01 -1.000001649259425696e+00 -1.000005642391002203e+00 -1.000004150766384603e+00 -9.999989030061567918e-01 -9.999945031291599395e-01 -9.999951247132606857e-01 -9.999999666913534924e-01 -1.000005228529977197e+00 -1.000005153051072071e+00 -1.000000599705023197e+00 -9.999955004362547362e-01 -9.999944880546194348e-01 -9.999982906363016344e-01 -1.000004088605031205e+00 -1.000005593202574428e+00 -1.000002244380393979e+00 -9.999968100713279018e-01 -9.999943592824395155e-01 -9.999967457292401329e-01 -1.000002185345483907e+00 -1.000005643946614509e+00 -1.000003720006287233e+00 -9.999983506357719154e-01 -9.999943799145210033e-01 -9.999954614274424447e-01 -1.000000534103291816e+00 -1.000005496729241994e+00 -1.000004875454678421e+00 -1.000000033313517278e+00 -9.999951165950072829e-01 -9.999946341601548827e-01 -9.999988401550966577e-01 -1.000004088597379326e+00 -1.000005511932688718e+00 -1.000001709471213696e+00 -9.999963473085962118e-01 -9.999943690904292692e-01 -9.999972393582837737e-01 -1.000002700093403751e+00 -1.000005640071133683e+00 -1.000003254450146484e+00 -9.999978145134313934e-01 -9.999943579543062455e-01 -9.999958494264483688e-01 -1.000001096926753208e+00 -1.000005621367910047e+00 -1.000004538757343298e+00 -9.999994658650096513e-01 -9.999947714452580572e-01 -9.999948471014514784e-01 -9.999994003356373895e-01 -1.000004499431749405e+00 -1.000005365881810881e+00 -1.000001159920228755e+00 -9.999959111303704518e-01 -9.999944046160988975e-01 -9.999977557556637420e-01 -1.000003189742899812e+00 -1.000005625765048034e+00 -1.000002760802242818e+00 -9.999972997503764338e-01 -9.999943553682689723e-01 -9.999962801842160465e-01 -1.000001649259704806e+00 -1.000005621363527553e+00 -1.000004150765572586e+00 -9.999989030014744262e-01 -9.999945031670292028e-01 -9.999951247189768910e-01 -9.999999666911644214e-01 -1.000004883320198656e+00 -1.590258709880481725e-06 -1.850727402495279277e-07 --1.388541769673926653e-06 --1.701017291696231477e-06 --5.275213154298756335e-07 -1.127180152538241350e-06 -1.726491620999807195e-06 -6.926305838952891814e-07 --9.844231305223494377e-07 --1.740702235800200361e-06 --1.004289952315689854e-06 -6.744114282080159465e-07 -1.741052633384862507e-06 -1.148018577632170975e-06 --5.089969905231052128e-07 --1.734358073696038867e-06 --1.400633232471084825e-06 -1.648280955158712544e-07 -1.613497050054945738e-06 -1.504603245167010947e-06 -1.028046073127142543e-08 --1.507032766629912559e-06 --1.655943718395572940e-06 --3.579362193198742469e-07 -1.261768751916517784e-06 -1.726444650652859829e-06 -5.275540717886982975e-07 --1.127233184103640488e-06 --1.738629705526233234e-06 --8.519526867277922594e-07 -8.332669303254228888e-07 -1.742314501460785511e-06 -1.004344893216444626e-06 --6.744446347904300477e-07 --1.741190486442828142e-06 --1.280895671953873192e-06 -3.385187000265915793e-07 -1.696500812349482576e-06 -1.400691858939751180e-06 --1.648380796717641011e-07 --1.613539989580305443e-06 --1.590222811317883300e-06 --1.850607679087758196e-07 -1.388528183416090092e-06 -1.701097407026286314e-06 -3.579591081825361336e-07 --1.261811969936731450e-06 --1.727216086362100305e-06 --6.925890060411777431e-07 -9.843755628953675314e-07 -1.741468567047642194e-06 -8.520019343554935675e-07 --8.333134720201472977e-07 --1.742154442407720301e-06 --1.147960835105777450e-06 -5.089716510225123269e-07 -1.734636259046536423e-06 -1.400691742656256031e-06 --3.385365853475210078e-07 --1.696322441869284449e-06 --1.504545098921229713e-06 --1.027950600037699394e-08 -1.507017642893446091e-06 -1.655912282090512858e-06 -1.850736573253833457e-07 --1.388593103670551729e-06 --1.701123011159464797e-06 --5.275213203113701307e-07 -1.127175887512785350e-06 -1.737084630554120061e-06 -6.926305374906872101e-07 --9.844220556220729739e-07 --1.741418280039016872e-06 --1.004290181794073400e-06 -6.744100915122527316e-07 -1.741136234302182269e-06 -1.280953800339244119e-06 --5.089968161688048876e-07 --1.734469996461215453e-06 --1.400635145554576073e-06 -1.648280393050493682e-07 -1.613482434497394589e-06 -1.590260350272604001e-06 -1.028156819874885824e-08 --1.507095212807075582e-06 --1.655887238029578999e-06 --3.579362205533576648e-07 -1.261753103397525227e-06 -1.726157707051833215e-06 -6.926304404517859172e-07 --1.127228675567379859e-06 --1.736257195298326751e-06 --8.519527011095154370e-07 -8.332680875729218544e-07 -1.741496280575035026e-06 -1.148018540650626313e-06 --6.744480534802965719e-07 --1.741171439352269567e-06 --1.280895278023926046e-06 -3.385191498947034055e-07 -1.696326457884578856e-06 -1.504599592364469134e-06 --1.648375014590240681e-07 --1.613480874528436288e-06 --1.590224424181760898e-06 --1.850607843687100034e-07 -1.388532291010664415e-06 -1.701034487815878602e-06 -5.275537985538963861e-07 --1.261853076819824106e-06 --1.726152893555389757e-06 --6.925890073988103037e-07 -9.843836909720386743e-07 -1.740201062440504268e-06 -1.004344883696872571e-06 --8.333172255183000351e-07 --1.742519564130635849e-06 --1.147960182234560617e-06 -5.089719730719171151e-07 -1.734569191161493850e-06 -1.400691149879991598e-06 --1.648380427839468633e-07 --1.696560750486992819e-06 --1.504551627464877021e-06 --1.027944716728731548e-08 -1.507085960251261140e-06 -1.655975652851829364e-06 -3.579588946777138358e-07 --1.388629294146524730e-06 --1.701182326728532360e-06 --5.275213253407199463e-07 -1.127181229005059510e-06 -1.736714200870366039e-06 -8.520018482203611131e-07 --9.844321538280139964e-07 --1.741225550081222717e-06 --1.004290007461292582e-06 -6.744106687903739636e-07 -1.741462204149202398e-06 -1.280954222530781399e-06 --3.385401668286348946e-07 --1.734649093637919336e-06 --1.400628663622346324e-06 -1.648279882689770145e-07 -1.613614075780983244e-06 -1.590271522813106817e-06 -1.850730311209705578e-07 --1.507153830918298451e-06 --1.655961476946026026e-06 --3.579361763399364898e-07 -1.261759921644646834e-06 -1.725893974381015557e-06 -6.926305516229005278e-07 --9.844361182042738464e-07 --1.736686570992468948e-06 --8.519527346364049901e-07 -8.332672720311000855e-07 -1.741577450178477170e-06 -1.148018643972756175e-06 --5.090028878685917445e-07 --1.741071664343919371e-06 --1.280893007492375202e-06 -3.385190283198168349e-07 -1.696273442558272825e-06 -1.504596047420710591e-06 -1.028067565849080836e-08 --1.613678524575431118e-06 --1.590228735450712356e-06 --1.850607004991338731e-07 -1.388526752923310147e-06 -1.700983221527749291e-06 -5.275539677781154082e-07 --1.127237941922898376e-06 --1.726101689114446397e-06 --6.925889769606349690e-07 -9.843780847074831729e-07 -1.740988709107291496e-06 -1.004344852365307235e-06 --6.744507678687679026e-07 --1.741609546519275544e-06 --1.147960506446668160e-06 -5.089723585953228279e-07 -1.734442162620808360e-06 -1.400690583567017859e-06 --1.648377356755172739e-07 --1.613716912519909083e-06 --1.504541679001261497e-06 --1.027937814965926895e-08 -1.507029924103069356e-06 -1.655965816649125593e-06 -3.579589664754058362e-07 --1.261828778088528421e-06 --1.700922393705801968e-06 --5.275213071047663598e-07 -1.127183456889939933e-06 -1.737589558734849914e-06 -8.520018474006426491e-07 --8.333136941412435331e-07 --1.741108267578420554e-06 --1.004289945897832174e-06 -6.744118128911313061e-07 -1.741426051359071790e-06 -1.280953975366231753e-06 --3.385392001881804617e-07 --1.696289274454028947e-06 --1.400634676346342227e-06 -1.648280454427183166e-07 -1.613620484792240372e-06 -1.590266227185648271e-06 -1.850729201873577123e-07 --1.388598740879103959e-06 --1.655949969668479244e-06 --3.579361874326070693e-07 -1.261769798818398850e-06 -1.726421594208738017e-06 -6.926305803879282936e-07 --9.844292928256493959e-07 --1.741103988185197408e-06 --8.519527256967228089e-07 -8.332681204532097489e-07 -1.742137816808617435e-06 -1.148018603647752241e-06 --5.090022385865434066e-07 --1.734594009746377118e-06 --1.280895065173918608e-06 -3.385189830825236208e-07 -1.696358701328580575e-06 -1.504598802422561717e-06 -1.028066519374389938e-08 --1.507085425042355993e-06 --1.590223265583818352e-06 --1.850607303696900364e-07 -1.388530208700179477e-06 -1.701003410918592177e-06 -5.275539720348859433e-07 --1.127236328972740943e-06 --1.737072019608247472e-06 --6.925890397434343505e-07 -9.843797688342706320e-07 -1.741151865984534068e-06 -1.004344847574267386e-06 --6.744500214686274675e-07 --1.741180623148688443e-06 --1.147960241359076898e-06 -5.089720195309898694e-07 -1.734542070068430320e-06 -1.400691576856052290e-06 --1.648377264877983024e-07 --1.613567264205507283e-06 --1.590220997882724452e-06 --1.027945266177097877e-08 -1.507054779731643882e-06 -1.655948986141254314e-06 -3.579589405865119535e-07 --1.261822669735039415e-06 --1.726248378931100790e-06 --5.275213256528804924e-07 -1.127180212960296698e-06 -1.737145793846374856e-06 -8.520018437027837338e-07 --8.333135839839537744e-07 --1.741826166559895173e-06 --1.004290002577691937e-06 -6.744107524456056007e-07 -1.741434120495588456e-06 -1.280954066336914685e-06 --3.385393446547938588e-07 --1.696408346730930952e-06 --1.504547546145360437e-06 -1.648279238310219440e-07 -1.613593358049607904e-06 -1.590265266622223248e-06 -1.850728906943091782e-07 --1.388592312864333375e-06 --1.701033181207115733e-06 --3.579361994878603425e-07 -1.261762142237637242e-06 -1.726154469385124064e-06 -6.926305729210776948e-07 --9.844306732773516617e-07 --1.740935075061179071e-06 --1.004290005992815914e-06 -8.332668107363248065e-07 -1.742047849474825750e-06 -1.148018594455816554e-06 --5.090024883766607723e-07 --1.734576814070674148e-06 --1.400634242297095216e-06 -3.385188554784090977e-07 -1.696263364638979683e-06 -1.504596670866742508e-06 -1.028062377219007074e-08 --1.507102089845137072e-06 --1.655924495095443164e-06 --1.850607415461862521e-07 -1.388521422721493471e-06 -1.700993753069471308e-06 -5.275539553648519073e-07 --1.127238335727100728e-06 --1.737259124443960916e-06 --8.519527452616847480e-07 -9.843784977436696009e-07 -1.741509171049854340e-06 -1.004344852353452585e-06 --6.744502569944053872e-07 --1.741384013451149539e-06 --1.280894988554857104e-06 -5.089720508299514252e-07 -1.734418632636085218e-06 -1.400690944410196418e-06 --1.648377472793484936e-07 --1.613640522525692076e-06 --1.590222066274167874e-06 --1.850607705281996391e-07 -1.507045512717325373e-06 -1.655953953555566205e-06 -3.579588754104873128e-07 --1.261827371769381423e-06 --1.726241646624968192e-06 --6.925890056911455069e-07 -1.127179221072367417e-06 -1.737438349978383794e-06 -8.520018353432443708e-07 --8.333134243790168780e-07 --1.742091801623881296e-06 --1.147960315055135911e-06 -6.744108640788155416e-07 -1.741383014368384549e-06 -1.280954068588190404e-06 --3.385395013434910266e-07 --1.696407975714907660e-06 --1.504546496833054631e-06 --1.027945156890405462e-08 -1.613558248167692884e-06 -1.590264887097033811e-06 -1.850727480043968336e-07 --1.388582710117581190e-06 --1.701007943560229509e-06 --5.275213266703994667e-07 -1.261761576141305520e-06 -1.726000676928844671e-06 -6.926305956639591272e-07 --9.844267280894752903e-07 --1.741512944258879865e-06 --1.004290016499870624e-06 -6.744106055143346973e-07 -1.741555952554223046e-06 -1.148018595474808413e-06 --5.090019614378216274e-07 --1.734591562866344000e-06 --1.400634284221151453e-06 -1.648279524011102915e-07 -1.696295494532174206e-06 -1.504597371495823923e-06 -1.028054244384596495e-08 --1.507071808459608073e-06 --1.655908573310841154e-06 --3.579362582855198438e-07 -1.388517896125253766e-06 -1.701017301470918208e-06 -5.275539556353993128e-07 --1.127217618902127510e-06 --1.736963978597491788e-06 --8.519527402804675796e-07 -8.332656401994154939e-07 -1.741378217346416395e-06 -1.004344844050856600e-06 --6.744444954404536350e-07 --1.741344194946463995e-06 --1.280894810056264701e-06 -3.385185005648651511e-07 -1.734500300970092121e-06 -1.400690527060552305e-06 --1.648381025857745115e-07 --1.613514314594990808e-06 --1.590218367364895719e-06 --1.850608069330568487e-07 -1.388516357169056198e-06 -1.655945546814741661e-06 -3.579588203429640820e-07 --1.261782497877215767e-06 --1.725857671376120261e-06 --6.925889913294783754e-07 -9.843740187301974938e-07 -1.737679640457574131e-06 -8.520018294553982875e-07 --8.333077363292762579e-07 --1.741350671057119731e-06 --1.147960198128919997e-06 -5.089719448722273021e-07 -1.741072341592975031e-06 -1.280953859019597922e-06 --3.385398952397837006e-07 --1.696051325085198054e-06 --1.504543627573693006e-06 --1.027935900639229383e-08 -1.506996708215537967e-06 -1.590261173452735001e-06 -1.850728075432969316e-07 --1.388549659267483009e-06 --1.701102775001740137e-06 --5.275213000104285693e-07 -1.127181349487917218e-06 -1.726545434173700165e-06 -6.926306403836626415e-07 --9.844364571400124864e-07 --1.739552051894324131e-06 --1.004289941535728116e-06 -6.744120903403956215e-07 -1.741029127265671066e-06 -1.148018530681769560e-06 --5.089966504975827709e-07 --1.734766272327423066e-06 --1.400632526955695013e-06 -1.648280872786722481e-07 -1.613579841192132622e-06 -1.504596093198193721e-06 -1.028166291379170719e-08 --1.507180347925612950e-06 --1.655880508224637709e-06 --3.579362120797978204e-07 -1.261744760914462781e-06 -1.701066777656902194e-06 -5.275540513833159856e-07 --1.127243239365637466e-06 --1.737121590860897213e-06 --8.519526930896695320e-07 -8.332666001719421804e-07 -1.742288573428257718e-06 -1.004344838581993247e-06 --6.744443069608969913e-07 --1.741736801215740741e-06 --1.280894858590050639e-06 -3.385191693555600721e-07 -1.696188634757403962e-06 -1.400692097674729520e-06 --1.648376974355455552e-07 --1.613589568463775903e-06 --1.590217322025785994e-06 --1.850606813297324558e-07 -1.388517346821858019e-06 -1.701060300065911117e-06 -3.579588228472856531e-07 --1.261814090928261297e-06 --1.725769989100899558e-06 --6.925890047770793251e-07 -9.843808966886625604e-07 -1.739938131701850016e-06 -8.520018907075687934e-07 --8.333163162997386263e-07 --1.742726832424178203e-06 --1.147960427891790358e-06 -5.089724213524740518e-07 -1.734550541368317171e-06 -1.280954140092062080e-06 --3.385398077827267889e-07 --1.696243955884063799e-06 --1.504544917060379414e-06 --1.027936504216994339e-08 -1.507069983893762695e-06 -1.655982354507205710e-06 -1.850729744934016058e-07 --1.388623461001337052e-06 --1.701187672885859959e-06 --5.275213246265532225e-07 -1.127182163690784350e-06 -1.737875380626029099e-06 -6.926305334885291674e-07 --9.844297310867681608e-07 --1.740987370623198207e-06 --1.004290109880700681e-06 -6.744115823117965700e-07 -1.741332681612490186e-06 -1.280954073280507716e-06 --5.089958973099097255e-07 --1.734428316721273204e-06 --1.400630363082169511e-06 -1.648280114153578379e-07 -1.613587251137130100e-06 -1.590257712069878907e-06 -1.028162319877566815e-08 --1.507123472649657072e-06 --1.656026673134441481e-06 --3.579362138230268306e-07 -1.261765932968881193e-06 -1.725849496633837332e-06 -5.275538640396327477e-07 --1.127226714463011616e-06 --1.735888205023552666e-06 --8.519527164467000378e-07 -8.332671510606626846e-07 -1.742076328180530369e-06 -1.148018497632888122e-06 --6.744419986192260213e-07 --1.741087924878268738e-06 --1.280895072535686088e-06 -3.385186950226027242e-07 -1.696444750421543404e-06 -1.504597086469492552e-06 --1.648375210356707209e-07 --1.613646454703819000e-06 --1.590229964066098528e-06 --1.850607901780342002e-07 -1.388525104051798746e-06 -1.701031427843256524e-06 -5.275538641904303409e-07 --1.261802586419445627e-06 --1.726519944635073156e-06 --6.925889748452972382e-07 -9.843743424279099744e-07 -1.740619921443903349e-06 -1.004344848179437275e-06 --8.333060441746281620e-07 --1.741997207858713043e-06 --1.147960334818376722e-06 -5.089717784803551655e-07 -1.734438698613764786e-06 -1.400690994529669114e-06 --3.385392492815753392e-07 --1.696032358417492168e-06 --1.504546007199993032e-06 --1.027937479700997961e-08 -1.507000651863114781e-06 -1.655949143046259821e-06 -3.579588447697392843e-07 --1.388596859089842529e-06 --1.700945379316332144e-06 --5.275213139274402495e-07 -1.127175366846241379e-06 -1.737338985482136595e-06 -8.520018480164420604e-07 --9.844242560294862260e-07 --1.739567432785072124e-06 --1.004289853599555752e-06 -6.744114505553351100e-07 -1.741105823579371588e-06 -1.280954036891148774e-06 --3.385393510609883186e-07 --1.734659917651862858e-06 --1.400636506470185294e-06 -1.648280844023104601e-07 -1.613517913534661131e-06 -1.590270511347760083e-06 -1.850729527781549711e-07 --1.507126222466079221e-06 --1.656024543609069937e-06 --3.579361632341287926e-07 -1.261766949185430122e-06 -1.726403622829481694e-06 -6.926305417498411900e-07 --1.127229822279195776e-06 --1.738721891103089493e-06 --8.519527005897984670e-07 -8.332677691466162183e-07 -1.741387876275864048e-06 -1.148018684700642225e-06 --5.090018832493002919e-07 --1.741396064626738821e-06 --1.280893680614266850e-06 -3.385189161820171912e-07 -1.696365357801187906e-06 -1.504601305946100156e-06 -1.028070636010076308e-08 --1.613504046691272219e-06 --1.590228768881209665e-06 --1.850607021379894719e-07 -1.388533158037275724e-06 -1.700993704208600421e-06 -5.275539425489083079e-07 --1.127227302984051667e-06 --1.725761071736155471e-06 --6.925889864461445462e-07 -9.843777984914883103e-07 -1.741707439146850073e-06 -1.004344866413877849e-06 --6.744499271448763110e-07 --1.742301470748352831e-06 --1.147960505668042653e-06 -5.089721092365632374e-07 -1.734692526625950660e-06 -1.400690738996414468e-06 --1.648377664950101919e-07 --1.696481721756107406e-06 --1.504535658134828716e-06 --1.027945642029108999e-08 -1.507038462891205193e-06 -1.655931791920991752e-06 -3.579589416177502146e-07 --1.261825244286295888e-06 --1.701096132573522042e-06 --5.275213078853165381e-07 -1.127178254342393248e-06 -1.736675458571481429e-06 -8.520018446102271767e-07 --8.333132946683656141e-07 --1.741916864233069986e-06 --1.004290180398935809e-06 -6.744108342309223272e-07 -1.741463583973312753e-06 -1.280953857998383449e-06 --3.385394450034097421e-07 --1.696418732256577872e-06 --1.400632910854172307e-06 -1.648278809057894656e-07 -1.613455641596044816e-06 -1.590268176716472360e-06 -1.850729003681355142e-07 --1.388590622007244039e-06 --1.655880814542978364e-06 --3.579361863642931702e-07 -1.261757307338035565e-06 -1.726175047362156238e-06 -6.926305767162723425e-07 --9.844320923223519404e-07 --1.735997048522879651e-06 --8.519526792089144618e-07 -8.332669767177802576e-07 -1.741753592211998155e-06 -1.148018598796977087e-06 --5.090024483558281695e-07 --1.734742097728113351e-06 --1.280895070223601450e-06 -3.385189691855595693e-07 -1.696209839631811581e-06 -1.504598674809486538e-06 -1.028063672740347746e-08 --1.507118270675432684e-06 --1.590225360960248558e-06 --1.850606841024569039e-07 -1.388536125426654509e-06 -1.701073599894859555e-06 -5.275539714795897778e-07 --1.127238326380262197e-06 --1.736386034132772162e-06 --6.925890310144362625e-07 -9.843783720537162368e-07 -1.741184655186639981e-06 -1.004344859441287492e-06 --6.744501205690743933e-07 --1.741347003965438300e-06 --1.147960188013552591e-06 -5.089723477368453705e-07 -1.734683605293342141e-06 -1.400691216655022114e-06 --1.648377232827669521e-07 --1.613639073771378785e-06 --1.504546951810305930e-06 --1.027935849609450054e-08 -1.507031901604741637e-06 -1.655970651008055552e-06 -3.579589442097077203e-07 --1.261822109719881423e-06 --1.726100093657448231e-06 --5.275213162748761252e-07 -1.127179837933315962e-06 -1.736693167282370843e-06 -8.520018476712604643e-07 --8.333134571715394537e-07 --1.742135981851321251e-06 --1.004289964789669630e-06 -6.744117110257168481e-07 -1.741323323509899397e-06 -1.280954131873073330e-06 --3.385392883702650713e-07 --1.696409550551136906e-06 --1.504546208740563941e-06 -1.648279394542165631e-07 -1.613506799932028768e-06 -1.590266190346411073e-06 -1.850729048899052037e-07 --1.388594471725811615e-06 --1.701019203045762002e-06 --3.579361805888667291e-07 -1.261769824495432481e-06 -1.726288055965731559e-06 -6.926305783625514968e-07 --9.844301214014016578e-07 --1.740464699078939085e-06 --8.519527518181598992e-07 -8.332674829461268232e-07 -1.741571421405706087e-06 -1.148018638352196632e-06 --5.090023255024796197e-07 --1.734609699043444968e-06 --1.400634254259911360e-06 -3.385189465036579976e-07 -1.696291260974119887e-06 -1.504597220138932502e-06 -1.028066263148378901e-08 --1.507099488988410566e-06 --1.655929819017904328e-06 --1.850607351979993349e-07 -1.388549290265500355e-06 -1.701031011994575546e-06 -5.275539831265354775e-07 --1.127235640672913983e-06 --1.737181060595688390e-06 --8.519527483444261135e-07 -9.843785821019824182e-07 -1.741384825638160630e-06 -1.004344864225491785e-06 --6.744497073654033995e-07 --1.741376394832104721e-06 --1.280894980975188916e-06 -5.089719669952205893e-07 -1.734757892297364502e-06 -1.400690477234687062e-06 --1.648377135314778650e-07 --1.613558531366298255e-06 --1.590222270901464062e-06 --1.027943369972034169e-08 -1.507045185363996357e-06 -1.655946250243168076e-06 -3.579588754105352231e-07 --1.261820450975973593e-06 --1.726149289851318984e-06 --6.925890043500817579e-07 -1.127178811464622440e-06 -1.736408948979261214e-06 -8.520018235399207189e-07 --8.333136147244381326e-07 --1.741693107350031993e-06 --1.147960319184804599e-06 -6.744107084741447340e-07 -1.741337794941990256e-06 -1.280953962835299388e-06 --3.385396297787803991e-07 --1.696308525721171551e-06 --1.504546450550012052e-06 --1.027941507945039562e-08 -1.613556261190885380e-06 -1.590266803856713060e-06 -1.850727478384819257e-07 --1.388601975904503919e-06 --1.701032983039951580e-06 --5.275213016298772139e-07 -1.261764715639710775e-06 -1.726304122691646887e-06 -6.926305423912686419e-07 --9.844325595067758946e-07 --1.741047477913806350e-06 --1.004290022246545538e-06 -8.332673411759573693e-07 -1.741457078219720895e-06 -1.148018614368116106e-06 --5.090028821533999868e-07 --1.734714497432303231e-06 --1.400634117236224328e-06 -1.648282962189800269e-07 -1.696385721374679255e-06 -1.504597503501864806e-06 -1.028074892073632168e-08 --1.507125115336429930e-06 --1.655930948330665069e-06 --3.579362270733610067e-07 -1.388532249601626233e-06 -1.701021720937121161e-06 -5.275540004170614873e-07 --1.127240662429297996e-06 --1.736866638400707936e-06 --8.519527506836343347e-07 -8.332683080834365036e-07 -1.741372787339747066e-06 -1.004344826808355347e-06 --6.744497853296252225e-07 --1.741162167778092546e-06 --1.280894975451796334e-06 -3.385194008389073970e-07 -1.734714508869997487e-06 -1.400690561398616406e-06 --1.648378227492758156e-07 --1.613741319337345870e-06 --1.590220371641575078e-06 --1.850607998008570295e-07 -1.507083042256146194e-06 -1.655942387161881569e-06 -3.579588108104002364e-07 --1.261850015219649184e-06 --1.726119544887293523e-06 --6.925890080527540438e-07 -9.843862574081318839e-07 -1.736932390636651791e-06 -8.520017698944641626e-07 --8.333131218680515216e-07 --1.741623702504237426e-06 --1.147960258204311864e-06 -5.089719878535787572e-07 -1.741455053460048243e-06 -1.280953931907779991e-06 --3.385392209754343960e-07 --1.696564441042693487e-06 --1.504545697033834113e-06 --1.027955911943850797e-08 -1.507093267830827907e-06 -1.590263201748633413e-06 -1.850728330872704220e-07 --1.388602988230600451e-06 --1.700975162906574283e-06 --5.275213068780647678e-07 -1.127183628247477078e-06 -1.725905081414313856e-06 -6.926305493046929004e-07 --9.844282250944378751e-07 --1.740833565780563327e-06 --1.004289960997508842e-06 -6.744095488001096150e-07 -1.742332148485337142e-06 -1.148018579738191767e-06 --5.089996976389653168e-07 --1.734546990192994622e-06 --1.400633580966856378e-06 -1.648279621732273100e-07 -1.613567778279256706e-06 -1.504597234032535897e-06 -1.028137303124423731e-08 --1.507049086268338031e-06 --1.655913108463599139e-06 --3.579362485832093780e-07 -1.261743205018147734e-06 +1.000005632655319499e+00 +1.000002760808383684e+00 +9.999972996949745285e-01 +9.999943565266583567e-01 +9.999962802371848980e-01 +1.000001649208487331e+00 +1.000005620483547686e+00 +1.000004150727892061e+00 +9.999989029694753562e-01 +9.999945052546052038e-01 +9.999951245617294537e-01 +9.999999666961703060e-01 +1.000004883968774960e+00 +1.000005153020244952e+00 +1.000000599734518048e+00 +9.999955005058174251e-01 +9.999944869738877085e-01 +9.999982906507698388e-01 +1.000003652558603573e+00 +1.000005628965033422e+00 +1.000002244374423199e+00 +9.999968100084866141e-01 +9.999943649241527233e-01 +9.999967457378612368e-01 +1.000002185287540257e+00 +1.000005642989404420e+00 +1.000003719999225105e+00 +9.999983506855543158e-01 +9.999943790434682178e-01 +9.999954615166134486e-01 +1.000000534120877971e+00 +1.000005228450156824e+00 +1.000004875483002431e+00 +1.000000033345461725e+00 +9.999951164123052072e-01 +9.999946323421767813e-01 +9.999988401746482403e-01 +1.000004088373919187e+00 +1.000005589093612368e+00 +1.000001709466136646e+00 +9.999963473093869126e-01 +9.999943761293018651e-01 +9.999972393619038780e-01 +1.000002700093578278e+00 +1.000005644115180781e+00 +1.000003254457284108e+00 +9.999978146264839607e-01 +9.999943568631461943e-01 +9.999958495106286982e-01 +1.000001096971994130e+00 +1.000005495535843947e+00 +1.000004875450465347e+00 +9.999994658331410324e-01 +9.999947709284359165e-01 +9.999948465842898315e-01 +9.999994003330860970e-01 +1.000004499339977482e+00 +1.000005513960985581e+00 +1.000001159938610495e+00 +9.999959108496043747e-01 +9.999944056627527722e-01 +9.999977557711768883e-01 +1.000003189886431443e+00 +1.000005636202187853e+00 +1.000002760808383684e+00 +9.999972998744335317e-01 +9.999943559924534542e-01 +9.999962802054666033e-01 +1.000001649278303040e+00 +1.000005620628910519e+00 +1.000004538623388450e+00 +9.999989029677013308e-01 +9.999945028654575463e-01 +9.999951245676423905e-01 +9.999999667078377508e-01 +1.000004883749974649e+00 +1.000005367520260258e+00 +1.000000599734518048e+00 +9.999955009783095816e-01 +9.999944860242014899e-01 +9.999982906563524843e-01 +1.000003652592456493e+00 +1.000005626612752385e+00 +1.000002760794253653e+00 +9.999968099938674193e-01 +9.999943548367293822e-01 +9.999967457329563825e-01 +1.000002185325947535e+00 +1.000005642809836504e+00 +1.000004150697972882e+00 +9.999983506855543158e-01 +9.999943783760981697e-01 +9.999954615040358430e-01 +1.000000534158342669e+00 +1.000005228311358962e+00 +1.000005153536246860e+00 +1.000000033271464250e+00 +9.999951155518908008e-01 +9.999946330534588146e-01 +9.999988401823904915e-01 +1.000004088583912543e+00 +1.000005588376624566e+00 +1.000002244346593239e+00 +9.999963473093869126e-01 +9.999943743511141570e-01 +9.999972393620715216e-01 +1.000002700174408510e+00 +1.000005643771175512e+00 +1.000003719963573623e+00 +9.999978144589464213e-01 +9.999943569558090717e-01 +9.999958495208238762e-01 +1.000001096972067183e+00 +1.000005495200696481e+00 +1.000004875516921521e+00 +1.000000033248674702e+00 +9.999947709284359165e-01 +9.999948467812824759e-01 +9.999994003578855928e-01 +1.000004499543848624e+00 +1.000005513648303479e+00 +1.000001709420340612e+00 +9.999959110738979584e-01 +9.999944112094069082e-01 +9.999977557586826604e-01 +1.000003189792319835e+00 +1.000005635582378538e+00 +1.000003254426833799e+00 +9.999972996949745285e-01 +9.999943559924534542e-01 +9.999962802051375332e-01 +1.000001649284675054e+00 +1.000005621498109232e+00 +1.000004538640674401e+00 +9.999994658109249146e-01 +9.999945035818743611e-01 +9.999951246263967253e-01 +9.999999666998068415e-01 +1.000004883316716331e+00 +1.000005367234137355e+00 +1.000001159884960744e+00 +9.999955005058174251e-01 +9.999944860242014899e-01 +9.999982906307588459e-01 +1.000003652605387927e+00 +1.000005624334296517e+00 +1.000002760792013445e+00 +9.999972997383014262e-01 +9.999943614488633337e-01 +9.999967457341174537e-01 +1.000002185393187970e+00 +1.000005642143994233e+00 +1.000004150708544870e+00 +9.999989029061402412e-01 +9.999943790434682178e-01 +9.999954615040358430e-01 +1.000000534081493475e+00 +1.000005228295131277e+00 +1.000005153196704244e+00 +1.000000599682605351e+00 +9.999951170624205021e-01 +9.999946333588105585e-01 +9.999988401642738722e-01 +1.000004088706652361e+00 +1.000005595181765905e+00 +1.000002244379371907e+00 +9.999968101646514729e-01 +9.999943761293018651e-01 +9.999972393620715216e-01 +1.000002700108178377e+00 +1.000005642882210610e+00 +1.000003720001498175e+00 +9.999983506322962512e-01 +9.999943596158442638e-01 +9.999958494945223597e-01 +1.000001096905887232e+00 +1.000005495644664899e+00 +1.000004875475132726e+00 +1.000000033312556269e+00 +9.999951170408881707e-01 +9.999948465842898315e-01 +9.999994003578855928e-01 +1.000004499522637147e+00 +1.000005512885039138e+00 +1.000001709484434675e+00 +9.999963472312672907e-01 +9.999944020628815933e-01 +9.999977557613660695e-01 +1.000003189716275331e+00 +1.000005639801760493e+00 +1.000003254451671930e+00 +9.999978145217349734e-01 +9.999943565266583567e-01 +9.999962802054666033e-01 +1.000001649284675054e+00 +1.000005620784933713e+00 +1.000004538678776589e+00 +9.999994658770350320e-01 +9.999947714133969878e-01 +9.999951245978127012e-01 +9.999999667342928111e-01 +1.000004883138554179e+00 +1.000005367418171476e+00 +1.000001159934313488e+00 +9.999959109807341484e-01 +9.999944869738877085e-01 +9.999982906563524843e-01 +1.000003652605387927e+00 +1.000005619484136021e+00 +1.000002760804144186e+00 +9.999972997261444840e-01 +9.999943567613929218e-01 +9.999967457339060672e-01 +1.000002185403360278e+00 +1.000005642411317064e+00 +1.000004150689643767e+00 +9.999989030539646606e-01 +9.999945040795561457e-01 +9.999954615166134486e-01 +1.000000534158342669e+00 +1.000005228295131277e+00 +1.000005153882767006e+00 +1.000000599705133553e+00 +9.999955003056808511e-01 +9.999946334501745859e-01 +9.999988401734757337e-01 +1.000004088639646405e+00 +1.000005593670743709e+00 +1.000002244387094841e+00 +9.999968100584690767e-01 +9.999943599343450451e-01 +9.999972393619038780e-01 +1.000002700174408510e+00 +1.000005642882210610e+00 +1.000003719982992090e+00 +9.999983506613785433e-01 +9.999943787217002722e-01 +9.999958495178359330e-01 +1.000001096954051372e+00 +1.000005496312835085e+00 +1.000004875478968547e+00 +1.000000033287579804e+00 +9.999951168764663612e-01 +9.999946331056811522e-01 +9.999994003330860970e-01 +1.000004499543848624e+00 +1.000005512885039138e+00 +1.000001709477368106e+00 +9.999963473939638137e-01 +9.999943780502227320e-01 +9.999977557667091288e-01 +1.000003189830621864e+00 +1.000005637416242932e+00 +1.000003254456625301e+00 +9.999978144411570957e-01 +9.999943589123083676e-01 +9.999962802371848980e-01 +1.000001649278303040e+00 +1.000005621498109232e+00 +1.000004538678776589e+00 +9.999994658622803900e-01 +9.999947722049712384e-01 +9.999948463156687062e-01 +9.999999666883998550e-01 +1.000004883059735450e+00 +1.000005366258192918e+00 +1.000001159913265214e+00 +9.999959109662573953e-01 +9.999944021225303237e-01 +9.999982906507698388e-01 +1.000003652592456493e+00 +1.000005624334296517e+00 +1.000002760804144186e+00 +9.999972997290592636e-01 +9.999943563340063513e-01 +9.999962802322818201e-01 +1.000002185330710391e+00 +1.000005640836852283e+00 +1.000004150751921728e+00 +9.999989029865307133e-01 +9.999945034238885144e-01 +9.999951246482331468e-01 +1.000000534120877971e+00 +1.000005228311358962e+00 +1.000005153196704244e+00 +1.000000599705133553e+00 +9.999955002942768623e-01 +9.999944865621374079e-01 +9.999988401662210924e-01 +1.000004088660306545e+00 +1.000005597738118590e+00 +1.000002244387627748e+00 +9.999968100301691587e-01 +9.999943593569411426e-01 +9.999967457255370951e-01 +1.000002700093578278e+00 +1.000005643771175512e+00 +1.000003720001498175e+00 +9.999983506613785433e-01 +9.999943800857316045e-01 +9.999954615287282023e-01 +1.000001096930779321e+00 +1.000005496538477034e+00 +1.000004875481131261e+00 +1.000000033332056004e+00 +9.999951162199782750e-01 +9.999946336329671448e-01 +9.999988401593464804e-01 +1.000004499339977482e+00 +1.000005513648303479e+00 +1.000001709484434675e+00 +9.999963473939638137e-01 +9.999943649475041552e-01 +9.999972393553445693e-01 +1.000003189733285724e+00 +1.000005644760634915e+00 +1.000003254444560064e+00 +9.999978145439466504e-01 +9.999943571569811507e-01 +9.999958494474100457e-01 +1.000001649208487331e+00 +1.000005620628910519e+00 +1.000004538640674401e+00 +9.999994658770350320e-01 +9.999947722049712384e-01 +9.999948468252387590e-01 +9.999994003217230754e-01 +1.000004883005678025e+00 +1.000005366956320030e+00 +1.000001159915184790e+00 +9.999959114697550833e-01 +9.999944071037120130e-01 +9.999977557691764884e-01 +1.000003652558603573e+00 +1.000005626612752385e+00 +1.000002760792013445e+00 +9.999972997261444840e-01 +9.999943563340063513e-01 +9.999962801845050375e-01 +1.000001649218661415e+00 +1.000005641543503909e+00 +1.000004150665656066e+00 +9.999989029996332324e-01 +9.999945047613947313e-01 +9.999951247846718960e-01 +9.999999667155246019e-01 +1.000005228450156824e+00 +1.000005153536246860e+00 +1.000000599682605351e+00 +9.999955003056808511e-01 +9.999944865621374079e-01 +9.999982906611800670e-01 +1.000004088755765075e+00 +1.000005598271136220e+00 +1.000002244348032532e+00 +9.999968100916505342e-01 +9.999943640835056158e-01 +9.999967457235865442e-01 +1.000002185347521610e+00 +1.000005644115180781e+00 +1.000003719963573623e+00 +9.999983506322962512e-01 +9.999943787217002722e-01 +9.999954615287282023e-01 +1.000000534153414167e+00 +1.000005496702192076e+00 +1.000004875448733621e+00 +1.000000033272859801e+00 +9.999951163138114385e-01 +9.999946326145957576e-01 +9.999988401736024102e-01 +1.000004088469371721e+00 +1.000005513960985581e+00 +1.000001709420340612e+00 +9.999963472312672907e-01 +9.999943780502227320e-01 +9.999972393553445693e-01 +1.000002700212754503e+00 +1.000005645330762860e+00 +1.000003254434076005e+00 +9.999978145630771253e-01 +9.999943576993874927e-01 +9.999958495086429533e-01 +1.000001096954989510e+00 +1.000005620483547686e+00 +1.000004538623388450e+00 +9.999994658109249146e-01 +9.999947714133969878e-01 +9.999948463156687062e-01 +9.999994003217230754e-01 +1.000004499638560196e+00 +1.000005366980012411e+00 +1.000001159875671508e+00 +9.999959114989783737e-01 +9.999944078390725632e-01 +9.999977557599983857e-01 +1.000003189869540288e+00 +1.000005628965033422e+00 +1.000002760794253653e+00 +9.999972997383014262e-01 +9.999943567613929218e-01 +9.999962802322818201e-01 +1.000001649218661415e+00 +1.000005620196233735e+00 +1.000004150703476480e+00 +9.999989029721221279e-01 +9.999945044132163563e-01 +9.999951245066918126e-01 +9.999999666814920474e-01 +1.000004883573349268e+00 +1.590367539041457537e-06 +1.850912408548692162e-07 +-1.388671263919384155e-06 +-1.701402812589428375e-06 +-5.275245537920439392e-07 +1.127207486787609383e-06 +1.725513118501249725e-06 +6.926330706003840947e-07 +-9.843926778009791954e-07 +-1.741847553483955041e-06 +-1.004286466214067983e-06 +6.744469082427657453e-07 +1.741808948583900944e-06 +1.148018083313236715e-06 +-5.089801355624572556e-07 +-1.734300937657732912e-06 +-1.400619844916229366e-06 +1.648243680517588008e-07 +1.613372567611652862e-06 +1.504581194016223573e-06 +1.027057519218336824e-08 +-1.506965207472381200e-06 +-1.656413222052862091e-06 +-3.579387071900327507e-07 +1.261832909686356920e-06 +1.726530510927856447e-06 +5.275505036001899248e-07 +-1.127283476069760551e-06 +-1.738991200931393030e-06 +-8.519558241342447753e-07 +8.332683681776776070e-07 +1.741543189546330788e-06 +1.004348184969015373e-06 +-6.744859534995672576e-07 +-1.741128142201565000e-06 +-1.280871119453824414e-06 +3.385081253452617528e-07 +1.695890104207147870e-06 +1.400661979984600656e-06 +-1.648645316816077902e-07 +-1.613580376379654210e-06 +-1.590413693388966491e-06 +-1.850598198757230344e-07 +1.388569531686582505e-06 +1.701594467880254092e-06 +3.579597281925138896e-07 +-1.262011462870778688e-06 +-1.727044134608643941e-06 +-6.925960486877090200e-07 +9.843413329179845028e-07 +1.741951384656060736e-06 +8.520003192028401959e-07 +-8.334344866947607621e-07 +-1.741882352976100373e-06 +-1.147954552841384033e-06 +5.089651065550712443e-07 +1.734499890489319603e-06 +1.400657029138366716e-06 +-3.385102729909338517e-07 +-1.696068296688998337e-06 +-1.504578035365303085e-06 +-1.027792459486570175e-08 +1.506889434950186658e-06 +1.656390619447936956e-06 +1.850579395021460447e-07 +-1.388808107645545854e-06 +-1.701449497581291826e-06 +-5.275302993430160891e-07 +1.127129333673650356e-06 +1.739139556716176440e-06 +6.926275968399265429e-07 +-9.843286574677046199e-07 +-1.738833342797969636e-06 +-1.004290676010142285e-06 +6.744290190047559574e-07 +1.741608192516847482e-06 +1.280938630120717270e-06 +-5.089962051112259242e-07 +-1.734595962635921047e-06 +-1.400613937252086356e-06 +1.648187793660271902e-07 +1.613277429219103755e-06 +1.590372361279220448e-06 +1.026314296979694244e-08 +-1.507199462265586231e-06 +-1.656498548720226997e-06 +-3.579462891905561392e-07 +1.261786933656091673e-06 +1.727036888705413617e-06 +6.926335290490646140e-07 +-1.127305826552776190e-06 +-1.734666481337623469e-06 +-8.519554876257128211e-07 +8.332817916134800692e-07 +1.741570517560293097e-06 +1.148022411434485064e-06 +-6.745069916869999551e-07 +-1.740893134080086125e-06 +-1.280863526478345686e-06 +3.385007361086511826e-07 +1.695925012644290348e-06 +1.504607413513912528e-06 +-1.648357971565737352e-07 +-1.613293906584121117e-06 +-1.590371049557741721e-06 +-1.850698468576710981e-07 +1.388584273480631005e-06 +1.701619351222137502e-06 +5.275664409881972963e-07 +-1.261890835872668019e-06 +-1.726952582184757882e-06 +-6.925939866257344096e-07 +9.843387710206114620e-07 +1.741827775922222212e-06 +1.004355416021173411e-06 +-8.332887313101556406e-07 +-1.741493618313671237e-06 +-1.147954127011784552e-06 +5.089536504049958006e-07 +1.734405951374010952e-06 +1.400665060755398470e-06 +-1.648147270693007677e-07 +-1.696026167294465809e-06 +-1.504516816255540277e-06 +-1.028570061164219897e-08 +1.506966518059593660e-06 +1.656403618202785754e-06 +3.579714705349063427e-07 +-1.388536818637949710e-06 +-1.701761632405453606e-06 +-5.275227261590045916e-07 +1.127176760524699177e-06 +1.738899137111849145e-06 +8.520110766617004472e-07 +-9.843379307777278582e-07 +-1.741806689421038490e-06 +-1.004295364866518706e-06 +6.743993238623917024e-07 +1.741251734869187301e-06 +1.280939977094745456e-06 +-3.385363576962681561e-07 +-1.734477658715999417e-06 +-1.400598327554048858e-06 +1.648342583341889763e-07 +1.613392098859980280e-06 +1.590351309578219981e-06 +1.850793076461093879e-07 +-1.506990158082079201e-06 +-1.656348914111078322e-06 +-3.579349890952416767e-07 +1.261748446896396022e-06 +1.726954622768805722e-06 +6.926407633502231383e-07 +-9.843909964545581157e-07 +-1.737771171945459064e-06 +-8.519585978329506073e-07 +8.332596957649066877e-07 +1.741883632174271865e-06 +1.148021789733716254e-06 +-5.089646463921921279e-07 +-1.741108470592745993e-06 +-1.280864836161371862e-06 +3.385379663369058124e-07 +1.696309625747959570e-06 +1.504609226014816693e-06 +1.029970426659359921e-08 +-1.613350527324997949e-06 +-1.590302724258083714e-06 +-1.850619383300565058e-07 +1.388450251314567264e-06 +1.701199448021600421e-06 +5.275580778973880034e-07 +-1.127288025163849908e-06 +-1.726563348516592840e-06 +-6.925950592092194851e-07 +9.843972034917504489e-07 +1.741986631626964369e-06 +1.004347296229844827e-06 +-6.744476113853419399e-07 +-1.741626188449242454e-06 +-1.147951644189643252e-06 +5.089802152540511556e-07 +1.734464186140258252e-06 +1.400680181652775881e-06 +-1.648386383732958587e-07 +-1.613458033031303169e-06 +-1.504577710435353542e-06 +-1.028608283527124951e-08 +1.506928969757272522e-06 +1.656256499901239233e-06 +3.579525644479939614e-07 +-1.261872342220463811e-06 +-1.701610355631484363e-06 +-5.275271344634275999e-07 +1.127198902199723192e-06 +1.738062302039671397e-06 +8.520017892937290931e-07 +-8.333539228821495243e-07 +-1.739065206108945372e-06 +-1.004291260290182602e-06 +6.744101958019806226e-07 +1.740803442299043410e-06 +1.280937036840117323e-06 +-3.385636630176459017e-07 +-1.696207774224466475e-06 +-1.400602580196387577e-06 +1.648167040171728496e-07 +1.613295641558825584e-06 +1.590472857050408798e-06 +1.850645034035481419e-07 +-1.388546378061128693e-06 +-1.656265722646537589e-06 +-3.579423946587949524e-07 +1.261757825222560384e-06 +1.727582033192779542e-06 +6.926310400115253552e-07 +-9.844404762875896280e-07 +-1.740739744440883911e-06 +-8.519585516470582605e-07 +8.332593986167771238e-07 +1.742379863835736858e-06 +1.148012931887122658e-06 +-5.090402292842649397e-07 +-1.734407858537690999e-06 +-1.280868972595230153e-06 +3.384999746174194464e-07 +1.696185029077676697e-06 +1.504623543518964511e-06 +1.028223359323060603e-08 +-1.507075600978145329e-06 +-1.590475652061203466e-06 +-1.850639222710864662e-07 +1.388557421002558836e-06 +1.701684251364760647e-06 +5.275512265607479172e-07 +-1.127286411336689808e-06 +-1.737395596505640334e-06 +-6.925970737149970458e-07 +9.843206542173732238e-07 +1.741832499020683313e-06 +1.004348077440545941e-06 +-6.744556533661682400e-07 +-1.740774003594915025e-06 +-1.147947685019042703e-06 +5.089704719231065102e-07 +1.734460067992656717e-06 +1.400663872155841370e-06 +-1.648378112527584822e-07 +-1.613128151027974754e-06 +-1.590478454220823225e-06 +-1.027735954261626783e-08 +1.506985638456615903e-06 +1.656094988251150500e-06 +3.579615998315412844e-07 +-1.261784008739208135e-06 +-1.726486447474969463e-06 +-5.275304255328125951e-07 +1.127163313841367346e-06 +1.738319230226277259e-06 +8.520030490968967593e-07 +-8.333047458639765553e-07 +-1.741716031385066482e-06 +-1.004287515401397678e-06 +6.744269820492903251e-07 +1.741063303667368066e-06 +1.280944197266247243e-06 +-3.385375740156370755e-07 +-1.696114681665394089e-06 +-1.504607320341262842e-06 +1.648247549435342491e-07 +1.613279411811395556e-06 +1.590272262240389021e-06 +1.850763445089354181e-07 +-1.388543432206800262e-06 +-1.701579535502788515e-06 +-3.579417799535259882e-07 +1.261825112859412937e-06 +1.726522186015714916e-06 +6.926298797425628815e-07 +-9.844451534597710166e-07 +-1.739154462062785886e-06 +-1.004287950956870439e-06 +8.332800073558788083e-07 +1.741705419103144528e-06 +1.148018190478056903e-06 +-5.089998106922283231e-07 +-1.734196579128229177e-06 +-1.400607469785205336e-06 +3.385147902400103756e-07 +1.696188697615942383e-06 +1.504598469109849392e-06 +1.028602692912153095e-08 +-1.507149469115720502e-06 +-1.656302740691107855e-06 +-1.850615676620913606e-07 +1.388597063993844476e-06 +1.701598914005255437e-06 +5.275488683394952047e-07 +-1.127246230087635691e-06 +-1.735630160281970369e-06 +-8.519545802429056960e-07 +9.843770273985061775e-07 +1.738994245936507767e-06 +1.004344838744613408e-06 +-6.744224451044774395e-07 +-1.741648171681971972e-06 +-1.280885326922608207e-06 +5.089825455281064240e-07 +1.734625715760534243e-06 +1.400662582495071480e-06 +-1.648427259867010545e-07 +-1.613604219551383351e-06 +-1.590309690913316708e-06 +-1.850585260142685255e-07 +1.507093017892425353e-06 +1.656433997566742238e-06 +3.579501898796839463e-07 +-1.261749084508135586e-06 +-1.724793762976749918e-06 +-6.925917484604561872e-07 +1.127192347382897711e-06 +1.734482175110829926e-06 +8.520028454191943851e-07 +-8.332712925714974420e-07 +-1.741297315315238620e-06 +-1.147961367548260921e-06 +6.744510731501280489e-07 +1.741062871064385555e-06 +1.280935962263143594e-06 +-3.385459381808079238e-07 +-1.695877980024644523e-06 +-1.504538951390613589e-06 +-1.027454453753881087e-08 +1.613311494828473454e-06 +1.590490523396869758e-06 +1.850695136122860865e-07 +-1.388564292089949174e-06 +-1.701054595687863222e-06 +-5.275258794933226751e-07 +1.261802895159962122e-06 +1.725896452052280683e-06 +6.926385417940661657e-07 +-9.843876064109259510e-07 +-1.742343816791561549e-06 +-1.004292673205916525e-06 +6.744367176308645790e-07 +1.741774388432576809e-06 +1.148021657984709214e-06 +-5.090048672716291640e-07 +-1.734424760616408426e-06 +-1.400604165194647486e-06 +1.648261974188203075e-07 +1.696104137809677338e-06 +1.504611812673658253e-06 +1.028752509823473208e-08 +-1.506835049173077311e-06 +-1.656367124297569766e-06 +-3.579465736156574359e-07 +1.388594163091088900e-06 +1.701600197062117496e-06 +5.275662193295627331e-07 +-1.127189509723428754e-06 +-1.739140251596924240e-06 +-8.519555146344949567e-07 +8.332803391730492215e-07 +1.739083119561703616e-06 +1.004355888144604765e-06 +-6.744274295560516419e-07 +-1.741157556063786170e-06 +-1.280874370092632454e-06 +3.384979849638089985e-07 +1.734512488043795267e-06 +1.400653876346155224e-06 +-1.648191437038570666e-07 +-1.613357878702286843e-06 +-1.590415680388212322e-06 +-1.850664008729975969e-07 +1.388616654257874299e-06 +1.656410100155959239e-06 +3.579718078982790880e-07 +-1.261742194831940182e-06 +-1.727519293799483035e-06 +-6.925931453648272117e-07 +9.842945020186080504e-07 +1.734544076054722863e-06 +8.520144031590257785e-07 +-8.332964747435799942e-07 +-1.742269274102159542e-06 +-1.147947761448823896e-06 +5.089688992861910061e-07 +1.741034279003410450e-06 +1.280937689492520769e-06 +-3.385423636830984105e-07 +-1.696046231791591712e-06 +-1.504618301607008335e-06 +-1.027719594533354700e-08 +1.506805351404585447e-06 +1.590436401080494455e-06 +1.851002823797434524e-07 +-1.388659116484511387e-06 +-1.701540621950765053e-06 +-5.275256108826528761e-07 +1.127166171203310547e-06 +1.725110778275020941e-06 +6.926393770930866481e-07 +-9.844323146218771484e-07 +-1.741784308782007532e-06 +-1.004286311019523964e-06 +6.744500208297343382e-07 +1.741938329515638949e-06 +1.148014876225550924e-06 +-5.089785309167015760e-07 +-1.734497322243950889e-06 +-1.400614927829939636e-06 +1.648214631468875859e-07 +1.613184576308203552e-06 +1.504606788480569668e-06 +1.027859207800901093e-08 +-1.507276578279591926e-06 +-1.656275217839760236e-06 +-3.579405943675265619e-07 +1.261816236083089866e-06 +1.700849706196831590e-06 +5.275493876965313214e-07 +-1.127198411639394720e-06 +-1.734249179605773024e-06 +-8.519561298198576426e-07 +8.332521341154560291e-07 +1.741764638932950718e-06 +1.004347431623458245e-06 +-6.744596310186115941e-07 +-1.741748314809797923e-06 +-1.280869111820689546e-06 +3.385171037921494635e-07 +1.695901592563524869e-06 +1.400678617835233387e-06 +-1.648565488028164046e-07 +-1.613561807138776761e-06 +-1.590478919667941100e-06 +-1.850638870732223280e-07 +1.388590675464703539e-06 +1.701433691138633940e-06 +3.579618944853186432e-07 +-1.261779512233267517e-06 +-1.726504418213852149e-06 +-6.925968883290548120e-07 +9.843697821249760282e-07 +1.741881945693986421e-06 +8.520030971635724024e-07 +-8.333062008962849171e-07 +-1.742331654135019675e-06 +-1.147952122313779147e-06 +5.089734434596512497e-07 +1.734662045847583257e-06 +1.280942301929910373e-06 +-3.384895853315081937e-07 +-1.696161302241622052e-06 +-1.504564104073877981e-06 +-1.027659593009093002e-08 +1.507009727967658865e-06 +1.656445198085565982e-06 +1.850641972503618116e-07 +-1.388693950052137218e-06 +-1.701040840053128679e-06 +-5.275290161123662693e-07 +1.127187285536431175e-06 +1.734723813921674311e-06 +6.926307761678843503e-07 +-9.843077859588004430e-07 +-1.739164557000898997e-06 +-1.004292273601492378e-06 +6.744365888437088794e-07 +1.742010130835613369e-06 +1.280938872272856700e-06 +-5.090256218436111423e-07 +-1.734564342862144349e-06 +-1.400616732860756689e-06 +1.648170136371230491e-07 +1.613363813253878648e-06 +1.590438272052297480e-06 +1.025305568602594536e-08 +-1.506926596493520195e-06 +-1.656431298117337317e-06 +-3.579410847930974386e-07 +1.261875034073571594e-06 +1.726397781239574585e-06 +5.275604617026915171e-07 +-1.127137339160950026e-06 +-1.739180713235708337e-06 +-8.519550466243470749e-07 +8.332820516221241366e-07 +1.741974380368915069e-06 +1.148021876526727446e-06 +-6.744862491138618203e-07 +-1.741113204422817864e-06 +-1.280873208255707068e-06 +3.385022989464865713e-07 +1.695972778829141366e-06 +1.504589322247046242e-06 +-1.648193240548547792e-07 +-1.613117730023353339e-06 +-1.590520135504460998e-06 +-1.850727975776422814e-07 +1.388654221902107873e-06 +1.701326020156905512e-06 +5.275682709626586915e-07 +-1.261947505715531078e-06 +-1.726078486393322259e-06 +-6.925948186026352393e-07 +9.843279579363112002e-07 +1.741660351150165014e-06 +1.004355483758935434e-06 +-8.333099001568666338e-07 +-1.742964785873333881e-06 +-1.147945751530698009e-06 +5.089534599001360521e-07 +1.734599858759064124e-06 +1.400664681976267818e-06 +-3.385016663087549197e-07 +-1.695870676087512766e-06 +-1.504554066993212823e-06 +-1.028948149217973521e-08 +1.507067109023507444e-06 +1.656464995976781995e-06 +3.579755870487922869e-07 +-1.388613154987747987e-06 +-1.701443479170704020e-06 +-5.275255413453798391e-07 +1.127166071302384452e-06 +1.734503200677021406e-06 +8.520142368580808288e-07 +-9.844001539690870857e-07 +-1.741656127705483338e-06 +-1.004294452983678689e-06 +6.744018012792296245e-07 +1.741914613915466185e-06 +1.280937446172547368e-06 +-3.385295895269946168e-07 +-1.734981101659022615e-06 +-1.400588052416376249e-06 +1.648316694002071529e-07 +1.613486565481482708e-06 +1.590483001996038449e-06 +1.850813389808183833e-07 +-1.507046180103455067e-06 +-1.656478180211654670e-06 +-3.579349198709450374e-07 +1.261721756539615536e-06 +1.726441261398884717e-06 +6.926417775169320062e-07 +-1.127277146577362529e-06 +-1.735176888438899801e-06 +-8.519584793928184132e-07 +8.332792681452140573e-07 +1.742157773790548184e-06 +1.148017886747695187e-06 +-5.090103794232628682e-07 +-1.741570079490782926e-06 +-1.280866889089690358e-06 +3.385394777859105416e-07 +1.696098376610258425e-06 +1.504587084259393158e-06 +1.028785662284046328e-08 +-1.613721909539123835e-06 +-1.590494503503302450e-06 +-1.850666926647154778e-07 +1.388542047507202778e-06 +1.701004739452484825e-06 +5.275550978710463273e-07 +-1.127242889265855080e-06 +-1.724557476696404319e-06 +-6.925939616939340237e-07 +9.844457602245243228e-07 +1.739765484603756400e-06 +1.004345694661083607e-06 +-6.744448456179310710e-07 +-1.741858256020505802e-06 +-1.147945817186612501e-06 +5.089811917854148751e-07 +1.734594781773205028e-06 +1.400685564424060619e-06 +-1.648459173018137831e-07 +-1.696301470419201054e-06 +-1.504565791851446476e-06 +-1.028660033549784524e-08 +1.507102629175416761e-06 +1.656165813660437616e-06 +3.579516650658177460e-07 +-1.261767226410950790e-06 +-1.701502509763890978e-06 +-5.275255364334288959e-07 +1.127258241735445696e-06 +1.738397037242816289e-06 +8.519992627385563252e-07 +-8.333155900301203234e-07 +-1.739221311409126995e-06 +-1.004289595998266989e-06 +6.743959648938684969e-07 +1.741615939271367849e-06 +1.280948672552883438e-06 +-3.385524990879865896e-07 +-1.696233800921703517e-06 +-1.400615320269782821e-06 +1.648185908126635101e-07 +1.613437983733279720e-06 +1.590376068386415433e-06 +1.850698379245635073e-07 +-1.388559203520277344e-06 +-1.656422452523023210e-06 +-3.579448456761001503e-07 +1.261793790873533917e-06 +1.724840191291488373e-06 +6.926291817612108503e-07 +-9.844155398481135369e-07 +-1.739034044670750390e-06 +-8.519585025669481418e-07 +8.332434238338562824e-07 +1.742439124717288334e-06 +1.148015029905874473e-06 +-5.090224365676433484e-07 +-1.734886862324567029e-06 +-1.280870799559112837e-06 +3.384997738350057584e-07 +1.695894495341171505e-06 +1.504610243716362610e-06 +1.028324945307817011e-08 +-1.507023215201327417e-06 +-1.590389675798358597e-06 +-1.850716221977319632e-07 +1.388596165212837243e-06 +1.701872268799283530e-06 +5.275521592616450156e-07 +-1.127213027036639802e-06 +-1.738868422425264749e-06 +-6.925967090994584708e-07 +9.843423265322836092e-07 +1.739658055559617132e-06 +1.004347820281614206e-06 +-6.744326650540783592e-07 +-1.741589486302994863e-06 +-1.147951091273796405e-06 +5.089665116460267154e-07 +1.734732998372292519e-06 +1.400646831852552351e-06 +-1.648303833899749357e-07 +-1.613310940965321688e-06 +-1.504616294783919238e-06 +-1.028165214295071204e-08 +1.506992341721636064e-06 +1.656530616962355248e-06 +3.579615973366801713e-07 +-1.261896119614707845e-06 +-1.725099656955503595e-06 +-5.275287952149862621e-07 +1.127169131548336438e-06 +1.736726332399427330e-06 +8.520031875083780306e-07 +-8.332788361584564856e-07 +-1.741739780205732736e-06 +-1.004287541940401894e-06 +6.744270469032312699e-07 +1.741887255151474092e-06 +1.280932833180469980e-06 +-3.385350963948087381e-07 +-1.695992647258216778e-06 +-1.504617839395090027e-06 +1.648217192121360978e-07 +1.613600533049282307e-06 +1.590344747451266808e-06 +1.850778758784153934e-07 +-1.388497459901435881e-06 +-1.701694326018869044e-06 +-3.579403082721944544e-07 +1.261788553692148542e-06 +1.725249752752758465e-06 +6.926305704182332962e-07 +-9.844267509989011877e-07 +-1.739205736135453973e-06 +-8.519543788221921723e-07 +8.332742280914557694e-07 +1.741817340405137131e-06 +1.148018306641354476e-06 +-5.089990489542986116e-07 +-1.734404740651791544e-06 +-1.400611542120221849e-06 +3.385030299852301277e-07 +1.696213744038989187e-06 +1.504595461529449285e-06 +1.028736312899410621e-08 +-1.507070054026904670e-06 +-1.656396408560201997e-06 +-1.850607566820000722e-07 +1.388605038353589448e-06 +1.701245856308883713e-06 +5.275505345612582854e-07 +-1.127252558694745794e-06 +-1.738070038224574988e-06 +-8.519549505936305591e-07 +9.843313317470496922e-07 +1.739225041207011783e-06 +1.004346551016468279e-06 +-6.744387716338965728e-07 +-1.740866185716938826e-06 +-1.280870248436358718e-06 +5.089666806394039543e-07 +1.734652445953194618e-06 +1.400686053030659741e-06 +-1.648523921474952950e-07 +-1.613681664011344824e-06 +-1.590316014061567975e-06 +-1.027950810415140155e-08 +1.506931218489344571e-06 +1.656346474519701670e-06 +3.579517781802106437e-07 +-1.261829655859476321e-06 +-1.727214126451728340e-06 +-6.925930708928146570e-07 +1.127135099675787796e-06 +1.739042613578682417e-06 +8.520049857849844323e-07 +-8.333550763716032250e-07 +-1.741223931084412381e-06 +-1.147960783092397317e-06 +6.744292292009189618e-07 +1.741077513878259454e-06 +1.280943363517365134e-06 +-3.385752748583262396e-07 +-1.696353572023426503e-06 +-1.504547365475786380e-06 +-1.027654991288985169e-08 +1.613222830497776154e-06 +1.590460028730889177e-06 +1.850729280492534397e-07 +-1.388806688619792916e-06 +-1.701267205312941893e-06 +-5.275260701054195476e-07 +1.261773888119183541e-06 +1.725800301839937553e-06 +6.926372703249363807e-07 +-9.844933858999205134e-07 +-1.742084440336276215e-06 +-1.004293243730852016e-06 +8.332837782442656153e-07 +1.741681429266680129e-06 +1.148018244108631514e-06 +-5.090140825880851954e-07 +-1.734648181794856410e-06 +-1.400615529873207839e-06 +1.648194043834948003e-07 +1.696085223384804340e-06 +1.504625704785217146e-06 +1.029153370613536142e-08 +-1.507118939351625321e-06 +-1.655993897924089542e-06 +-3.579469875844075512e-07 +1.388652301991538058e-06 +1.701558517642712207e-06 +5.275674631140396064e-07 +-1.127395666019526808e-06 +-1.734941680691810245e-06 +-8.519556369383823392e-07 +8.332644240114245081e-07 +1.739256246116809773e-06 +1.004353996353650715e-06 +-6.744530441635463773e-07 +-1.742187448380313789e-06 +-1.280886367995999871e-06 +3.384972043392220203e-07 +1.734506840627408643e-06 +1.400668240434210029e-06 +-1.648250127578891211e-07 +-1.613605146426856459e-06 +-1.590345985028447486e-06 +-1.850701461615776516e-07 +1.507153337590947545e-06 +1.656372251468303937e-06 +3.579702221657228762e-07 +-1.261874832669292944e-06 +-1.726027473690091609e-06 +-6.925920677498453038e-07 +9.843190773326566283e-07 +1.739058466695084452e-06 +8.520157362474218902e-07 +-8.333066511850163643e-07 +-1.742433685415770509e-06 +-1.147955603304021991e-06 +5.089720785437955275e-07 +1.741096844482296675e-06 +1.280939701917045734e-06 +-3.385226925826214097e-07 +-1.696133847163606545e-06 +-1.504574588735058261e-06 +-1.027584873402550068e-08 +1.506900793475349959e-06 +1.590413619220728538e-06 +1.850912530880386366e-07 +-1.388642336697858351e-06 +-1.701249147872202525e-06 +-5.275242307718781953e-07 +1.127191767414049902e-06 +1.726028621147986852e-06 +6.926392242527476744e-07 +-9.844647089534297558e-07 +-1.739939757177752584e-06 +-1.004288355418123485e-06 +6.744529207682478307e-07 +1.742380102408182492e-06 +1.148016912806621430e-06 +-5.089940474249223597e-07 +-1.734603309747968123e-06 +-1.400616570006886963e-06 +1.648218615872110605e-07 +1.613295571530132512e-06 +1.504603161806901259e-06 +1.029429429793979145e-08 +-1.506989540017813416e-06 +-1.656398114490014354e-06 +-3.579385752429690782e-07 +1.261835544753514274e-06 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 From bf1a96b92ead852b3eff41b750bb44ecf2afb1df Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Thu, 30 Jan 2025 06:13:15 +0000 Subject: [PATCH 39/48] workflow error? --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 64572664f..fa1df2f1f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,17 +37,17 @@ jobs: cd .. mkdir build_fmks_release cd build_fmks_release - cmake -DCMAKE_BUILD_TYPE=Release -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=FMKS -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON .-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off ../../../ + cmake -DCMAKE_BUILD_TYPE=Release -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=FMKS -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON -DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off ../../../ cmake --build . --parallel 8 cd .. mkdir build_minkowski_debug cd build_minkowski_debug - cmake -DCMAKE_BUILD_TYPE=Debug -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=Minkowski -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON .-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off ../../../ + cmake -DCMAKE_BUILD_TYPE=Debug -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=Minkowski -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON -DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off ../../../ cmake --build . --parallel 8 cd .. mkdir build_fmks_debug cd build_fmks_debug - cmake -DCMAKE_BUILD_TYPE=Debug -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=FMKS -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON .-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off ../../../ + cmake -DCMAKE_BUILD_TYPE=Debug -DPHOEBUS_ENABLE_UNIT_TESTS=ON -DMAX_NUMBER_CONSERVED_VARS=10 -DPHOEBUS_GEOMETRY=FMKS -DPHOEBUS_CACHE_GEOMETRY=ON -DPARTHENON_DISABLE_HDF5_COMPRESSION=ON -DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off ../../../ cmake --build . --parallel 8 cd .. # is there a run directory that shouldn't be here? From 21ed0ed9a9ad3202106aa5786672bdaf704ff85e Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Fri, 31 Jan 2025 18:31:15 +0000 Subject: [PATCH 40/48] make cmake_extra_args a list instead of string --- tst/regression/bondi.py | 2 +- tst/regression/friedmann.py | 2 +- tst/regression/homogeneous_sphere.py | 2 +- tst/regression/linear_modes.py | 2 +- tst/regression/mocmc_diffusion.py | 2 +- tst/regression/mocmc_equilibration.py | 3 +-- tst/regression/radiation_diffusion.py | 2 +- tst/regression/radiation_equilibration.py | 2 +- tst/regression/regression_test.py | 9 ++++----- tst/regression/thincooling.py | 2 +- tst/regression/thincooling_coolingfunction.py | 2 +- 11 files changed, 14 insertions(+), 16 deletions(-) diff --git a/tst/regression/bondi.py b/tst/regression/bondi.py index 9d86f6e83..8a07083fc 100755 --- a/tst/regression/bondi.py +++ b/tst/regression/bondi.py @@ -46,7 +46,7 @@ input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, - cmake_extra_args="", + cmake_extra_args=[], geometry="FMKS", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/friedmann.py b/tst/regression/friedmann.py index 0f639826c..4be34c50a 100755 --- a/tst/regression/friedmann.py +++ b/tst/regression/friedmann.py @@ -39,7 +39,7 @@ variables=["p.density", "p.energy"], input_file=args.input, executable=args.executable, - cmake_extra_args="", + cmake_extra_args=[], geometry="FLRW", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/homogeneous_sphere.py b/tst/regression/homogeneous_sphere.py index 30633ea5b..c6fe5f4a2 100755 --- a/tst/regression/homogeneous_sphere.py +++ b/tst/regression/homogeneous_sphere.py @@ -46,7 +46,7 @@ modified_inputs["radiation_advection/vx"] = 0.0 modified_inputs["radiation_advection/radius"] = 1.0 -cmake_extra_args = "-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off" +cmake_extra_args = ["-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off", "-DPHOEBUS_DO_NU_HEAVY=Off"] code = rt.gold_comparison( variables=["r.p.J", "r.p.H"], diff --git a/tst/regression/linear_modes.py b/tst/regression/linear_modes.py index 350ff2cd3..ca945d311 100755 --- a/tst/regression/linear_modes.py +++ b/tst/regression/linear_modes.py @@ -46,7 +46,7 @@ input_file=args.input, modified_inputs=modified_inputs, executable=args.executable, - cmake_extra_args="", + cmake_extra_args=[], geometry="Minkowski", use_gpu=args.use_gpu, use_mpiexec=args.use_mpiexec, diff --git a/tst/regression/mocmc_diffusion.py b/tst/regression/mocmc_diffusion.py index 6cbc85460..b14b12385 100755 --- a/tst/regression/mocmc_diffusion.py +++ b/tst/regression/mocmc_diffusion.py @@ -56,7 +56,7 @@ modified_inputs["parthenon/mesh/nx1"] = 64 modified_inputs["parthenon/meshblock/nx1"] = 64 -cmake_extra_args = "-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off" +cmake_extra_args = ["-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off", "-DPHOEBUS_DO_NU_HEAVY=Off"] code = rt.gold_comparison( variables=["r.p.J", "r.p.H"], diff --git a/tst/regression/mocmc_equilibration.py b/tst/regression/mocmc_equilibration.py index 8ea4bbf54..6a50a1574 100755 --- a/tst/regression/mocmc_equilibration.py +++ b/tst/regression/mocmc_equilibration.py @@ -47,8 +47,7 @@ modified_inputs["radiation/mocmc/nsamp_per_zone"] = 32 modified_inputs["radiation_equilibration/J"] = 0.2 - -cmake_extra_args = "-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off" +cmake_extra_args = ["-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off", "-DPHOEBUS_DO_NU_HEAVY=Off"] code = rt.gold_comparison( variables=["r.p.J", "r.p.H"], diff --git a/tst/regression/radiation_diffusion.py b/tst/regression/radiation_diffusion.py index f8a296630..ce995b35a 100755 --- a/tst/regression/radiation_diffusion.py +++ b/tst/regression/radiation_diffusion.py @@ -50,7 +50,7 @@ modified_inputs["radiation_advection/width"] = 0.0333 modified_inputs["radiation_advection/kappas_init"] = 1.0e3 -cmake_extra_args = "-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off" +cmake_extra_args = ["-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off", "-DPHOEBUS_DO_NU_HEAVY=Off"] code = rt.gold_comparison( variables=["r.p.J", "r.p.H"], diff --git a/tst/regression/radiation_equilibration.py b/tst/regression/radiation_equilibration.py index 4dc511599..4b2582791 100755 --- a/tst/regression/radiation_equilibration.py +++ b/tst/regression/radiation_equilibration.py @@ -43,7 +43,7 @@ modified_inputs["opacity/gray_kappa"] = 0.1 modified_inputs["radiation_equilibration/J"] = 0.2 -cmake_extra_args = "-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off" +cmake_extra_args = ["-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off", "-DPHOEBUS_DO_NU_HEAVY=Off"] code = rt.gold_comparison( variables=["r.p.J", "r.p.H"], diff --git a/tst/regression/regression_test.py b/tst/regression/regression_test.py index 7a26a1770..f04f2577e 100644 --- a/tst/regression/regression_test.py +++ b/tst/regression/regression_test.py @@ -148,7 +148,7 @@ def modify_input(dict_key, value, input_file): # -- Configure and build phoebus with problem-specific options -def build_code(geometry, use_gpu=False, build_type="Release", cmake_extra_args=""): +def build_code(geometry, use_gpu=False, build_type="Release", cmake_extra_args=[]): if os.path.isdir(BUILD_DIR): print( f'BUILD_DIR "{BUILD_DIR}" already exists! Clean up before calling a regression test script!' @@ -166,9 +166,8 @@ def build_code(geometry, use_gpu=False, build_type="Release", cmake_extra_args=" else: print(f'Build type "{build_type}" not known!') sys.exit(os.EX_SOFTWARE) - #configure_options.append(cmake_extra_args) - for arg in cmake_extra_args.split(): - configure_options.append(arg) + + configure_options += cmake_extra_args configure_options.append("-DPHOEBUS_ENABLE_UNIT_TESTS=OFF") configure_options.append("-DMAX_NUMBER_CONSERVED_VARS=10") configure_options.append("-DPHOEBUS_CACHE_GEOMETRY=ON") @@ -242,7 +241,7 @@ def gold_comparison( input_file, modified_inputs={}, executable=None, - cmake_extra_args="", + cmake_extra_args=[], geometry="Minkowski", use_gpu=False, use_mpiexec=False, diff --git a/tst/regression/thincooling.py b/tst/regression/thincooling.py index 96177dbe9..1379723bb 100755 --- a/tst/regression/thincooling.py +++ b/tst/regression/thincooling.py @@ -38,7 +38,7 @@ modified_inputs = {} modified_inputs["radiation/method"] = "monte_carlo" -cmake_extra_args = "-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off" +cmake_extra_args = ["-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off", "-DPHOEBUS_DO_NU_HEAVY=Off"] code = rt.gold_comparison( variables=["p.density", "p.energy"], diff --git a/tst/regression/thincooling_coolingfunction.py b/tst/regression/thincooling_coolingfunction.py index bf6f98644..f9d7195f5 100755 --- a/tst/regression/thincooling_coolingfunction.py +++ b/tst/regression/thincooling_coolingfunction.py @@ -38,7 +38,7 @@ modified_inputs = {} modified_inputs["radiation/method"] = "cooling_function" -cmake_extra_args = "-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off -DPHOEBUS_DO_NU_HEAVY=Off" +cmake_extra_args = ["-DPHOEBUS_DO_NU_ELECTRON_ANTI=Off", "-DPHOEBUS_DO_NU_HEAVY=Off"] code = rt.gold_comparison( variables=["p.density", "p.energy"], From f9e65c3c0b0ac079aa5a47c2215fe556dda6dea9 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Fri, 31 Jan 2025 20:10:05 +0000 Subject: [PATCH 41/48] typo --- inputs/homogeneous_sphere.pin | 1 - 1 file changed, 1 deletion(-) diff --git a/inputs/homogeneous_sphere.pin b/inputs/homogeneous_sphere.pin index 38f0e5c91..a865b078b 100644 --- a/inputs/homogeneous_sphere.pin +++ b/inputs/homogeneous_sphere.pin @@ -98,7 +98,6 @@ nu_min = 1.e15 nu_max = 1.e22 nu_bins = 200 absorption = true -:x B_fake = 1.0 use_B_fake = true From 97add4136259f2d798d1fdee5a8b6c81dccde615 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Fri, 31 Jan 2025 20:20:47 +0000 Subject: [PATCH 42/48] forgotten comments --- src/phoebus_boundaries/phoebus_boundaries.cpp | 5 ----- src/phoebus_driver.cpp | 1 - 2 files changed, 6 deletions(-) diff --git a/src/phoebus_boundaries/phoebus_boundaries.cpp b/src/phoebus_boundaries/phoebus_boundaries.cpp index 3255eb81d..b7dbade6b 100644 --- a/src/phoebus_boundaries/phoebus_boundaries.cpp +++ b/src/phoebus_boundaries/phoebus_boundaries.cpp @@ -494,11 +494,6 @@ void ProcessBoundaryConditions(parthenon::ParthenonManager &pman) { auto &face = inner_outer[outer]; const std::string name = face + "x" + std::to_string(d) + "_bc"; const std::string parth_bc = pman.pinput->GetString("parthenon/mesh", name); - // TODO: proper check here?? - // PARTHENON_REQUIRE(parth_bc == "user" || parth_bc == "periodic", - // "Only \"user\" and \"periodic\" allowed for - // parthenon/mesh/" + - // name); const std::string bc = pman.pinput->GetOrAddString("phoebus", name, "gr_outflow"); if (bc == "reflect") { diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index e53b479bb..631224a59 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -560,7 +560,6 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { } // Fix up flux failures - // TODO (DO NOT MERGE) is this loop right // TODO (BLB) once the fixups can work with MeshData, // -- then stitch this section back together TaskRegion &async_region_2 = tc.AddRegion(num_independent_task_lists); From d421164f3807428c54753f22e71e8f1c46bc3439 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Fri, 31 Jan 2025 20:45:57 +0000 Subject: [PATCH 43/48] invert and correct constexpr logic for radiation/geodesics:GetKSource --- src/radiation/geodesics.hpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/radiation/geodesics.hpp b/src/radiation/geodesics.hpp index 7c9fae78c..6996e1931 100644 --- a/src/radiation/geodesics.hpp +++ b/src/radiation/geodesics.hpp @@ -39,7 +39,12 @@ template KOKKOS_INLINE_FUNCTION void GetKSource(Real &X0, Real &X1, Real &X2, Real &X3, Real &Kcov0, Real &Kcov1, Real &Kcov2, Real &Kcov3, Real &Kcon0, Geom &geom, Real source[4]) { - if constexpr (std::is_same::value) { + // TODO(BLB): include options for more geometries to exploit spacetime symmetries + if constexpr (std::is_same::value) { // flat space + SPACETIMELOOP(mu) { source[mu] = 0.0; } + } else { // generic + // TODO(BLB): hook into metric machinery instead of doing it here Real Kcov[NDFULL] = {Kcov0, Kcov1, Kcov2, Kcov3}; Real Xm[NDFULL], Xp[NDFULL]; Real Gconm[NDFULL][NDFULL], Gconp[NDFULL][NDFULL]; @@ -68,8 +73,6 @@ KOKKOS_INLINE_FUNCTION void GetKSource(Real &X0, Real &X1, Real &X2, Real &X3, source[mu] *= -1.0 / (2.0 * Kcon0); } - } else { - SPACETIMELOOP(mu) { source[mu] = 0.; } } } From f04c7442e5ac9c7603551503905932e98c13c71a Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Fri, 31 Jan 2025 16:45:00 -0500 Subject: [PATCH 44/48] cleanup of boundar conditions registration logic --- inputs/blandford_mckee.pin | 8 +- inputs/blast_wave.pin | 6 +- inputs/bondi.pin | 8 +- inputs/homogeneous_sphere.pin | 4 +- inputs/homologous.pin | 6 +- inputs/leptoneq.pin | 12 +++ inputs/mhd_shocktube.pin | 4 +- inputs/progenitor.pin | 6 +- inputs/radiation_equilibration.pin | 4 +- inputs/rhs_tester.pin | 6 +- inputs/rotor.pin | 4 +- inputs/shocktube.pin | 2 - inputs/shocktube_nonrel.pin | 4 +- inputs/shocktube_stellarcollapse.pin | 2 - .../shocktube_stellarcollapse_faketable.pin | 2 - inputs/spherical_shock_tube.pin | 4 - inputs/torus.pin | 15 ++- inputs/torus_cutout.pin | 15 ++- inputs/torus_rad.pin | 15 ++- inputs/tov-static.pin | 4 +- inputs/tov.pin | 6 +- src/phoebus_boundaries/phoebus_boundaries.cpp | 93 ++++++++----------- 22 files changed, 97 insertions(+), 133 deletions(-) diff --git a/inputs/blandford_mckee.pin b/inputs/blandford_mckee.pin index 4c431ba32..bd8c15ed5 100644 --- a/inputs/blandford_mckee.pin +++ b/inputs/blandford_mckee.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -15,12 +15,6 @@ problem = blandford_mckee -ix1_bc = reflect -ox1_bc = gr_outflow -ix2_bc = reflect -ox2_bc = reflect -ix3_bc = periodic -ox3_bc = periodic problem_id = blast # problem ID: basename of output filenames diff --git a/inputs/blast_wave.pin b/inputs/blast_wave.pin index 8ca004559..6f38722ea 100644 --- a/inputs/blast_wave.pin +++ b/inputs/blast_wave.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -18,10 +18,6 @@ problem = sedov -ix1_bc = reflect -ox1_bc = gr_outflow -ix2_bc = reflect -ox2_bc = reflect problem_id = sedov # problem ID: basename of output filenames diff --git a/inputs/bondi.pin b/inputs/bondi.pin index d19e098e6..a15ce64a1 100644 --- a/inputs/bondi.pin +++ b/inputs/bondi.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -12,12 +12,6 @@ problem = bondi -ix1_bc = gr_outflow -ox1_bc = gr_outflow -ix2_bc = reflect -ox2_bc = reflect -ix3_bc = periodic -ox3_bc = periodic problem_id = bondi # problem ID: basename of output filenames diff --git a/inputs/homogeneous_sphere.pin b/inputs/homogeneous_sphere.pin index a865b078b..2b406d03f 100644 --- a/inputs/homogeneous_sphere.pin +++ b/inputs/homogeneous_sphere.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -12,8 +12,6 @@ problem = homogeneous_sphere -ix1_bc = reflect -ox1_bc = gr_outflow problem_id = hom_sph # problem ID: basename of output filenames diff --git a/inputs/homologous.pin b/inputs/homologous.pin index 175244d81..74b11e05b 100644 --- a/inputs/homologous.pin +++ b/inputs/homologous.pin @@ -1,4 +1,4 @@ -# © 2021-2022. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -14,10 +14,6 @@ problem = homologous -ix1_bc = reflect -ox1_bc = reflect -ix2_bc = reflect -ox2_bc = reflect problem_id = homologous # problem ID: basename of output filenames diff --git a/inputs/leptoneq.pin b/inputs/leptoneq.pin index 303f85dca..2a6cc09f5 100644 --- a/inputs/leptoneq.pin +++ b/inputs/leptoneq.pin @@ -1,3 +1,15 @@ +# © 2021-2025. Triad National Security, LLC. All rights reserved. This +# program was produced under U.S. Government contract 89233218CNA000001 +# for Los Alamos National Laboratory (LANL), which is operated by Triad +# National Security, LLC for the U.S. Department of Energy/National +# Nuclear Security Administration. All rights in the program are +# reserved by Triad National Security, LLC, and the U.S. Department of +# Energy/National Nuclear Security Administration. The Government is +# granted for itself and others acting on its behalf a nonexclusive, +# paid-up, irrevocable worldwide license in this material to reproduce, +# prepare derivative works, distribute copies to the public, perform +# publicly and display publicly, and to permit others to do so. + problem = leptoneq diff --git a/inputs/mhd_shocktube.pin b/inputs/mhd_shocktube.pin index e6599a5ee..aea0f505e 100644 --- a/inputs/mhd_shocktube.pin +++ b/inputs/mhd_shocktube.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -12,8 +12,6 @@ problem = shock_tube -ix1_bc = gr_outflow -ox1_bc = gr_outflow problem_id = sod # problem ID: basename of output filenames diff --git a/inputs/progenitor.pin b/inputs/progenitor.pin index 67c4d8930..9af901330 100644 --- a/inputs/progenitor.pin +++ b/inputs/progenitor.pin @@ -1,4 +1,4 @@ -# © 2021-2022. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -14,10 +14,6 @@ problem = homologous -ix1_bc = reflect -ox1_bc = reflect -ix2_bc = reflect -ox2_bc = reflect problem_id = homologous # problem ID: basename of output filenames diff --git a/inputs/radiation_equilibration.pin b/inputs/radiation_equilibration.pin index b6b2b9260..e8f9d5bc5 100644 --- a/inputs/radiation_equilibration.pin +++ b/inputs/radiation_equilibration.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -12,8 +12,6 @@ problem = radiation_equilibration -bc_ix1 = outflow -bc_ox1 = outflow problem_id = rad_eql # problem ID: basename of output filenames diff --git a/inputs/rhs_tester.pin b/inputs/rhs_tester.pin index fa237fb91..e72e1ab23 100644 --- a/inputs/rhs_tester.pin +++ b/inputs/rhs_tester.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -14,10 +14,6 @@ problem = rhs_tester -ix1_bc = reflect -ox1_bc = gr_outflow -ix2_bc = reflect -ox2_bc = reflect problem_id = test # problem ID: basename of output filenames diff --git a/inputs/rotor.pin b/inputs/rotor.pin index ef6aa1e27..186cb87c0 100644 --- a/inputs/rotor.pin +++ b/inputs/rotor.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -12,8 +12,6 @@ problem = rotor -ix1_bc = outflow -ox1_bc = outflow problem_id = mhd_rotor # problem ID: basename of output filenames diff --git a/inputs/shocktube.pin b/inputs/shocktube.pin index ec775c20e..ca8d9dfec 100644 --- a/inputs/shocktube.pin +++ b/inputs/shocktube.pin @@ -12,8 +12,6 @@ problem = shock_tube -ix1_bc = gr_outflow -ox1_bc = gr_outflow problem_id = sod # problem ID: basename of output filenames diff --git a/inputs/shocktube_nonrel.pin b/inputs/shocktube_nonrel.pin index b0d6cd4e6..fab1b67c3 100644 --- a/inputs/shocktube_nonrel.pin +++ b/inputs/shocktube_nonrel.pin @@ -1,4 +1,4 @@ -# © 2021. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -12,8 +12,6 @@ problem = shock_tube -ix1_bc = gr_outflow -ox1_bc = gr_outflow problem_id = sod # problem ID: basename of output filenames diff --git a/inputs/shocktube_stellarcollapse.pin b/inputs/shocktube_stellarcollapse.pin index f6cbefc97..4dec468a2 100644 --- a/inputs/shocktube_stellarcollapse.pin +++ b/inputs/shocktube_stellarcollapse.pin @@ -15,8 +15,6 @@ problem = shock_tube -ix1_bc = gr_outflow -ox1_bc = gr_outflow problem_id = sod # problem ID: basename of output filenames diff --git a/inputs/shocktube_stellarcollapse_faketable.pin b/inputs/shocktube_stellarcollapse_faketable.pin index 8ab22a110..757b3f9ed 100644 --- a/inputs/shocktube_stellarcollapse_faketable.pin +++ b/inputs/shocktube_stellarcollapse_faketable.pin @@ -19,8 +19,6 @@ problem = shock_tube -ix1_bc = gr_outflow -ox1_bc = gr_outflow problem_id = sod # problem ID: basename of output filenames diff --git a/inputs/spherical_shock_tube.pin b/inputs/spherical_shock_tube.pin index a5d40eb9a..9070fd4dd 100644 --- a/inputs/spherical_shock_tube.pin +++ b/inputs/spherical_shock_tube.pin @@ -24,10 +24,6 @@ problem = shock_tube -ix1_bc = reflect -ox1_bc = gr_outflow -ix2_bc = reflect -ox2_bc = reflect problem_id = sedov # problem ID: basename of output filenames diff --git a/inputs/torus.pin b/inputs/torus.pin index 87a040d3b..eff3fc183 100644 --- a/inputs/torus.pin +++ b/inputs/torus.pin @@ -1,10 +1,17 @@ +# © 2021-2025. Triad National Security, LLC. All rights reserved. This +# program was produced under U.S. Government contract 89233218CNA000001 +# for Los Alamos National Laboratory (LANL), which is operated by Triad +# National Security, LLC for the U.S. Department of Energy/National +# Nuclear Security Administration. All rights in the program are +# reserved by Triad National Security, LLC, and the U.S. Department of +# Energy/National Nuclear Security Administration. The Government is +# granted for itself and others acting on its behalf a nonexclusive, +# paid-up, irrevocable worldwide license in this material to reproduce, +# prepare derivative works, distribute copies to the public, perform +# publicly and display publicly, and to permit others to do so. problem = torus -ix1_bc = gr_outflow -ox1_bc = gr_outflow -ix2_bc = gr_outflow -ox2_bc = gr_outflow bc_vars = primitive diff --git a/inputs/torus_cutout.pin b/inputs/torus_cutout.pin index 892f85e4e..0957c9346 100644 --- a/inputs/torus_cutout.pin +++ b/inputs/torus_cutout.pin @@ -1,10 +1,17 @@ +# © 2021-2025. Triad National Security, LLC. All rights reserved. This +# program was produced under U.S. Government contract 89233218CNA000001 +# for Los Alamos National Laboratory (LANL), which is operated by Triad +# National Security, LLC for the U.S. Department of Energy/National +# Nuclear Security Administration. All rights in the program are +# reserved by Triad National Security, LLC, and the U.S. Department of +# Energy/National Nuclear Security Administration. The Government is +# granted for itself and others acting on its behalf a nonexclusive, +# paid-up, irrevocable worldwide license in this material to reproduce, +# prepare derivative works, distribute copies to the public, perform +# publicly and display publicly, and to permit others to do so. problem = torus -ix1_bc = gr_outflow -ox1_bc = gr_outflow -ix2_bc = gr_outflow -ox2_bc = gr_outflow problem_id = torus # problem ID: basename of output filenames diff --git a/inputs/torus_rad.pin b/inputs/torus_rad.pin index efa327355..0d618646a 100644 --- a/inputs/torus_rad.pin +++ b/inputs/torus_rad.pin @@ -1,10 +1,17 @@ +# © 2021-2025. Triad National Security, LLC. All rights reserved. This +# program was produced under U.S. Government contract 89233218CNA000001 +# for Los Alamos National Laboratory (LANL), which is operated by Triad +# National Security, LLC for the U.S. Department of Energy/National +# Nuclear Security Administration. All rights in the program are +# reserved by Triad National Security, LLC, and the U.S. Department of +# Energy/National Nuclear Security Administration. The Government is +# granted for itself and others acting on its behalf a nonexclusive, +# paid-up, irrevocable worldwide license in this material to reproduce, +# prepare derivative works, distribute copies to the public, perform +# publicly and display publicly, and to permit others to do so. problem = torus -ix1_bc = gr_outflow -ox1_bc = gr_outflow -ix2_bc = gr_outflow -ox2_bc = gr_outflow bc_vars = primitive diff --git a/inputs/tov-static.pin b/inputs/tov-static.pin index 7938fd1b0..7e4a11896 100644 --- a/inputs/tov-static.pin +++ b/inputs/tov-static.pin @@ -1,4 +1,4 @@ -# © 2021-2022. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -18,8 +18,6 @@ problem = tov -ix1_bc = reflect -ox1_bc = gr_outflow problem_id = tov # problem ID: basename of output filenames diff --git a/inputs/tov.pin b/inputs/tov.pin index 56686d663..54dc69803 100644 --- a/inputs/tov.pin +++ b/inputs/tov.pin @@ -1,4 +1,4 @@ -# © 2021-2022. Triad National Security, LLC. All rights reserved. This +# © 2021-2025. Triad National Security, LLC. All rights reserved. This # program was produced under U.S. Government contract 89233218CNA000001 # for Los Alamos National Laboratory (LANL), which is operated by Triad # National Security, LLC for the U.S. Department of Energy/National @@ -14,10 +14,6 @@ problem = tov -ix1_bc = reflect -ox1_bc = gr_outflow -ix2_bc = reflect -ox2_bc = reflect problem_id = tov # problem ID: basename of output filenames diff --git a/src/phoebus_boundaries/phoebus_boundaries.cpp b/src/phoebus_boundaries/phoebus_boundaries.cpp index b7dbade6b..0000d6e48 100644 --- a/src/phoebus_boundaries/phoebus_boundaries.cpp +++ b/src/phoebus_boundaries/phoebus_boundaries.cpp @@ -496,64 +496,49 @@ void ProcessBoundaryConditions(parthenon::ParthenonManager &pman) { const std::string parth_bc = pman.pinput->GetString("parthenon/mesh", name); const std::string bc = pman.pinput->GetOrAddString("phoebus", name, "gr_outflow"); - if (bc == "reflect") { - pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "reflect", - reflect[d - 1][outer]); - } else if (bc == "polar") { - PARTHENON_REQUIRE(d == 2, "Polar boundary conditions only supported in X2!"); + pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "reflect", + reflect[d - 1][outer]); + if (d == 2) { // polar BC only supported for X2 pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "polar", polar[d - 1][outer]); - } else if (bc == "gr_outflow") { - pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "gr_outflow", - outflow[d - 1][outer]); - if (d == 1) { - if (outer == 0) { - if (rad_method == "mocmc") { - pman.app_input->RegisterSwarmBoundaryCondition( - parthenon::BoundaryFace::inner_x1, "swarm_no_bc", - &Boundaries::SwarmNoWorkBC); - } else { - pman.app_input->RegisterSwarmBoundaryCondition( - parthenon::BoundaryFace::inner_x1, "swarm_outflow", - &parthenon::BoundaryFunction::SwarmOutflowInnerX1); - } - } else if (outer == 1) { - if (rad_method == "mocmc") { - pman.app_input->RegisterSwarmBoundaryCondition( - parthenon::BoundaryFace::outer_x1, "swarm_no_bc", - &Boundaries::SwarmNoWorkBC); - } else { - pman.app_input->RegisterSwarmBoundaryCondition( - parthenon::BoundaryFace::outer_x1, "swarm_outflow", - &parthenon::BoundaryFunction::SwarmOutflowOuterX1); - } - } + } + pman.app_input->RegisterBoundaryCondition(loc[d - 1][outer], "gr_outflow", + outflow[d - 1][outer]); + if (d == 1) { + if (outer == 0) { + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::inner_x1, "swarm_no_bc", + &Boundaries::SwarmNoWorkBC); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::inner_x1, "swarm_outflow", + &parthenon::BoundaryFunction::SwarmOutflowInnerX1); + } else if (outer == 1) { + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::outer_x1, "swarm_no_bc", + &Boundaries::SwarmNoWorkBC); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::outer_x1, "swarm_outflow", + &parthenon::BoundaryFunction::SwarmOutflowOuterX1); } - if (d == 2) { - if (outer == 0) { - if (rad_method == "mocmc") { - pman.app_input->RegisterSwarmBoundaryCondition( - parthenon::BoundaryFace::inner_x2, "swarm_no_bc", - &Boundaries::SwarmNoWorkBC); - } else { - pman.app_input->RegisterSwarmBoundaryCondition( - parthenon::BoundaryFace::inner_x2, "swarm_outflow", - &parthenon::BoundaryFunction::SwarmOutflowInnerX1); - } - } else if (outer == 1) { - if (rad_method == "mocmc") { - pman.app_input->RegisterSwarmBoundaryCondition( - parthenon::BoundaryFace::outer_x2, "swarm_no_bc", - &Boundaries::SwarmNoWorkBC); - } else { - pman.app_input->RegisterSwarmBoundaryCondition( - parthenon::BoundaryFace::outer_x2, "swarm_outflow", - &parthenon::BoundaryFunction::SwarmOutflowOuterX2); - } - } + } + if (d == 2) { + if (outer == 0) { + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::inner_x2, "swarm_no_bc", + &Boundaries::SwarmNoWorkBC); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::inner_x2, "swarm_outflow", + &parthenon::BoundaryFunction::SwarmOutflowInnerX1); + } else if (outer == 1) { + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::outer_x2, "swarm_no_bc", + &Boundaries::SwarmNoWorkBC); + pman.app_input->RegisterSwarmBoundaryCondition( + parthenon::BoundaryFace::outer_x2, "swarm_outflow", + &parthenon::BoundaryFunction::SwarmOutflowOuterX2); } - } // periodic boundaries, which are handled by parthenon, so no need to set anything - } + } + } // periodic boundaries, which are handled by parthenon, so no need to set anything } } From b3d4ebff65953571c2d16135ee153750b3ae633e Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Fri, 31 Jan 2025 18:11:29 -0500 Subject: [PATCH 45/48] make temporary MeshDat level fixup Tasks that loop over blocks --- src/fixup/fixup.cpp | 7 +++++++ src/fixup/fixup_c2p.cpp | 8 ++++++++ src/fixup/fixup_radc2p.cpp | 7 +++++++ src/phoebus_driver.cpp | 37 ++++++++++++------------------------- 4 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/fixup/fixup.cpp b/src/fixup/fixup.cpp index ca95eebc9..0d46482ee 100644 --- a/src/fixup/fixup.cpp +++ b/src/fixup/fixup.cpp @@ -717,6 +717,13 @@ TaskStatus ApplyFloors(T *rc) { } template TaskStatus ApplyFloors>(MeshBlockData *rc); +template <> +TaskStatus ApplyFloors>(MeshData *rc) { + for (int b = 0; b < rc->NumBlocks(); b++) { + ApplyFloors(rc->GetBlockData(b).get()); + } + return TaskStatus::complete; +} TaskStatus FixFluxes(MeshBlockData *rc) { using parthenon::BoundaryFace; diff --git a/src/fixup/fixup_c2p.cpp b/src/fixup/fixup_c2p.cpp index 23ae5208b..525bb8b9f 100644 --- a/src/fixup/fixup_c2p.cpp +++ b/src/fixup/fixup_c2p.cpp @@ -383,5 +383,13 @@ TaskStatus ConservedToPrimitiveFixup(T *rc) { template TaskStatus ConservedToPrimitiveFixup>(MeshBlockData *rc); +// template TaskStatus +template <> +TaskStatus ConservedToPrimitiveFixup>(MeshData *rc) { + for (int b = 0; b < rc->NumBlocks(); b++) { + ConservedToPrimitiveFixup(rc->GetBlockData(b).get()); + } + return TaskStatus::complete; +} } // namespace fixup diff --git a/src/fixup/fixup_radc2p.cpp b/src/fixup/fixup_radc2p.cpp index f593e1f0c..6eefb2af4 100644 --- a/src/fixup/fixup_radc2p.cpp +++ b/src/fixup/fixup_radc2p.cpp @@ -251,5 +251,12 @@ TaskStatus RadConservedToPrimitiveFixup(T *rc) { template TaskStatus RadConservedToPrimitiveFixup>(MeshBlockData *rc); +template <> +TaskStatus RadConservedToPrimitiveFixup>(MeshData *rc) { + for (int b = 0; b < rc->NumBlocks(); b++) { + RadConservedToPrimitiveFixup(rc->GetBlockData(b).get()); + } + return TaskStatus::complete; +} } // namespace fixup diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 631224a59..c52dc086d 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -560,40 +560,27 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { } // Fix up flux failures - // TODO (BLB) once the fixups can work with MeshData, - // -- then stitch this section back together - TaskRegion &async_region_2 = tc.AddRegion(num_independent_task_lists); - for (int ib = 0; ib < num_independent_task_lists; ib++) { - auto pmb = blocks[ib].get(); - auto &tl = async_region_2[ib]; - - // first make other useful containers - // auto &base = pmb->meshblock_data.Get(); + TaskRegion &sync_region_fixup = tc.AddRegion(num_partitions); + TaskID gas_rad_int(0); + for (int ib = 0; ib < num_partitions; ib++) { + auto &tl = sync_region_fixup[ib]; // pull out the container we'll use to get fluxes and/or compute RHSs - auto &sc1 = pmb->meshblock_data.Get(stage_name[stage]); - using MDT = std::remove_pointer::type; + auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], ib); // fill in derived fields auto fill_derived = - tl.AddTask(none, parthenon::Update::FillDerived>, sc1.get()); + tl.AddTask(none, parthenon::Update::FillDerived>, sc1.get()); - auto fixup = tl.AddTask( - fill_derived, fixup::ConservedToPrimitiveFixup>, sc1.get()); - - auto radfixup = tl.AddTask( - fixup, fixup::RadConservedToPrimitiveFixup>, sc1.get()); + auto fixup = tl.AddTask(fill_derived, + fixup::ConservedToPrimitiveFixup>, sc1.get()); - auto floors = - tl.AddTask(radfixup, fixup::ApplyFloors>, sc1.get()); - } + auto radfixup = + tl.AddTask(fixup, fixup::RadConservedToPrimitiveFixup>, sc1.get()); - TaskRegion &sync_region_fixup = tc.AddRegion(num_partitions); - TaskID gas_rad_int(0); - if (rad_mocmc_active) { - for (int ib = 0; ib < num_partitions; ib++) { - auto &tl = sync_region_fixup[ib]; + auto floors = tl.AddTask(radfixup, fixup::ApplyFloors>, sc1.get()); + if (rad_mocmc_active) { auto &base = pmesh->mesh_data.GetOrAdd("base", ib); auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], ib); using MDT = std::remove_pointer::type; From d3e341273c0cdd2833c825b20922b3519c2e9866 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Wed, 5 Feb 2025 05:11:42 +0000 Subject: [PATCH 46/48] MeshData level swarms comms, MeshData level MomentFluidSource, refactor driver according to previous feats --- src/phoebus_driver.cpp | 98 +++++++++++++++++--------------- src/radiation/moments_source.cpp | 7 +++ 2 files changed, 58 insertions(+), 47 deletions(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index c52dc086d..6bdccef1a 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -50,6 +50,43 @@ using namespace Geometry; namespace phoebus { +TaskStatus MeshResetCommunication(MeshData *md) { + const int nblocks = md->NumBlocks(); + for (int n = 0; n < nblocks; n++) { + auto &mbd = md->GetBlockData(n); + auto &sc = mbd->GetSwarmData(); + sc->ResetCommunication(); + } + + return TaskStatus::complete; +} + +TaskStatus MeshSend(MeshData *md) { + const int nblocks = md->NumBlocks(); + for (int n = 0; n < nblocks; n++) { + auto &mbd = md->GetBlockData(n); + auto &sc = mbd->GetSwarmData(); + sc->Send(BoundaryCommSubset::all); + } + + return TaskStatus::complete; +} + +TaskStatus MeshReceive(MeshData *md) { + TaskStatus status = TaskStatus::complete; + const int nblocks = md->NumBlocks(); + for (int n = 0; n < nblocks; n++) { + auto &mbd = md->GetBlockData(n); + auto &sc = mbd->GetSwarmData(); + auto local_status = sc->Receive(BoundaryCommSubset::all); + if (local_status == TaskStatus::incomplete) { + status = TaskStatus::incomplete; + } + } + + return status; +} + // *************************************************// // define the application driver. in this case, *// // that mostly means defining the MakeTaskList *// @@ -324,17 +361,6 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { } if (rad_mocmc_active) { - TaskRegion &sync_region_tr = tc.AddRegion(1); - { - for (int i = 0; i < blocks.size(); i++) { - auto &tl = sync_region_tr[0]; - auto &pmb = blocks[i]; - auto &sc = pmb->meshblock_data.Get()->GetSwarmData(); - auto reset_comms = - tl.AddTask(none, &SwarmContainer::ResetCommunication, sc.get()); - } - } - TaskRegion &sync_region_mocmc = tc.AddRegion(num_partitions); for (int n = 0; n < num_partitions; n++) { auto &tl = sync_region_mocmc[n]; @@ -342,29 +368,13 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { auto &base = pmesh->mesh_data.GetOrAdd("base", n); using MDT = std::remove_pointer::type; // TODO(BRR) stage_name[stage - 1]? + auto reset_comms = tl.AddTask(none, MeshResetCommunication, base.get()); auto samples_transport = - tl.AddTask(none, radiation::MOCMCTransport, base.get(), beta * dt); - } - TaskRegion &sync_region_mocmc_comm = tc.AddRegion(blocks.size()); - for (int ib = 0; ib < blocks.size(); ib++) { - auto &tl = sync_region_mocmc_comm[ib]; - auto pmb = blocks[ib].get(); - - // TODO(BRR) stage_name[stage - 1]? - auto &sc = pmb->meshblock_data.Get()->GetSwarmData(); - auto send = - tl.AddTask(none, &SwarmContainer::Send, sc.get(), BoundaryCommSubset::all); - auto receive = - tl.AddTask(send, &SwarmContainer::Receive, sc.get(), BoundaryCommSubset::all); - } - - TaskRegion &async_region_mocmc_2 = tc.AddRegion(num_partitions); - for (int n = 0; n < num_partitions; n++) { - auto &tl = async_region_mocmc_2[n]; + tl.AddTask(reset_comms, radiation::MOCMCTransport, base.get(), beta * dt); + auto send = tl.AddTask(samples_transport, MeshSend, base.get()); + auto receive = tl.AddTask(samples_transport | send, MeshReceive, base.get()); - auto &base = pmesh->mesh_data.GetOrAdd("base", n); auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], n); - using MDT = std::remove_pointer::type; // TODO(BRR) stage_name[stage - 1]? auto sample_bounds = tl.AddTask(none, radiation::MOCMCSampleBoundaries, base.get(), sc1.get()); @@ -561,7 +571,6 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { // Fix up flux failures TaskRegion &sync_region_fixup = tc.AddRegion(num_partitions); - TaskID gas_rad_int(0); for (int ib = 0; ib < num_partitions; ib++) { auto &tl = sync_region_fixup[ib]; @@ -580,32 +589,27 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { auto floors = tl.AddTask(radfixup, fixup::ApplyFloors>, sc1.get()); + auto &base = pmesh->mesh_data.GetOrAdd("base", ib); + using MDT = std::remove_pointer::type; + TaskID gas_rad_int(0); if (rad_mocmc_active) { - auto &base = pmesh->mesh_data.GetOrAdd("base", ib); - auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], ib); - using MDT = std::remove_pointer::type; - auto impl_update = tl.AddTask(none, radiation::MOCMCFluidSource, base.get(), + auto impl_update = tl.AddTask(floors, radiation::MOCMCFluidSource, base.get(), sc1.get(), beta * dt, fluid_active); auto impl_edd = tl.AddTask(impl_update, radiation::MOCMCEddington, base.get(), sc1.get()); gas_rad_int = gas_rad_int | impl_edd; - } - } - - if (rad_moments_active) { - TaskRegion &async_region_fixup2 = tc.AddRegion(num_independent_task_lists); - for (int ib = 0; ib < num_independent_task_lists; ib++) { - auto pmb = blocks[ib].get(); - auto &tl = async_region_fixup2[ib]; - auto &sc1 = pmb->meshblock_data.Get(stage_name[stage]); + } else if (rad_moments_active) { auto impl_update = - tl.AddTask(none, radiation::MomentFluidSource>, sc1.get(), + tl.AddTask(floors, radiation::MomentFluidSource, sc1.get(), beta * dt, fluid_active); gas_rad_int = gas_rad_int | impl_update; + } + + if (rad_moments_active) { // Only apply floors because MomentFluidSource already ensured that a sensible state // was returned auto floors = - tl.AddTask(gas_rad_int, fixup::ApplyFloors>, sc1.get()); + tl.AddTask(gas_rad_int, fixup::ApplyFloors>, sc1.get()); } } diff --git a/src/radiation/moments_source.cpp b/src/radiation/moments_source.cpp index 38173b8d5..c96fa62dd 100644 --- a/src/radiation/moments_source.cpp +++ b/src/radiation/moments_source.cpp @@ -987,5 +987,12 @@ TaskStatus MomentFluidSource(T *rc, Real dt, bool update_fluid) { } template TaskStatus MomentFluidSource>(MeshBlockData *, Real, bool); +template <> +TaskStatus MomentFluidSource>(MeshData *md, Real dt, bool update_fluid) { + for (int b = 0; b < md->NumBlocks(); b++) { + MomentFluidSource(md->GetBlockData(b).get(), dt, update_fluid); + } + return TaskStatus::complete; +} } // namespace radiation From c1871361cbff1b53b1cee516c77157776bdab026 Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Wed, 5 Feb 2025 05:39:15 +0000 Subject: [PATCH 47/48] update golds --- tst/regression/mocmc_diffusion.gold | 50 +-- tst/regression/mocmc_equilibration.gold | 32 +- tst/regression/radiation_diffusion.gold | 408 ++++++++++---------- tst/regression/radiation_equilibration.gold | 8 +- 4 files changed, 249 insertions(+), 249 deletions(-) diff --git a/tst/regression/mocmc_diffusion.gold b/tst/regression/mocmc_diffusion.gold index be328867b..f6c33b283 100644 --- a/tst/regression/mocmc_diffusion.gold +++ b/tst/regression/mocmc_diffusion.gold @@ -1,25 +1,25 @@ -2.903293198869289909e-07 -8.917324339173791865e-11 -1.015753710095253432e-10 -2.902242925555535038e-08 -2.203269912692418925e-03 -6.667470045966119585e-01 -1.853892724151311666e-04 -2.337474611586995866e-03 -2.038951701080027150e-03 -1.008054500631459446e-03 -2.727958063904742988e-04 -9.915326561658027293e-04 -4.440062562521645412e-03 --1.078541213857498782e-19 --9.518559574664915933e-22 --1.821776703100472804e-20 --6.373869807731488544e-20 --4.544586618454861709e-20 -2.605898796556116674e-19 -2.824314855367294156e-19 --1.467053235248274521e-19 --8.360076325690497809e-21 -9.421651747181250117e-20 --5.419127957100820803e-19 -1.637435460668574465e-19 +2.578814482748386221e-07 +8.539726882830980148e-11 +9.881492890533372952e-11 +2.878000679792628713e-08 +2.198127933800962577e-03 +6.512936284317732971e-01 +1.589897082372890248e-04 +2.487714603868329197e-02 +2.197674234395428569e-02 +1.035157783173230352e-02 +2.531339100900188883e-03 +1.082842734053424301e-02 +4.744884335341727905e-02 +2.535851403270078324e-18 +4.297042292744506166e-19 +7.780364944017298687e-19 +-2.523409998529510235e-18 +-3.779919609858391220e-18 +-8.950313327148915038e-20 +2.582566784304384226e-18 +1.451093382697625446e-18 +-4.241573517575109588e-19 +-2.809441171738037641e-18 +1.644841674053745085e-19 +6.040096338555097036e-19 diff --git a/tst/regression/mocmc_equilibration.gold b/tst/regression/mocmc_equilibration.gold index 61d88ba5e..83414a878 100644 --- a/tst/regression/mocmc_equilibration.gold +++ b/tst/regression/mocmc_equilibration.gold @@ -1,16 +1,16 @@ -7.899461039993100897e-01 -7.899461039993102007e-01 -7.899461039993099787e-01 -7.899461039993100897e-01 --2.379322113936495115e-17 --1.129620046446157642e-17 -9.319386201426945604e-19 -3.415748298368384591e-17 --5.269569986218940562e-18 --4.383374134615519398e-18 -4.868801217256894866e-18 -4.784142903577562012e-18 -5.173045656497453725e-18 --3.052373860422978175e-18 --5.992335522073705248e-18 -3.871663725999230853e-18 +7.698791973673844158e-01 +7.698791973673844158e-01 +7.698791973673843048e-01 +7.698791973673843048e-01 +6.913800778225646729e-17 +-2.572591733296812912e-17 +-7.173655366961485181e-17 +2.832446322032650131e-17 +3.478217022700787467e-18 +-1.527472444189700360e-18 +-2.790333412930537917e-18 +8.395888344194466692e-19 +3.637066182233955877e-18 +-2.981293358266952529e-18 +-4.067698972485125619e-18 +3.411926148518122272e-18 diff --git a/tst/regression/radiation_diffusion.gold b/tst/regression/radiation_diffusion.gold index 52117688e..3444878e3 100644 --- a/tst/regression/radiation_diffusion.gold +++ b/tst/regression/radiation_diffusion.gold @@ -1,204 +1,204 @@ -5.727756994965728402e-07 -2.390134319486193814e-08 -8.552041533534621507e-10 -1.078296430112438385e-10 -9.711876459274814704e-11 -9.713910069953824051e-11 -9.714019477587794905e-11 -9.714018556719315477e-11 -9.714013583064870344e-11 -9.714004143906456544e-11 -9.714136057593570395e-11 -9.715605647034073990e-11 -9.723061089040996236e-11 -9.746785021414422651e-11 -9.798169248803654994e-11 -9.873601435495395353e-11 -9.944825354323743467e-11 -9.985170078244693900e-11 -9.997856789721988854e-11 -9.999862401572621337e-11 -9.999997429126063500e-11 -9.999999997608914604e-11 -1.000000000145484692e-10 -1.000000045740661299e-10 -1.000006404787215883e-10 -1.000486007678750447e-10 -1.023092093125954138e-10 -1.757621936255253417e-10 -1.941590168349628520e-09 -3.504491520959144366e-08 -5.225935904712619433e-07 -6.176559144957398269e-06 -5.779425074201662320e-05 -4.282440483189261051e-04 -2.512772859434293186e-03 -1.167086067769811455e-02 -4.287634506986499200e-02 -1.244570709266687308e-01 -2.850180079099510766e-01 -5.140050326773021316e-01 -7.282509198683035923e-01 -8.067801476054499155e-01 -6.998498162483933749e-01 -4.717382893901956309e-01 -2.462135553878651295e-01 -9.900483254205026673e-02 -3.049183983645771534e-02 -7.143371984330692398e-03 -1.262702652840829451e-03 -1.668041847985367678e-04 -1.627808859937456717e-05 -1.123635279827187067e-06 -5.016698617450419950e-02 -5.311995564210896664e-02 -1.680911903892158554e-02 -2.685308764008071427e-05 --2.355360567551283064e-06 --2.278357889466065552e-09 -2.666990845618345271e-09 -1.268488393554597766e-08 -1.269934976676971702e-08 --4.400954823655228269e-07 --4.079477796975521067e-06 --1.864875973519957813e-05 --5.415651297246726640e-05 --1.063016992923665233e-04 --1.377446224133029192e-04 --1.102671082456988558e-04 --5.036791500535575921e-05 --1.194434774010715570e-05 --1.280766957156953484e-06 --4.683094159786734856e-08 --1.484523354899965649e-10 --4.421560534975262109e-12 --1.451084030538389196e-09 --2.046534387257914043e-07 --1.533153330586361159e-05 --7.046323563657893750e-04 --1.645589022743774651e-02 --4.965832613189426203e-02 --5.020480634544625259e-02 --4.626722341124880900e-02 --4.213012095570772136e-02 --3.798606919926616338e-02 --3.384163722145849124e-02 --2.969559488533482852e-02 --2.554636732861065643e-02 --2.139230530721064616e-02 --1.723174961639098446e-02 --1.306297343227211250e-02 --8.883500610028981226e-03 --4.689001102128260642e-03 --2.397992361798591298e-04 -3.754639271729428174e-03 -8.014743424947094394e-03 -1.230509638607621588e-02 -1.663267006808121992e-02 -2.100717937714585704e-02 -2.543980051006110993e-02 -2.994404537918140055e-02 -3.453717733249578314e-02 -3.924181203534791995e-02 -4.407161256738489907e-02 --1.868767268482240491e-22 --7.641427680194446907e-23 --6.804516732338074833e-23 -7.696573611664989199e-23 --3.029789149918279561e-22 -1.325849849415484424e-22 -1.827833703677350937e-22 --1.798204906089769979e-22 --1.994375227157259929e-22 --1.128720256415789531e-22 -2.806257048517271053e-22 -4.353674803135171031e-23 --9.760997454810156110e-23 -1.896638842126431961e-22 -1.936603512183230315e-23 -1.580851173246986268e-22 --2.920286453783151726e-22 --6.458585200373196152e-23 -3.261438212187573023e-22 --1.635206802771003537e-22 -5.196807219326522329e-23 --1.940874631207208132e-22 -3.202876348596336658e-22 --9.205330069826698777e-23 -6.685343097314401535e-23 -1.876387094514924327e-22 --3.346456175067505297e-22 --8.661806913602193686e-23 -2.714175968898874512e-22 --1.375071315110659645e-22 --3.821544454524002346e-23 -3.363643437678766407e-23 --7.050516093164550709e-23 -1.685568510204387630e-22 --4.654457833402880344e-23 -1.290084028486700240e-22 -1.457948221271299510e-22 -9.072037070181711738e-23 -6.031616359304777192e-23 --3.219915272597726373e-22 --3.661177225926254289e-22 --1.292076190041262082e-22 --1.848766504132850588e-22 -3.576826733278346032e-22 -4.004773267704565626e-22 --1.536059366362080154e-22 --2.384660447196411676e-22 -3.717300217412897763e-22 --1.101313977316121747e-22 --3.319711620057026527e-22 -9.779126994823706531e-23 -2.006372983480654967e-22 --2.627955561810508638e-22 -2.168296403126974973e-23 -3.038608284578627255e-22 -4.545338251117119525e-23 --7.387304247504195904e-23 --5.245124621623921559e-23 -1.887761815603797742e-22 -5.071921698641457001e-24 -8.435045479648305304e-23 --1.516127675197357563e-22 --4.575003927093133333e-23 --1.963465725402569970e-22 -3.705826620196841251e-23 --1.750363343885057716e-22 -1.154917064906796530e-22 -4.486429391980784950e-22 --1.753935150281962650e-22 -3.120498712890031069e-22 -2.060536443397245257e-22 -1.772067652250144103e-22 --8.640881062529312116e-23 --5.903633035570862533e-23 --1.756470543022796535e-22 -7.092859059373325788e-24 --6.643883450454763159e-23 --1.713818450873749800e-22 --6.995206398125411142e-24 --7.727514724108877460e-23 --2.056816450131976979e-22 --2.122987666405352667e-22 --1.616826368358732784e-22 --1.555098971713018895e-22 --3.008697083894963898e-22 --1.637921629454198903e-22 --1.746625146671844762e-22 -2.731092170496370455e-22 --2.751892007889170968e-22 -3.410244855996552411e-23 -1.289209871199362097e-23 --2.333180364727235204e-22 --4.187978621485973221e-23 --1.825614611388922589e-22 -2.755308002943173615e-22 --2.711917168254534971e-22 -8.927134484809464633e-23 -1.693058093012223153e-22 --1.072491368960832741e-22 --5.021986005922945311e-22 -4.021009130812441423e-22 +6.148519464058904541e-07 +2.623892799365704451e-08 +9.539299957556873008e-10 +1.097925175521065226e-10 +9.698685107840106121e-11 +9.700735207148735836e-11 +9.700855170309239901e-11 +9.700854131505957682e-11 +9.700848604280721838e-11 +9.700839486333799533e-11 +9.700998705309238909e-11 +9.702660891623257019e-11 +9.710828702343892631e-11 +9.736260875897138338e-11 +9.790545571475139446e-11 +9.869430905366765583e-11 +9.943312020840976194e-11 +9.984853597427683277e-11 +9.997824932337328797e-11 +9.999861352928875433e-11 +9.999997436561353504e-11 +9.999999997660431154e-11 +1.000000000140515146e-10 +1.000000043840857233e-10 +1.000006084550747897e-10 +1.000456769896741220e-10 +1.021446086957923230e-10 +1.701063277402130924e-10 +1.827686282796141058e-09 +3.335576536443429658e-08 +5.034117650423832286e-07 +6.011002868924303051e-06 +5.672185718319043676e-05 +4.231424312440042060e-04 +2.495696167647177763e-03 +1.163478979356291458e-02 +4.284849082069772108e-02 +1.245451911643267323e-01 +2.853591843479902357e-01 +5.145531879348246296e-01 +7.286884038685831344e-01 +8.068838598063016176e-01 +6.998431817787323928e-01 +4.720041922922508237e-01 +2.467486611988026213e-01 +9.951367284863543716e-02 +3.079012872976234425e-02 +7.260759428897966931e-03 +1.294831044491474934e-03 +1.730107873849609415e-04 +1.712756748856066036e-05 +1.202183306907686552e-06 +5.074225279675498995e-02 +5.402810637784936226e-02 +1.888716606287351232e-02 +3.788337365701142240e-05 +-2.616499114530969193e-06 +-2.286177320353717435e-09 +3.000371620316649112e-09 +1.405787203100672773e-08 +6.773859766534667883e-09 +-5.232686354011619752e-07 +-4.589144658609571317e-06 +-2.034565222209474867e-05 +-5.787117026449888600e-05 +-1.120306814485762340e-04 +-1.437216322540358032e-04 +-1.140893004446133188e-04 +-5.171257580973905743e-05 +-1.217308947984350423e-05 +-1.295554805768857514e-06 +-4.694238308857297540e-08 +-1.452650655100214653e-10 +-4.272717129183675924e-12 +-1.391212583164108648e-09 +-1.944289561274645613e-07 +-1.440455946451621791e-05 +-6.541320671425316377e-04 +-1.543791468150581755e-02 +-4.872072052042272089e-02 +-4.953862261182483129e-02 +-4.569416026514530604e-02 +-4.164081235621738819e-02 +-3.757548493151072894e-02 +-3.350430264466900537e-02 +-2.942556656250817568e-02 +-2.533722626053625665e-02 +-2.123714005338387875e-02 +-1.712313852154304789e-02 +-1.299297077336182896e-02 +-8.843623314542374894e-03 +-4.670205125795492268e-03 +-2.305441085521117239e-04 +3.760799292794245446e-03 +8.030257233360404631e-03 +1.234052436289972098e-02 +1.669840962643823895e-02 +2.111341029293156007e-02 +2.559647842759555161e-02 +3.016088480912522407e-02 +3.482362737397468044e-02 +3.960701533656195428e-02 +4.454103932968823260e-02 +1.331543215889358418e-22 +1.449048012375765925e-22 +-5.840344864391299591e-23 +-4.142925430177124312e-22 +-6.625058095387177299e-23 +-7.140446846678581789e-23 +1.771553100880480093e-22 +9.781944953591423673e-23 +9.162720621882662440e-23 +1.204254755894580195e-22 +1.981917164133671776e-22 +1.837239786602573270e-22 +-6.532090072774898377e-23 +3.388608665940369599e-22 +-2.637135005108609832e-23 +-5.667218291935442503e-23 +3.824257239590099752e-22 +1.642723364845939941e-22 +-1.191764057220626943e-22 +2.032010375432194001e-22 +1.468539127018607578e-22 +1.362816788494320745e-22 +-1.490398551282741487e-22 +2.014876104064194665e-22 +2.945654730970317451e-22 +-2.566953706825307772e-22 +-2.568607864953637649e-22 +-2.248216315953183658e-22 +-8.535929972452356132e-23 +-1.226486645198053177e-22 +3.303108876537525858e-22 +-3.154040624455984280e-22 +1.115516992741287495e-22 +-2.722502965051908163e-22 +-2.214823134498584583e-22 +1.739254321183681078e-22 +2.880609836370706566e-22 +-2.605351091226014014e-22 +2.352664962607755773e-22 +-1.228618762985802809e-22 +7.917302483171992569e-23 +-2.088467446501521971e-22 +8.708499363504617154e-23 +-3.215095351150821572e-22 +1.716397533209515926e-22 +2.865097620584068819e-22 +-2.801131585172059201e-22 +7.248544286539459298e-23 +-1.118118003285572442e-22 +-4.246754181384880607e-23 +1.616548269686875165e-22 +3.536602752843460642e-22 +-2.562182240611372686e-22 +1.420944762750985085e-22 +3.327098699446866535e-23 +4.045582076110651761e-23 +-1.815692516630341649e-22 +-4.888613501607002443e-23 +-2.386824240657490642e-22 +5.610593882581087645e-23 +1.497790119297060339e-22 +1.384318036790286671e-22 +2.322793653141452714e-22 +6.097019390604360411e-23 +-1.271637813864604155e-22 +-9.103367589897254445e-23 +2.176824137475862983e-23 +-1.368971579691912148e-22 +7.639490373611592537e-23 +3.281970443953975512e-24 +2.233467307263979615e-22 +6.278623074712339672e-23 +5.929016282816159961e-23 +2.458367288273487457e-22 +-5.631456179917167376e-23 +-1.858499646409524352e-22 +1.187050096287243954e-22 +-8.394083332511742891e-23 +5.489616617044200368e-22 +4.282216076559168001e-22 +3.921078498099024952e-22 +1.170384473448311927e-22 +1.604361978512895507e-23 +-2.207706446395669817e-22 +-2.537958809470855371e-22 +3.549470522578481163e-22 +-3.072257625811678830e-22 +-2.462839338467965054e-22 +8.965576263635079820e-25 +2.278226835262596707e-22 +-3.860083743465736477e-22 +-2.637660224911399509e-22 +2.931952295852275688e-23 +2.356276635677745275e-22 +1.161521464425352583e-22 +-1.052166011900077566e-23 +1.913216458015005852e-22 +-1.241810591828786866e-23 +-4.486848283273262171e-22 +8.900011991657956885e-23 +-8.485179905361157198e-23 diff --git a/tst/regression/radiation_equilibration.gold b/tst/regression/radiation_equilibration.gold index c760e2a3b..d18cd1de5 100644 --- a/tst/regression/radiation_equilibration.gold +++ b/tst/regression/radiation_equilibration.gold @@ -1,7 +1,7 @@ -7.590477907760553311e-01 -7.590477907760553311e-01 -7.590477907760553311e-01 -7.590477907760553311e-01 +7.463101650882452143e-01 +7.463101650882452143e-01 +7.463101650882452143e-01 +7.463101650882452143e-01 0.000000000000000000e+00 0.000000000000000000e+00 0.000000000000000000e+00 From e494815c165719fe1e6acdde95b406d6e069dd9d Mon Sep 17 00:00:00 2001 From: Brandon Barker Date: Wed, 5 Feb 2025 05:47:44 +0000 Subject: [PATCH 48/48] fmt --- src/phoebus_driver.cpp | 5 ++--- src/radiation/mocmc.cpp | 4 ++-- src/radiation/moments_source.cpp | 3 ++- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 6bdccef1a..267b1c0f1 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -599,9 +599,8 @@ TaskCollection PhoebusDriver::RungeKuttaStage(const int stage) { tl.AddTask(impl_update, radiation::MOCMCEddington, base.get(), sc1.get()); gas_rad_int = gas_rad_int | impl_edd; } else if (rad_moments_active) { - auto impl_update = - tl.AddTask(floors, radiation::MomentFluidSource, sc1.get(), - beta * dt, fluid_active); + auto impl_update = tl.AddTask(floors, radiation::MomentFluidSource, sc1.get(), + beta * dt, fluid_active); gas_rad_int = gas_rad_int | impl_update; } diff --git a/src/radiation/mocmc.cpp b/src/radiation/mocmc.cpp index 51fb36ffe..937ad7d36 100644 --- a/src/radiation/mocmc.cpp +++ b/src/radiation/mocmc.cpp @@ -475,8 +475,8 @@ TaskStatus MOCMCReconstruction(T *rc_base, T *rc) { } break; } // if inside - } // m = 0..n - } // n = 0..nsamp + } // m = 0..n + } // n = 0..nsamp }); return TaskStatus::complete; diff --git a/src/radiation/moments_source.cpp b/src/radiation/moments_source.cpp index c96fa62dd..b11493b57 100644 --- a/src/radiation/moments_source.cpp +++ b/src/radiation/moments_source.cpp @@ -988,7 +988,8 @@ TaskStatus MomentFluidSource(T *rc, Real dt, bool update_fluid) { template TaskStatus MomentFluidSource>(MeshBlockData *, Real, bool); template <> -TaskStatus MomentFluidSource>(MeshData *md, Real dt, bool update_fluid) { +TaskStatus MomentFluidSource>(MeshData *md, Real dt, + bool update_fluid) { for (int b = 0; b < md->NumBlocks(); b++) { MomentFluidSource(md->GetBlockData(b).get(), dt, update_fluid); }