From 8c7a8f4b3a572a6460b8b5268fe031332cc47713 Mon Sep 17 00:00:00 2001 From: "Ben R. Ryan" Date: Thu, 9 Apr 2020 22:17:06 -0600 Subject: [PATCH 01/11] Added SwarmContainer --- src/CMakeLists.txt | 1 + src/interface/SwarmContainer.cpp | 128 ++++++++++++++++++++++++++ src/interface/SwarmContainer.hpp | 152 +++++++++++++++++++++++++++++++ 3 files changed, 281 insertions(+) create mode 100644 src/interface/SwarmContainer.cpp create mode 100644 src/interface/SwarmContainer.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 63d1b2c..1cd075a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -94,6 +94,7 @@ add_library(parthenon interface/SparseVariable.cpp interface/Metadata.cpp interface/Stage.cpp + interface/SwarmContainer.cpp interface/Update.cpp interface/Variable.cpp diff --git a/src/interface/SwarmContainer.cpp b/src/interface/SwarmContainer.cpp new file mode 100644 index 0000000..202d3ad --- /dev/null +++ b/src/interface/SwarmContainer.cpp @@ -0,0 +1,128 @@ +//======================================================================================== +// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== +#include +#include +#include +#include +#include "bvals/cc/bvals_cc.hpp" +#include "SwarmContainer.hpp" +#include "globals.hpp" // my_rank +#include "SparseVariable.hpp" +#include "mesh/mesh.hpp" + +namespace parthenon { + +void SwarmContainer::Add(const std::vector labelArray, + const Metadata &metadata) { + // generate the vector and call Add + for (auto label : labelArray) { + Add(label, metadata); + } +} + +/// +/// The internal routine for allocating a particle swarm. This subroutine +/// is topology aware and will allocate accordingly. +/// +/// @param label the name of the variable +/// @param metadata the metadata associated with the variable +void SwarmContainer::Add(const std::string label, + const Metadata &metadata) { + auto swarm = std::make_shared(label, metadata); + swarms.push_back(swarm); +} + +// TODO(JMM): this could be cleaned up, I think. +// Maybe do only one loop, or do the cleanup at the end. +void SwarmContainer::Remove(const std::string label) { + int idx, isize; + + // Find index of swarm + isize = swarms.size(); + idx = 0; + for (auto s : swarms) { + if ( ! label.compare(s->label())) { + break; + } + idx++; + } + if ( idx >= isize) { + throw std::invalid_argument ("swarm not found in Remove()"); + } + + // first delete the variable + swarms[idx].reset(); + + // Next move the last element into idx and pop last entry + isize--; + if ( isize >= 0) swarms[idx] = std::move(swarms.back()); + swarms.pop_back(); + return; +} + +void SwarmContainer::SendBoundaryBuffers() {} + +void SwarmContainer::SetupPersistentMPI() {} + +bool SwarmContainer::ReceiveBoundaryBuffers() {} + +void SwarmContainer::ReceiveAndSetBoundariesWithWait() {} + +void SwarmContainer::SetBoundaries() {} + +void SwarmContainer::StartReceiving(BoundaryCommSubset phase) {} + +void SwarmContainer::ClearBoundary(BoundaryCommSubset phase) {} + +void SwarmContainer::print() { + std::cout << "Swarms are:\n"; + for (auto s : swarms) { std::cout << " " << s->info() << std::endl; } +} + +static void AddSwarm(Swarm&V, std::vector& sRet) { + // adds aliases to sRet + sRet.push_back(Swarm(V)); +} + + +/// Gets an array of real variables from container. +/// @param index_ret is returned with starting index for each name +/// @param count_ret is returned with number of arrays for each name +/// @param sparse_ids if specified, only those sparse IDs are returned +int SwarmContainer::GetVariables(const std::vector& names, + std::vector& sRet, + std::map>& indexCount, + const std::vector& sparse_ids) { + // First count how many entries we need and fill in index and count + indexCount.clear(); + + int index = 0; + for (auto label : names) { + int count = 0; + try { // normal variable + Swarm& S = Get(label); + AddSwarm(S, sRet); + count++; + } + catch (const std::invalid_argument& x) { + throw std::invalid_argument (" Unable to find swarm " + + label + " in container"); + } + indexCount[label] = std::make_pair(index,count); + index += count; + } // (auto label : names) + + return index; +} + +} // namespace parthenon diff --git a/src/interface/SwarmContainer.hpp b/src/interface/SwarmContainer.hpp new file mode 100644 index 0000000..e921e82 --- /dev/null +++ b/src/interface/SwarmContainer.hpp @@ -0,0 +1,152 @@ +//======================================================================================== +// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== +#ifndef INTERFACE_SWARM_CONTAINER_HPP_ +#define INTERFACE_SWARM_CONTAINER_HPP_ + +#include +#include +#include +#include // +#include +#include "globals.hpp" +//#include "mesh/mesh.hpp" +#include "SparseVariable.hpp" +#include "Stage.hpp" +#include "Swarm.hpp" +#include "Variable.hpp" + +namespace parthenon { +/// +/// Interface to underlying infrastructure for particle data declaration and +/// access. +/// Date: August 22, 2019 +/// +/// +/// The SwarmContainer class is a container for the swarms of particles that +/// make up the simulation. +/// +/// The container class will provide the following methods: +/// + +class MeshBlock; + +class SwarmContainer { + public: + //----------------- + // Public Variables + //----------------- + MeshBlock* pmy_block = nullptr; // ptr to MeshBlock + + //----------------- + //Public Methods + //----------------- + // Constructor does nothing + SwarmContainer() { + } + + /// + /// Set the pointer to the mesh block for this container + void setBlock(MeshBlock *pmb) { pmy_block = pmb; } + + /// + /// Allocate and add a variable to the container + /// + /// This function will eventually look at the metadata flags to + /// identify the size of the first dimension based on the + /// topological location. Dimensions will be taken from the metadata. + /// + /// @param label the name of the variable + /// @param metadata the metadata associated with the variable + /// + void Add(const std::string label, const Metadata &metadata); + + /// + /// Allocate and add a variable to the container + /// + /// This function will eventually look at the metadata flags to + /// identify the size of the first dimension based on the + /// topological location. Dimensions will be taken from the metadata. + /// + /// @param labelArray the array of names of variables + /// @param metadata the metadata associated with the variable + /// + void Add(const std::vector labelArray, const Metadata &metadata); + + /// + /// Get a swarm from the container + /// @param label the name of the swarm + /// @return the Swarm if found or throw exception + Swarm& Get(std::string label) { + for (auto v : swarms) { + if (! v->label().compare(label)) return *v; + } + throw std::invalid_argument (std::string("\n") + + std::string(label) + + std::string(" swarm not found in Get()\n") ); + } + + Variable& Get(const int index) { + return *(s->swarms[index]); + } + + int Index(const std::string& label) { + for (int i = 0; i < swarmArray.size(); i++) { + if (! swarms[i]->label().compare(label)) return i; + } + return -1; + } + + /// Gets an array of real variables from container. + /// @param names is the variables we want + /// @param indexCount a map of names to std::pair for each name + /// @param sparse_ids if specified is list of sparse ids we are interested in. Note + /// that non-sparse variables specified are aliased in as is. + int GetSwarms(const std::vector& names, + std::vector& sRet, + std::map>& indexCount); + + /// + /// Remove a variable from the container or throw exception if not + /// found. + /// @param label the name of the variable to be deleted + void Remove(const std::string label); + + // Temporary functions till we implement a *real* iterator + + /// Print list of labels in container + void print(); + + // return number of stored arrays + int size() {return swarms.size();} + + // Element accessor functions + std::vector>& allSwarms() { + return swarms; + } + + // Communication routines + void SetupPersistentMPI(); + void SetBoundaries(); + void SendBoundaryBuffers(); + void ReceiveAndSetBoundariesWithWait(); + bool ReceiveBoundaryBuffers(); + void StartReceiving(BoundaryCommSubset phase); + void ClearBoundary(BoundaryCommSubset phase); + + private: + int debug=0; + std::vector> swarms = {}; +}; + +} // namespace parthenon +#endif // INTERFACE_SWARM_CONTAINER_HPP_ From e270277c1cb6831d3e5d51b62448cb45e1d1bb26 Mon Sep 17 00:00:00 2001 From: "Ben R. Ryan" Date: Thu, 9 Apr 2020 22:20:20 -0600 Subject: [PATCH 02/11] Added Swarm --- src/interface/Swarm.hpp | 69 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 src/interface/Swarm.hpp diff --git a/src/interface/Swarm.hpp b/src/interface/Swarm.hpp new file mode 100644 index 0000000..c7af295 --- /dev/null +++ b/src/interface/Swarm.hpp @@ -0,0 +1,69 @@ +//======================================================================================== +// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== +#ifndef INTERFACE_SWARM_HPP_ +#define INTERFACE_SWARM_HPP_ + +/// +/// A Variable type for Placebo-K. +/// Builds on AthenaArrays +/// Date: August 21, 2019 +/// +/// +/// The variable class typically contains state data for the +/// simulation but can also hold non-mesh-based data such as physics +/// parameters, etc. It inherits the AthenaArray class, which is used +/// for actural data storage and generation + +#include +#include +#include +#include +#include +#include +#include +#include "athena.hpp" +#include "athena_arrays.hpp" +#include "bvals/cc/bvals_cc.hpp" +#include "Metadata.hpp" + +namespace parthenon { +class MeshBlock; + +class Swarm { + public: + Swarm(const std::string label, Metadata &metadata) : + _label(label), + _m(metadata), + mpiStatus(true) {} + + ///< Assign label for variable + void setLabel(const std::string label) { _label = label; } + + ///< retrieve label for variable + std::string label() const { return _label; } + + ///< retrieve metadata for variable + const Metadata metadata() const { return _m; } + + std::string getAssociated() { return _m.getAssociated(); } + + bool mpiStatus; + + private: + Metadata _m; + std::string _label; +} + +} // namespace parthenon + +#endif // INTERFACE_VARIABLE_HPP_ From 971a47cd0865ae2c86e896141c80d83cee4f5533 Mon Sep 17 00:00:00 2001 From: "Ben R. Ryan" Date: Thu, 9 Apr 2020 23:08:47 -0600 Subject: [PATCH 03/11] Get code to compile --- src/interface/Swarm.hpp | 7 +++++-- src/interface/SwarmContainer.hpp | 11 ++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/interface/Swarm.hpp b/src/interface/Swarm.hpp index c7af295..fd5a717 100644 --- a/src/interface/Swarm.hpp +++ b/src/interface/Swarm.hpp @@ -41,7 +41,7 @@ class MeshBlock; class Swarm { public: - Swarm(const std::string label, Metadata &metadata) : + Swarm(const std::string label, const Metadata &metadata) : _label(label), _m(metadata), mpiStatus(true) {} @@ -57,12 +57,15 @@ class Swarm { std::string getAssociated() { return _m.getAssociated(); } + /// return information string + std::string info() { return std::string("Default information"); } + bool mpiStatus; private: Metadata _m; std::string _label; -} +}; } // namespace parthenon diff --git a/src/interface/SwarmContainer.hpp b/src/interface/SwarmContainer.hpp index e921e82..adaf415 100644 --- a/src/interface/SwarmContainer.hpp +++ b/src/interface/SwarmContainer.hpp @@ -95,12 +95,12 @@ class SwarmContainer { std::string(" swarm not found in Get()\n") ); } - Variable& Get(const int index) { - return *(s->swarms[index]); + Swarm& Get(const int index) { + return *(swarms[index]); } int Index(const std::string& label) { - for (int i = 0; i < swarmArray.size(); i++) { + for (int i = 0; i < swarms.size(); i++) { if (! swarms[i]->label().compare(label)) return i; } return -1; @@ -143,6 +143,11 @@ class SwarmContainer { void StartReceiving(BoundaryCommSubset phase); void ClearBoundary(BoundaryCommSubset phase); + int GetVariables(const std::vector& names, + std::vector& sRet, + std::map>& indexCount, + const std::vector& sparse_ids); + private: int debug=0; std::vector> swarms = {}; From 6803d3690df5c27376f9cd2e1aaef38f04654040 Mon Sep 17 00:00:00 2001 From: "Ben R. Ryan" Date: Fri, 10 Apr 2020 17:53:10 -0600 Subject: [PATCH 04/11] Compiling again --- src/interface/ParticleMetadata.hpp | 36 ++++++++++++++ src/interface/Swarm.hpp | 75 ++++++++++++++++++++++++++---- src/interface/SwarmContainer.cpp | 14 +++--- src/interface/SwarmContainer.hpp | 7 +-- src/interface/SwarmMetadata.hpp | 36 ++++++++++++++ 5 files changed, 145 insertions(+), 23 deletions(-) create mode 100644 src/interface/ParticleMetadata.hpp create mode 100644 src/interface/SwarmMetadata.hpp diff --git a/src/interface/ParticleMetadata.hpp b/src/interface/ParticleMetadata.hpp new file mode 100644 index 0000000..ab991d5 --- /dev/null +++ b/src/interface/ParticleMetadata.hpp @@ -0,0 +1,36 @@ +//======================================================================================== +// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== +#ifndef INTERFACE_PARTICLE_METADATA_HPP_ +#define INTERFACE_PARTICLE_METADATA_HPP_ + +#include +#include +#include +#include +#include +#include +#include + +namespace parthenon { + +/// @brief +/// +/// The ParticleMetadata class is a descriptor for particle species in the +/// simulation. +/// +/// Can set or query attributes specifed in flags. +/// +class ParticleMetadata { +}; +} +#endif // INTERFACE_PARTICLE_METADATA_HPP_ diff --git a/src/interface/Swarm.hpp b/src/interface/Swarm.hpp index fd5a717..78e8a38 100644 --- a/src/interface/Swarm.hpp +++ b/src/interface/Swarm.hpp @@ -34,37 +34,92 @@ #include "athena.hpp" #include "athena_arrays.hpp" #include "bvals/cc/bvals_cc.hpp" -#include "Metadata.hpp" +#include "ParticleMetadata.hpp" +#include "SwarmMetadata.hpp" namespace parthenon { class MeshBlock; +enum class PARTICLE_TYPE { + INT, REAL, STRING +}; + +enum class PARTICLE_STATUS { + UNALLOCATED, ALIVE, DEAD +}; + class Swarm { public: - Swarm(const std::string label, const Metadata &metadata) : + Swarm(const std::string label, const SwarmMetadata &smetadata, + const int nmax_pool_in = 1000) : _label(label), - _m(metadata), + _sm(smetadata), + _nmax_pool(nmax_pool_in), mpiStatus(true) {} - ///< Assign label for variable + ///< Assign label for swarm void setLabel(const std::string label) { _label = label; } - ///< retrieve label for variable + ///< retrieve label for swarm std::string label() const { return _label; } - ///< retrieve metadata for variable - const Metadata metadata() const { return _m; } + ///< retrieve metadata for swarm + const SwarmMetadata swarmmetadata() const { return _sm; } - std::string getAssociated() { return _m.getAssociated(); } + /// Assign info for swarm + void setInfo(const std::string info) { _info = info; } /// return information string - std::string info() { return std::string("Default information"); } + std::string info() { return _info; } + + /// Set max pool size + void setPoolMax(const int nmax_pool) { + _nmax_pool = nmax_pool; + // TODO(BRR) resize arrays and copy data + } bool mpiStatus; + void Add(const std::string &label, const ParticleMetadata) { + // TODO(BRR) fix this + PARTICLE_TYPE type = PARTICLE_TYPE::REAL; + // TODO(BRR) check that value isn't already enrolled? + _labelArray.push_back(label); + _typeArray.push_back(type); + if (type == PARTICLE_TYPE::INT) { + _intArray.push_back(std::make_shared>(_nmax_pool)); + } else if (type == PARTICLE_TYPE::REAL) { + _realArray.push_back(std::make_shared>(_nmax_pool)); + } else if (type == PARTICLE_TYPE::STRING) { + _stringArray.push_back(std::make_shared>(_nmax_pool)); + } else { + throw std::invalid_argument(std::string("\n") + + std::to_string(static_cast(type)) + + std::string(" not a PARTICLE_TYPE in Add()\n") ); + + } + } + + void AddParticle() { + // Check that particle fits, if not double size of pool via + // setPoolMax(2*_nmax_pool); + } + + void Defrag() { + // TODO(BRR) Put an O(N) algorithm here to defrag memory + } + private: - Metadata _m; + int _nmax_pool; + SwarmMetadata _sm; std::string _label; + std::string _info; + std::vector _labelArray; + std::vector _typeArray; + std::shared_ptr> _pstatus; + std::vector>> _intArray; + std::vector>> _realArray; + std::vector>> _stringArray; }; } // namespace parthenon diff --git a/src/interface/SwarmContainer.cpp b/src/interface/SwarmContainer.cpp index 202d3ad..7facb39 100644 --- a/src/interface/SwarmContainer.cpp +++ b/src/interface/SwarmContainer.cpp @@ -17,16 +17,15 @@ #include "bvals/cc/bvals_cc.hpp" #include "SwarmContainer.hpp" #include "globals.hpp" // my_rank -#include "SparseVariable.hpp" #include "mesh/mesh.hpp" namespace parthenon { void SwarmContainer::Add(const std::vector labelArray, - const Metadata &metadata) { + const SwarmMetadata &smetadata) { // generate the vector and call Add for (auto label : labelArray) { - Add(label, metadata); + Add(label, smetadata); } } @@ -35,15 +34,14 @@ void SwarmContainer::Add(const std::vector labelArray, /// is topology aware and will allocate accordingly. /// /// @param label the name of the variable -/// @param metadata the metadata associated with the variable +/// @param pmetadata the metadata associated with the particle void SwarmContainer::Add(const std::string label, - const Metadata &metadata) { - auto swarm = std::make_shared(label, metadata); + const SwarmMetadata &smetadata) { + auto swarm = std::make_shared(label, smetadata); + // TODO(BRR) check that swarm isn't already enrolled? swarms.push_back(swarm); } -// TODO(JMM): this could be cleaned up, I think. -// Maybe do only one loop, or do the cleanup at the end. void SwarmContainer::Remove(const std::string label) { int idx, isize; diff --git a/src/interface/SwarmContainer.hpp b/src/interface/SwarmContainer.hpp index adaf415..d4d262f 100644 --- a/src/interface/SwarmContainer.hpp +++ b/src/interface/SwarmContainer.hpp @@ -20,10 +20,7 @@ #include #include "globals.hpp" //#include "mesh/mesh.hpp" -#include "SparseVariable.hpp" -#include "Stage.hpp" #include "Swarm.hpp" -#include "Variable.hpp" namespace parthenon { /// @@ -68,7 +65,7 @@ class SwarmContainer { /// @param label the name of the variable /// @param metadata the metadata associated with the variable /// - void Add(const std::string label, const Metadata &metadata); + void Add(const std::string label, const SwarmMetadata &smetadata); /// /// Allocate and add a variable to the container @@ -80,7 +77,7 @@ class SwarmContainer { /// @param labelArray the array of names of variables /// @param metadata the metadata associated with the variable /// - void Add(const std::vector labelArray, const Metadata &metadata); + void Add(const std::vector labelArray, const SwarmMetadata &smetadata); /// /// Get a swarm from the container diff --git a/src/interface/SwarmMetadata.hpp b/src/interface/SwarmMetadata.hpp new file mode 100644 index 0000000..b68d7d5 --- /dev/null +++ b/src/interface/SwarmMetadata.hpp @@ -0,0 +1,36 @@ +//======================================================================================== +// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== +#ifndef INTERFACE_SWARM_METADATA_HPP_ +#define INTERFACE_SWARM_METADATA_HPP_ + +#include +#include +#include +#include +#include +#include +#include + +namespace parthenon { + +/// @brief +/// +/// The ParticleMetadata class is a descriptor for particle species in the +/// simulation. +/// +/// Can set or query attributes specifed in flags. +/// +class SwarmMetadata { +}; +} +#endif // INTERFACE_SWARM_METADATA_HPP_ From fabd811fb485157276286b5e3ed662bd4eb61a9c Mon Sep 17 00:00:00 2001 From: "Ben R. Ryan" Date: Fri, 10 Apr 2020 18:01:26 -0600 Subject: [PATCH 05/11] Quick fix --- src/interface/Swarm.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/interface/Swarm.hpp b/src/interface/Swarm.hpp index 78e8a38..a93b1f6 100644 --- a/src/interface/Swarm.hpp +++ b/src/interface/Swarm.hpp @@ -111,6 +111,7 @@ class Swarm { private: int _nmax_pool; + int _nmax_occupied = 0; SwarmMetadata _sm; std::string _label; std::string _info; From ec8d8ed8394e0d6f127690e0e2bd23559ae1ee40 Mon Sep 17 00:00:00 2001 From: "Ben R. Ryan" Date: Thu, 16 Apr 2020 13:45:41 -0600 Subject: [PATCH 06/11] Moved swarmcontainer into containercollection --- src/interface/container_collection.hpp | 30 +++++++++++- src/interface/swarm.hpp | 15 +++--- src/interface/swarm_container.cpp | 64 +++++++------------------- src/interface/swarm_container.hpp | 37 ++++++++++----- 4 files changed, 77 insertions(+), 69 deletions(-) diff --git a/src/interface/container_collection.hpp b/src/interface/container_collection.hpp index 4420547..5669bed 100644 --- a/src/interface/container_collection.hpp +++ b/src/interface/container_collection.hpp @@ -18,6 +18,7 @@ #include #include "interface/container.hpp" +#include "interface/swarm_container.hpp" #include "interface/metadata.hpp" namespace parthenon { @@ -26,11 +27,15 @@ template class ContainerCollection { public: ContainerCollection() { - containers_["base"] = std::make_shared>(); // always add "base" container + // Always add "base" containers + containers_["base"] = std::make_shared>(); + swarm_containers_["base"] = std::make_shared(); } void Add(const std::string &label, Container &src); + void Add(const std::string &label, SwarmContainer &src); + Container &Get() { return *containers_["base"]; } Container &Get(const std::string &label) { auto it = containers_.find(label); @@ -40,6 +45,15 @@ class ContainerCollection { return *(it->second); } + SwarmContainer &GetSwarm() { return *swarm_containers_["base"]; } + SwarmContainer &GetSwarm(const std::string &label) { + auto it = swarm_containers_.find(label); + if (it == swarm_containers_.end()) { + throw std::runtime_error("SwarmContainer " + label + " does not exist in collection."); + } + return *(it->second); + } + void PurgeNonBase() { auto c = containers_.begin(); while (c != containers_.end()) { @@ -49,6 +63,14 @@ class ContainerCollection { ++c; } } + auto sc = swarm_containers_.begin(); + while (sc != swarm_containers_.end()) { + if (sc->first != "base") { + sc = swarm_containers_.erase(sc); + } else { + ++sc; + } + } } void Print() { @@ -57,10 +79,16 @@ class ContainerCollection { c.second->Print(); std::cout << std::endl; } + for (auto &sc : swarm_containers_) { + std::cout << "SwarmContainer " << sc.first << " has:" << std::endl; + sc.second->Print(); + std::cout << std::endl; + } } private: std::map>> containers_; + std::map> swarm_containers_; }; } // namespace parthenon diff --git a/src/interface/swarm.hpp b/src/interface/swarm.hpp index 0dffa3b..bacc0b3 100644 --- a/src/interface/swarm.hpp +++ b/src/interface/swarm.hpp @@ -14,15 +14,8 @@ #define INTERFACE_SWARM_HPP_ /// -/// A Variable type for Placebo-K. -/// Builds on AthenaArrays +/// A swarm contains all particles of a particular species /// Date: August 21, 2019 -/// -/// -/// The variable class typically contains state data for the -/// simulation but can also hold non-mesh-based data such as physics -/// parameters, etc. It inherits the AthenaArray class, which is used -/// for actural data storage and generation #include #include @@ -106,7 +99,7 @@ class Swarm { } void Defrag() { - // TODO(BRR) Put an O(N) algorithm here to defrag memory + // TODO(BRR) Put a fast algorithm here to defrag memory } private: @@ -123,6 +116,10 @@ class Swarm { std::vector>> _stringArray; }; +using SP_Swarm = std::shared_ptr; +using SwarmVector = std::vector; +using SwarmMap = std::map; + } // namespace parthenon #endif // INTERFACE_VARIABLE_HPP_ diff --git a/src/interface/swarm_container.cpp b/src/interface/swarm_container.cpp index 4355642..d058931 100644 --- a/src/interface/swarm_container.cpp +++ b/src/interface/swarm_container.cpp @@ -37,18 +37,22 @@ void SwarmContainer::Add(const std::vector labelArray, /// @param pmetadata the metadata associated with the particle void SwarmContainer::Add(const std::string label, const SwarmMetadata &smetadata) { + if (swarmMap_.find(label) != swarmMap_.end()) { + throw std::invalid_argument ("swarm " + label +" already enrolled during Add()!"); + } + auto swarm = std::make_shared(label, smetadata); - // TODO(BRR) check that swarm isn't already enrolled? - swarms.push_back(swarm); + swarmVector_.push_back(swarm); + swarmMap_[label] = swarm; } void SwarmContainer::Remove(const std::string label) { int idx, isize; // Find index of swarm - isize = swarms.size(); + isize = swarmVector_.size(); idx = 0; - for (auto s : swarms) { + for (auto s : swarmVector_) { if ( ! label.compare(s->label())) { break; } @@ -59,20 +63,22 @@ void SwarmContainer::Remove(const std::string label) { } // first delete the variable - swarms[idx].reset(); + swarmVector_[idx].reset(); // Next move the last element into idx and pop last entry isize--; - if ( isize >= 0) swarms[idx] = std::move(swarms.back()); - swarms.pop_back(); - return; + if ( isize >= 0) swarmVector_[idx] = std::move(swarmVector_.back()); + swarmVector_.pop_back(); + + // Also remove swarm from map + swarmMap_.erase(label); } void SwarmContainer::SendBoundaryBuffers() {} void SwarmContainer::SetupPersistentMPI() {} -bool SwarmContainer::ReceiveBoundaryBuffers() {} +bool SwarmContainer::ReceiveBoundaryBuffers() { return true; } void SwarmContainer::ReceiveAndSetBoundariesWithWait() {} @@ -82,45 +88,9 @@ void SwarmContainer::StartReceiving(BoundaryCommSubset phase) {} void SwarmContainer::ClearBoundary(BoundaryCommSubset phase) {} -void SwarmContainer::print() { +void SwarmContainer::Print() { std::cout << "Swarms are:\n"; - for (auto s : swarms) { std::cout << " " << s->info() << std::endl; } -} - -static void AddSwarm(Swarm&V, std::vector& sRet) { - // adds aliases to sRet - sRet.push_back(Swarm(V)); -} - - -/// Gets an array of real variables from container. -/// @param index_ret is returned with starting index for each name -/// @param count_ret is returned with number of arrays for each name -/// @param sparse_ids if specified, only those sparse IDs are returned -int SwarmContainer::GetVariables(const std::vector& names, - std::vector& sRet, - std::map>& indexCount, - const std::vector& sparse_ids) { - // First count how many entries we need and fill in index and count - indexCount.clear(); - - int index = 0; - for (auto label : names) { - int count = 0; - try { // normal variable - Swarm& S = Get(label); - AddSwarm(S, sRet); - count++; - } - catch (const std::invalid_argument& x) { - throw std::invalid_argument (" Unable to find swarm " + - label + " in container"); - } - indexCount[label] = std::make_pair(index,count); - index += count; - } // (auto label : names) - - return index; + for (auto s : swarmMap_) { std::cout << " " << s.second->info() << std::endl; } } } // namespace parthenon diff --git a/src/interface/swarm_container.hpp b/src/interface/swarm_container.hpp index 7db25d5..61945bd 100644 --- a/src/interface/swarm_container.hpp +++ b/src/interface/swarm_container.hpp @@ -83,7 +83,7 @@ class SwarmContainer { /// @param label the name of the swarm /// @return the Swarm if found or throw exception Swarm& Get(std::string label) { - for (auto v : swarms) { + for (auto v : swarmVector_) { if (! v->label().compare(label)) return *v; } throw std::invalid_argument (std::string("\n") + @@ -92,12 +92,12 @@ class SwarmContainer { } Swarm& Get(const int index) { - return *(swarms[index]); + return *(swarmVector_[index]); } int Index(const std::string& label) { - for (int i = 0; i < swarms.size(); i++) { - if (! swarms[i]->label().compare(label)) return i; + for (int i = 0; i < swarmVector_.size(); i++) { + if (! swarmVector_[i]->label().compare(label)) return i; } return -1; } @@ -111,6 +111,9 @@ class SwarmContainer { std::vector& sRet, std::map>& indexCount); + const SwarmVector &GetSwarmVector() const { return swarmVector_; } + const SwarmMap &GetSwarmMap() const { return swarmMap_; } + /// /// Remove a variable from the container or throw exception if not /// found. @@ -120,14 +123,14 @@ class SwarmContainer { // Temporary functions till we implement a *real* iterator /// Print list of labels in container - void print(); + void Print(); // return number of stored arrays - int size() {return swarms.size();} + int Size() {return swarmVector_.size();} // Element accessor functions std::vector>& allSwarms() { - return swarms; + return swarmVector_; } // Communication routines @@ -139,14 +142,24 @@ class SwarmContainer { void StartReceiving(BoundaryCommSubset phase); void ClearBoundary(BoundaryCommSubset phase); - int GetVariables(const std::vector& names, - std::vector& sRet, - std::map>& indexCount, - const std::vector& sparse_ids); + bool operator==(const SwarmContainer &cmp) { + // Test that labels of swarms are the same + std::vector my_keys; + std::vector cmp_keys; + for (auto &s : swarmMap_) { + my_keys.push_back(s.first); + } + for (auto &s : cmp.GetSwarmMap()) { + cmp_keys.push_back(s.first); + } + return my_keys == cmp_keys; + } private: int debug=0; - std::vector> swarms = {}; + + SwarmVector swarmVector_ = {}; + SwarmMap swarmMap_ = {}; }; } // namespace parthenon From 3547eb5f2a30494850be63c9c7a0720ce879988d Mon Sep 17 00:00:00 2001 From: "Ben R. Ryan" Date: Thu, 16 Apr 2020 15:53:20 -0600 Subject: [PATCH 07/11] Adding example --- CMakeLists.txt | 3 +- example/particles/CMakeLists.txt | 19 ++ example/particles/main.cpp | 55 +++ example/particles/parthinput.advection | 64 ++++ example/particles/particles.cpp | 452 +++++++++++++++++++++++++ example/particles/particles.hpp | 96 ++++++ src/interface/metadata.hpp | 21 +- src/interface/particle_metadata.hpp | 36 -- src/interface/swarm.hpp | 51 ++- src/interface/swarm_container.cpp | 10 +- src/interface/swarm_container.hpp | 7 +- src/interface/swarm_metadata.hpp | 36 -- src/interface/variable.cpp | 18 + src/interface/variable.hpp | 32 ++ 14 files changed, 790 insertions(+), 110 deletions(-) create mode 100644 example/particles/CMakeLists.txt create mode 100644 example/particles/main.cpp create mode 100644 example/particles/parthinput.advection create mode 100644 example/particles/particles.cpp create mode 100644 example/particles/particles.hpp delete mode 100644 src/interface/particle_metadata.hpp delete mode 100644 src/interface/swarm_metadata.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 000798d..96bc953 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -171,11 +171,12 @@ endif() # Note that these options may not play nice with nvcc wrapper if (TEST_INTEL_OPTIMIZATION) add_compile_options(-fp-model fast=2 -qopt_report5 -vec-threshold0 -qopt_report_phase=vec) -endif() +endif() add_subdirectory(src) add_subdirectory(example/calculate_pi) add_subdirectory(example/face_fields) add_subdirectory(example/advection) +add_subdirectory(example/particles) include(cmake/CheckCopyright.cmake) diff --git a/example/particles/CMakeLists.txt b/example/particles/CMakeLists.txt new file mode 100644 index 0000000..7a87429 --- /dev/null +++ b/example/particles/CMakeLists.txt @@ -0,0 +1,19 @@ +#========================================================================================= +# (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. +# +# This program was produced under U.S. Government contract 89233218CNA000001 for Los +# Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +# for the U.S. Department of Energy/National Nuclear Security Administration. All rights +# in the program are reserved by Triad National Security, LLC, and the U.S. Department +# of Energy/National Nuclear Security Administration. The Government is granted for +# itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +# license in this material to reproduce, prepare derivative works, distribute copies to +# the public, perform publicly and display publicly, and to permit others to do so. +#========================================================================================= + +add_executable( + particles-example + main.cpp + particles.cpp +) +target_link_libraries(particles-example PRIVATE parthenon) diff --git a/example/particles/main.cpp b/example/particles/main.cpp new file mode 100644 index 0000000..3ed20e6 --- /dev/null +++ b/example/particles/main.cpp @@ -0,0 +1,55 @@ +//======================================================================================== +// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== + +#include "parthenon_manager.hpp" + +#include "particles.hpp" + +int main(int argc, char *argv[]) { + using parthenon::ParthenonManager; + using parthenon::ParthenonStatus; + ParthenonManager pman; + + // call ParthenonInit to initialize MPI and Kokkos, parse the input deck, and set up + auto manager_status = pman.ParthenonInit(argc, argv); + if (manager_status == ParthenonStatus::complete) { + pman.ParthenonFinalize(); + return 0; + } + if (manager_status == ParthenonStatus::error) { + pman.ParthenonFinalize(); + return 1; + } + // Now that ParthenonInit has been called and setup succeeded, the code can now + // make use of MPI and Kokkos + + // Initialize the driver + advection_example::AdvectionDriver driver(pman.pinput.get(), pman.pmesh.get(), + pman.pouts.get()); + + // start a timer + pman.PreDriver(); + + // This line actually runs the simulation + auto driver_status = driver.Execute(); + + // Make final outputs, print diagnostics + pman.PostDriver(driver_status); + + // call MPI_Finalize and Kokkos::finalize if necessary + pman.ParthenonFinalize(); + + // MPI and Kokkos can no longer be used + + return (0); +} diff --git a/example/particles/parthinput.advection b/example/particles/parthinput.advection new file mode 100644 index 0000000..bcb80f7 --- /dev/null +++ b/example/particles/parthinput.advection @@ -0,0 +1,64 @@ +# ======================================================================================== +# Athena++ astrophysical MHD code +# Copyright(C) 2014 James M. Stone and other code contributors +# Licensed under the 3-clause BSD License, see LICENSE file for details +# ======================================================================================== +# (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. +# +# This program was produced under U.S. Government contract 89233218CNA000001 for Los +# Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +# for the U.S. Department of Energy/National Nuclear Security Administration. All rights +# in the program are reserved by Triad National Security, LLC, and the U.S. Department +# of Energy/National Nuclear Security Administration. The Government is granted for +# itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +# license in this material to reproduce, prepare derivative works, distribute copies to +# the public, perform publicly and display publicly, and to permit others to do so. +# ======================================================================================== + + +problem = Particle streaming + + +problem_id = particles + + +refinement = none +#numlevel = 3 + +nx1 = 64 +x1min = -0.5 +x1max = 0.5 +ix1_bc = periodic +ox1_bc = periodic + +nx2 = 64 +x2min = -0.5 +x2max = 0.5 +ix2_bc = periodic +ox2_bc = periodic + +nx3 = 1 +x3min = -0.5 +x3max = 0.5 + + +nx1 = 16 +nx2 = 16 +#nx3 = 1 + + +file_type = hdf5 +dt = 0.05 +variable = prim + +