adding Adios2 io support

CMakeLists.txt

@@ -29,6 +29,7 @@ bob_option(Omega_h_USE_Kokkos "Use Kokkos as a backend" OFF)
 bob_input(Kokkos_PREFIX "" PATH "Path to Kokkos install")
 bob_option(Omega_h_USE_CUDA_AWARE_MPI "Assume MPI is CUDA-aware, make use of that" OFF)
 bob_option(Omega_h_USE_SimModSuite "Enable reading Simmetrix MeshSim meshes" OFF)
+bob_option(Omega_h_USE_ADIOS2 "Enable ADIOS2" OFF)
 bob_input(Omega_h_VALGRIND "" STRING "Valgrind plus arguments for testing")
 bob_option(Omega_h_EXAMPLES "Compile examples" OFF)
 
@@ -161,6 +162,7 @@ set(Omega_h_KEY_BOOLS
     Omega_h_USE_EGADS
     Omega_h_USE_SEACASExodus
     Omega_h_USE_SimModSuite
+    Omega_h_USE_ADIOS2
     Omega_h_USE_DOLFIN
     Omega_h_USE_dwarf
     Omega_h_CHECK_BOUNDS

cmake/FindADIOS2.cmake

@@ -0,0 +1,104 @@
+# - Try to find ADIOS2
+# Once done this will define
+#  ADIOS2_FOUND - System has ADIOS2
+#  ADIOS2_INCLUDE_DIR - The ADIOS2 include directories
+#  ADIOS2_LIBS - The libraries needed to use ADIOS2 
+#  ADIOS2_<library>_FOUND - System has <library>
+#  ADIOS2_MAJOR_VERSION - the leading integer of the version string
+#  ADIOS2_MINOR_VERSION - the date code from the version string
+#
+# Based on input variables:
+#  ADIOS2_LIB_DIR
+#  ADIOS2_INCLUDE_DIR
+# And environment variable:
+#  CMAKE_PREFIX_PATH
+#
+# This implementation assumes an adios2 install has the following structure
+# VERSION/
+#         include/*.h
+#         lib64/*.a
+
+macro(adios2LibCheck libs isRequired)
+  foreach(lib ${libs})
+    unset(adios2lib CACHE)
+    find_library(adios2lib "${lib}" PATHS ${ADIOS2_LIB_DIR})
+    if(adios2lib MATCHES "^adios2lib-NOTFOUND$")
+      if(${isRequired})
+	      message(FATAL_ERROR "adios2 library ${lib} not found in ${ADIOS2_LIB_DIR}")
+      else()
+	      message("adios2 library ${lib} not found in ${ADIOS2_LIB_DIR}")
+      endif()
+    else()
+	    set("ADIOS2_${lib}_FOUND" TRUE CACHE INTERNAL "ADIOS2 library present")
+      set(ADIOS2_LIBS ${ADIOS2_LIBS} ${adios2lib})
+    endif()
+  endforeach()
+endmacro(adios2LibCheck)
+
+find_path(ADIOS2_INCLUDE_DIR
+  NAMES adios2_c.h  adios2.h
+  PATHS ${ADIOS2_INCLUDE_DIR})
+if(NOT EXISTS "${ADIOS2_INCLUDE_DIR}")
+  message(FATAL_ERROR "adios2 include dir not found")
+endif()
+
+string(REGEX REPLACE
+  "/include$" ""
+  ADIOS2_INSTALL_DIR
+  "${ADIOS2_INCLUDE_DIR}")
+
+string(REGEX MATCH
+  "[0-9]+[.][0-9]+-[0-9]+"
+  ADIOS2_VERSION
+  "${ADIOS2_INCLUDE_DIR}")
+
+#VERSION_LESS and VERSION_GREATER need '.' delimited version strings.
+string(REGEX REPLACE
+  "([0-9]+[.][0-9]+)-([0-9]+)"
+  "\\1.\\2" ADIOS2_DOT_VERSION
+  "${ADIOS2_VERSION}")
+string(REGEX REPLACE
+  "([0-9]+)[.]([0-9]+)-([0-9]+)"
+  "\\1" ADIOS2_MAJOR_VERSION
+  "${ADIOS2_VERSION}")
+string(REGEX REPLACE
+  "([0-9]+)[.]([0-9]+)-([0-9]+)"
+  "\\3" ADIOS2_MINOR_VERSION
+  "${ADIOS2_VERSION}")
+
+set(MIN_VALID_ADIOS2_VERSION 2.10.0)
+set(MAX_VALID_ADIOS2_VERSION 2.10.10)
+#if( ${SKIP_ADIOS2_VERSION_CHECK} )
+#	message(STATUS "Skipping ADIOS2 version check."
+#    " This may result in undefined behavior")
+#elseif( (ADIOS2_DOT_VERSION VERSION_LESS MIN_VALID_ADIOS2_VERSION) OR
+#	(ADIOS2_DOT_VERSION VERSION_GREATER MAX_VALID_ADIOS2_VERSION) )
+#  MESSAGE(FATAL_ERROR 
+#	  "invalid ADIOS2 version: ${ADIOS2_DOT_VERSION}, \
+#    valid versions are ${MIN_VALID_ADIOS2_VERSION} to ${MAX_VALID_ADIOS2_VERSION}")
+#endif()
+message(STATUS "Building with ADIOS2 ${ADIOS2_DOT_VERSION}")
+
+set(ADIOS2_LIBS "")
+
+if (Omega_h_USE_MPI)
+set(ADIOS2_LIB_NAMES
+	adios2_core_mpi
+	adios2_cxx11_mpi
+)
+else()
+set(ADIOS2_LIB_NAMES
+   	adios2_core
+    	adios2_cxx11
+)
+endif()
+
+adios2LibCheck("${ADIOS2_LIB_NAMES}" TRUE)
+
+# handle the QUIETLY and REQUIRED arguments and set ADIOS2_FOUND to TRUE
+# if all listed variables are TRUE
+find_package_handle_standard_args(ADIOS2 DEFAULT_MSG
+  ADIOS2_LIBS ADIOS2_INCLUDE_DIR
+)
+
+mark_as_advanced(ADIOS2_INCLUDE_DIR ADIOS2_LIBS)

src/CMakeLists.txt

@@ -170,6 +170,17 @@ if(Omega_h_USE_SimModSuite)
     "${CMAKE_CURRENT_BINARY_DIR}/Omega_h_simConfig.h")
 endif()
 
+message(STATUS "Omega_h_USE_ADIOS2: ${Omega_h_USE_ADIOS2}")
+if(Omega_h_USE_ADIOS2)
+  list(APPEND Omega_h_SOURCES Omega_h_adios2.cpp)
+  #Is there a better way? I assume CMAKE_MODULE_PATH was purposely not set in
+  # the top level CMakeLists.txt
+  set(OLD_CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH})
+  set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/")
+  find_package(ADIOS2 MODULE REQUIRED)
+  set(CMAKE_MODULE_PATH ${OLD_CMAKE_MODULE_PATH})
+endif()
+
 if(Omega_h_USE_SEACASExodus)
   set(Omega_h_SOURCES ${Omega_h_SOURCES} Omega_h_exodus.cpp)
 endif()
@@ -222,6 +233,16 @@ if(Omega_h_USE_SimModSuite)
   target_link_libraries(omega_h PUBLIC "${SIMMODSUITE_LIBS}")
 endif()
 
+if(Omega_h_USE_ADIOS2)
+  set(ADIOS_REQUIRED_VERSION 2.10)
+  if (Omega_h_USE_MPI)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DADIOS2_USE_MPI")
+    target_include_directories(omega_h PUBLIC "${ADIOS2_INCLUDE_DIR}")
+    target_link_libraries(omega_h PUBLIC "${ADIOS2_LIBS}")
+  else()
+    message("set Omega_h_USE_MPI to use ADIOS2")
+  endif()
+endif()
 
 bob_link_dependency(omega_h PUBLIC SEACASExodus)
 
@@ -609,6 +630,15 @@ if(BUILD_TESTING)
   else()
     test_func(describe_serial 1 ./describe ${CMAKE_SOURCE_DIR}/meshes/box_3d.osh)
   endif()
+
+  if (Omega_h_USE_ADIOS2)
+    osh_add_util(bp2osh)
+    osh_add_util(osh2bp)
+    osh_add_exe(adios2_io)
+    test_basefunc(adios2_io 1 ./adios2_io
+	  ${CMAKE_SOURCE_DIR}/meshes/unitbox_cutTriCube_1k.osh
+	  output.bp)
+  endif()
 endif()
 
 bob_config_header("${CMAKE_CURRENT_BINARY_DIR}/Omega_h_config.h")
@@ -724,6 +754,10 @@ else()
   list(APPEND Omega_h_HEADERS Omega_h_array_default.hpp)
 endif()
 
+if(Omega_h_USE_ADIOS2)
+  list(APPEND Omega_h_HEADERS Omega_h_adios2.hpp)
+endif()
+
 install(FILES ${Omega_h_HEADERS} DESTINATION include)
 
 if (Omega_h_USE_pybind11)

src/adios2_io.cpp

@@ -0,0 +1,156 @@
+/*
+ *
+ * .cpp : adios2 low-level API example to write and read arrays and string
+ *
+ *  Created on: Aug 25, 2024
+ *  Author: Seegyoung Seol [email protected]
+ *
+ */
+#include <Omega_h_timer.hpp>
+#include <Omega_h_file.hpp>
+#include <Omega_h_cmdline.hpp>
+#include "Omega_h_build.hpp" // build_box
+#include "Omega_h_library.hpp" // world
+#include "Omega_h_mesh.hpp"
+#include "Omega_h_inertia.hpp" // Rib
+#include "Omega_h_tag.hpp" //class_ids
+#include "Omega_h_defines.hpp"
+#include "Omega_h_adios2.hpp"
+#include <Omega_h_compare.hpp> // MeshCompareOpts
+#include "Omega_h_array_ops.hpp" // each_eq_to
+#include "Omega_h_element.hpp" // topological_singular_name
+#include "Omega_h_mixedMesh.hpp" // family
+#include <iostream>
+#include <stdexcept>
+#include <iomanip>
+
+#include <adios2.h>
+#if ADIOS2_USE_MPI
+#include <mpi.h>
+#endif
+
+using namespace Omega_h;
+using namespace std;
+
+// printTagInfo and getNumEq copied from describe.cpp
+template <typename T>
+void printTagInfo(Omega_h::Mesh mesh, std::ostringstream& oss, int dim, int tag, std::string type) {
+    auto tagbase = mesh.get_tag(dim, tag);
+    auto array = Omega_h::as<T>(tagbase)->array();
+
+    Omega_h::Real min = get_min(array);
+    Omega_h::Real max = get_max(array);
+
+    oss << std::setw(18) << std::left << tagbase->name().c_str()
+        << std::setw(5) << std::left << dim
+        << std::setw(7) << std::left << type
+        << std::setw(5) << std::left << tagbase->ncomps()
+        << std::setw(10) << std::left << min
+        << std::setw(10) << std::left << max
+        << "\n";
+}
+
+template <typename T>
+int getNumEq(Omega_h::Mesh mesh, std::string tagname, int dim, int value) {
+    auto array = mesh.get_array<T>(dim, tagname);
+    auto each_eq_to = Omega_h::each_eq_to<T>(array, value);
+    return Omega_h::get_sum(each_eq_to);
+}
+
+void print_info(Library* lib, Omega_h::Mesh mesh);
+
+// to check the content of .bp, run /lore/seols/romulus-install/bin/bpls
+// ex. ./bpls mesh.bp
+
+int main(int argc, char *argv[])
+{
+  auto lib = Omega_h::Library(&argc, &argv);
+  auto world = lib.world();
+
+  Omega_h::CmdLine cmdline;
+
+  cmdline.add_arg<std::string>("input.osh");
+  cmdline.add_arg<std::string>("output.bp");
+  if (!cmdline.parse_final(world, &argc, argv)) return -1;
+  auto inpath = cmdline.get<std::string>("input.osh");
+  auto outpath=cmdline.get<std::string>("output.bp");
+
+  Omega_h::Mesh mesh(&lib);
+  Omega_h::binary::read(inpath, world, &mesh);
+  cout<<"\n--- Mesh loaded from \""<<inpath<<"\" ---\n";
+  print_info(&lib, mesh);
+
+//  Omega_h::Mesh mesh = build_box(world, OMEGA_H_SIMPLEX, 1., 1., 0., 2, 2, 0);
+  Omega_h::binary::write("omegah.osh", &mesh);
+  Omega_h::vtk::write_parallel("omegah.vtk", &mesh);
+
+  try
+  {
+    write_adios2(outpath, &mesh);
+    Omega_h::Mesh mesh2 = read_adios2(outpath, &lib);
+    Omega_h::vtk::write_parallel("adios2.vtk", &mesh2);
+
+    cout<<"\n\n--- Mesh loaded back from \""<<outpath<<"\" ---\n";
+    print_info(&lib, mesh2);
+
+    double tol = 1e-6, floor = 0.0;
+    bool allow_superset = false;
+    auto opts = MeshCompareOpts::init(
+                &mesh, VarCompareOpts{VarCompareOpts::RELATIVE, tol, floor});
+    auto res = compare_meshes(&mesh, &mesh2, opts, true);
+    if (res == OMEGA_H_SAME || (allow_superset && res == OMEGA_H_MORE))
+    {
+      cout << "\nSUCCESS! Two meshes (.osh and .bp) are the same\n";
+      return 0;
+    }
+    cout << "\nFAIL! Two meshes (.osh and .bp) are NOT the same\n";
+     return 2;
+   }
+   catch (std::exception &e)
+   {
+     std::cout << "\nERROR: ADIOS2 exception: " << e.what() << "\n";
+#if ADIOS2_USE_MPI
+     MPI_Abort(lib.world()->get_impl(), -1);
+#endif
+   }
+    return 0;
+}
+
+// serial only at the moment
+void print_info(Library* lib, Omega_h::Mesh mesh)
+{
+  auto rank = lib->world()->rank();
+
+  ostringstream oss;
+  // always print two places to the right of the decimal
+  // for floating point types (i.e., imbalance)
+  oss.precision(2);
+  oss << std::fixed;
+
+  if (!rank) 
+  {
+    oss << "\nEntity Type: " << Omega_h::topological_singular_name(mesh.family(), mesh.dim()) << "\n";
+
+    oss << "\nEntity Count and Imbalance: (Dim, #Global, #Local, Max/Ave Imbalance)\n";
+    for (int dim=0; dim <= mesh.dim(); dim++)
+      oss << "(" << dim << ", " 
+  	  << mesh.nglobal_ents(dim) << ", " 
+    	  <<mesh.nents(dim)<< ", " 
+	  << mesh.imbalance(dim) << ")\n";
+
+    oss << "\nTag by Dimension: (Name, Dim, Type, #Components, Min/Max Value)\n";
+    for (int dim=0; dim <= mesh.dim(); dim++)
+      for (int tag=0; tag < mesh.ntags(dim); tag++) {
+        auto tagbase = mesh.get_tag(dim, tag);
+        if (tagbase->type() == OMEGA_H_I8)
+          printTagInfo<Omega_h::I8>(mesh, oss, dim, tag, "I8");
+        if (tagbase->type() == OMEGA_H_I32)
+          printTagInfo<Omega_h::I32>(mesh, oss, dim, tag, "I32");
+        if (tagbase->type() == OMEGA_H_I64)
+     	  printTagInfo<Omega_h::I64>(mesh, oss, dim, tag, "I64");
+        if (tagbase->type() == OMEGA_H_F64)
+          printTagInfo<Omega_h::Real>(mesh, oss, dim, tag, "F64");
+      }
+    std::cout << oss.str();
+  }
+}

src/bp2osh.cpp

@@ -0,0 +1,17 @@
+#include <Omega_h_file.hpp>
+#include <Omega_h_library.hpp>
+#include <Omega_h_mesh.hpp>
+#include <Omega_h_adios2.hpp>
+#include <cstdlib>
+
+int main(int argc, char** argv) {
+  auto lib = Omega_h::Library(&argc, &argv);
+  if( argc != 3) {
+    fprintf(stderr, "Usage: %s inputMesh.bp outputMesh.osh\n", argv[0]);
+    exit(EXIT_FAILURE);
+  }
+  OMEGA_H_CHECK(argc == 3);
+  Omega_h::Mesh mesh = read_adios2(argv[1], &lib);
+  Omega_h::binary::write(argv[2], &mesh);
+  return 0;
+}

src/osh2bp.cpp

@@ -0,0 +1,18 @@
+#include <Omega_h_file.hpp>
+#include <Omega_h_library.hpp>
+#include <Omega_h_mesh.hpp>
+#include <Omega_h_adios2.hpp>
+#include <cstdlib>
+
+int main(int argc, char** argv) 
+{
+  auto lib = Omega_h::Library(&argc, &argv);
+  if( argc != 3 ) {
+    fprintf(stderr, "Usage: %s inputMesh.osh outputMesh.bp\n", argv[0]);
+    exit(EXIT_FAILURE);
+  }
+  OMEGA_H_CHECK(argc == 3);
+  Omega_h::Mesh mesh(&lib);
+  Omega_h::binary::read(argv[1], lib.world(), &mesh);
+  Omega_h::write_adios2(argv[2], &mesh);
+}