Merge remote-tracking branch 'origin/develop' into feature/stdfs

# Conflicts: # CMakeLists.txt # include/metall/basic_manager.hpp # include/metall/kernel/manager_kernel.hpp # include/metall/kernel/manager_kernel_impl.ipp # test/kernel/multimanager_test.cpp
LLNL · Jan 16, 2024 · a698e98 · a698e98
2 parents f24face + 3a0ae6f
commit a698e98
Show file tree

Hide file tree

Showing 23 changed files with 2,040 additions and 1,205 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -92,6 +92,7 @@ set(COMPILER_DEFS "" CACHE STRING "A list of Metall compile definitions to be ad
 
 # ---------- Experimental options ---------- #
 set(UMAP_ROOT "" CACHE PATH "UMap installed root directory")
+set(PRIVATEER_ROOT "" CACHE PATH "Privateer installed root directory")
 
 option(ONLY_DOWNLOAD_GTEST "Only downloading Google Test" OFF)
 option(SKIP_DOWNLOAD_GTEST "Skip downloading Google Test" OFF)
@@ -179,6 +180,11 @@ if (UMAP_ROOT)
  endif ()
 endif ()
 
+# ---------- Privateer ---------- #
+if (PRIVATEER_ROOT)
+ message(STATUS "Privateer Root is: ${PRIVATEER_ROOT}")
+ find_library(LIBPRIVATEER NAMES privateer PATHS ${PRIVATEER_ROOT}/lib)
+endif ()
 
 # ---------- Boost ---------- #
 # Disable the boost-cmake feature (BoostConfig.cmake or boost-config.cmake) since
@@ -265,6 +271,37 @@ function(common_setup_for_metall_executable name)
  endif ()
  endif ()
  # --------------------
+
+ # ----- Privateer----- #
+ if (PRIVATEER_ROOT)
+ target_include_directories(${name} PRIVATE ${PRIVATEER_ROOT}/include)
+ if (LIBPRIVATEER)
+ # 1) Privateer Dependencies
+ FIND_PACKAGE(OpenSSL)
+ if (OpenSSL_FOUND)
+ target_link_libraries(${name} PRIVATE OpenSSL::SSL)
+ target_link_libraries(${name} PRIVATE OpenSSL::Crypto)
+ endif ()
+ target_link_libraries(${name} PRIVATE rt)
+ FIND_PACKAGE(OpenMP REQUIRED)
+ if (OpenMP_CXX_FOUND)
+ target_link_libraries(${name} PRIVATE OpenMP::OpenMP_CXX)
+ else ()
+ message(FATAL_ERROR "OpenMP is required to build Metall with Privateer")
+ endif ()
+ if (ZSTD_ROOT)
+ find_library(LIBZSTD NAMES zstd PATHS ${ZSTD_ROOT}/lib)
+ target_include_directories(${name} PRIVATE ${ZSTD_ROOT}/lib)
+ target_link_libraries(${name} PRIVATE ${LIBZSTD})
+ target_compile_definitions(${name} PRIVATE USE_COMPRESSION)
+ endif ()
+
+ # 2) Link Privateer
+ target_link_libraries(${name} PRIVATE ${LIBPRIVATEER})
+ target_compile_definitions(${name} PRIVATE METALL_USE_PRIVATEER)
+ endif ()
+ endif ()
+ # --------------------
 endfunction()
 
 function(add_metall_executable name source)

diff --git a/include/metall/basic_manager.hpp b/include/metall/basic_manager.hpp
diff --git a/include/metall/defs.hpp b/include/metall/defs.hpp
@@ -49,7 +49,7 @@
 #endif
 
 // --------------------
-// Macros for the default segment storage manager
+// Macros for the default segment storage
 // --------------------
 
 /// \def METALL_SEGMENT_BLOCK_SIZE

diff --git a/include/metall/detail/file.hpp b/include/metall/detail/file.hpp
@@ -179,7 +179,9 @@ inline bool create_file(const fs::path &file_path) {
  const int fd =
  ::open(file_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
  if (fd == -1) {
- logger::perror(logger::level::error, __FILE__, __LINE__, "open");
+ std::stringstream ss;
+ ss << "Failed to create: " << file_path;
+ logger::perror(logger::level::error, __FILE__, __LINE__, ss.str().c_str());
  return false;
  }
 
@@ -456,8 +458,10 @@ inline bool copy_files_in_directory_in_parallel_helper(
  const std::function<bool(const fs::path &, const fs::path &)> &copy_func) {
  std::vector<fs::path> src_file_names;
  if (!get_regular_file_names(source_dir_path, &src_file_names)) {
+ std::stringstream ss;
+ ss << "Failed to get file list in " << source_dir_path;
  logger::out(logger::level::error, __FILE__, __LINE__,
- "Failed to get file list");
+ ss.str().c_str());
  return false;
  }
 

diff --git a/include/metall/detail/mmap.hpp b/include/metall/detail/mmap.hpp
@@ -250,8 +250,9 @@ inline bool munmap(const int fd, void *const addr, const size_t length,
 }
 
 inline bool map_with_prot_none(void *const addr, const size_t length) {
- return (os_mmap(addr, length, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1,
- 0) == addr);
+ auto flag = MAP_PRIVATE | MAP_ANONYMOUS;
+ if (addr != nullptr) flag |= MAP_FIXED;
+ return (os_mmap(addr, length, PROT_NONE, flag, -1, 0) == addr);
 }
 
 inline bool os_mprotect(void *const addr, const size_t length, const int prot) {