Merge pull request #321 from LLNL/release/v0.28

Release/v0.28

CMakeLists.txt

@@ -5,19 +5,19 @@ include(FetchContent)
 # -------------------------------------------------------------------------------- #
 # CMake policy
 # -------------------------------------------------------------------------------- #
-if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13")
+if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13")
  cmake_policy(SET CMP0077 NEW)
-endif()
+endif ()
 
-if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24")
+if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24")
  cmake_policy(SET CMP0135 NEW)
-endif()
+endif ()
 
 # -------------------------------------------------------------------------------- #
 # Metall general configuration
 # -------------------------------------------------------------------------------- #
 project(Metall
- VERSION 0.27
+ VERSION 0.28
  DESCRIPTION "A persistent memory allocator for data-centric analytics"
  HOMEPAGE_URL "https://github.com/LLNL/metall")
 
@@ -92,6 +92,7 @@ set(COMPILER_DEFS "" CACHE STRING "A list of Metall compile definitions to be ad
 
 # ---------- Experimental options ---------- #
 set(UMAP_ROOT "" CACHE PATH "UMap installed root directory")
+set(PRIVATEER_ROOT "" CACHE PATH "Privateer installed root directory")
 
 option(ONLY_DOWNLOAD_GTEST "Only downloading Google Test" OFF)
 option(SKIP_DOWNLOAD_GTEST "Skip downloading Google Test" OFF)
@@ -113,11 +114,11 @@ endif ()
 # -------------------------------------------------------------------------------- #
 if (INSTALL_HEADER_ONLY)
  message(WARNING "INSTALL_HEADER_ONLY option has been replaced with JUST_INSTALL_METALL_HEADER.")
-endif()
+endif ()
 
 if (JUST_INSTALL_METALL_HEADER)
  return()
-endif()
+endif ()
 # -------------------------------------------------------------------------------- #
 
 # -------------------------------------------------------------------------------- #
@@ -138,7 +139,7 @@ endif ()
 # -------------------------------------------------------------------------------- #
 
 # -------------------------------------------------------------------------------- #
-# Executables
+# Set up for building executables
 # -------------------------------------------------------------------------------- #
 
 # Requirements for GCC
@@ -151,9 +152,9 @@ if (NOT RUN_BUILD_AND_TEST_WITH_CI)
 endif ()
 
 # ---------- Metall Macros ---------- #
-foreach(X ${COMPILER_DEFS})
+foreach (X ${COMPILER_DEFS})
  message(STATUS "Metall compile definition: ${X}")
-endforeach()
+endforeach ()
 
 
 # ---------- CMAKE_BUILD_TYPE ---------- #
@@ -168,12 +169,8 @@ find_package(Threads REQUIRED)
 
 
 # ---------- filesystem ---------- #
-include(include_cxx_filesystem_library)
-include_cxx_filesystem_library()
-
-# Xcode 11 Beta Release Notes
-# Clang now supports the C++17 <filesystem> library for iOS 13, macOS 10.15, watchOS 6, and tvOS 13. (50988273)
-# https://developer.apple.com/documentation/xcode_release_notes/xcode_11_beta_release_notes?language=objc
+include(check_cxx_filesystem_library)
+check_cxx_filesystem_library()
 
 
 # ---------- UMap ---------- #
@@ -183,6 +180,11 @@ if (UMAP_ROOT)
  endif ()
 endif ()
 
+# ---------- Privateer ---------- #
+if (PRIVATEER_ROOT)
+ message(STATUS "Privateer Root is: ${PRIVATEER_ROOT}")
+ find_library(LIBPRIVATEER NAMES privateer PATHS ${PRIVATEER_ROOT}/lib)
+endif ()
 
 # ---------- Boost ---------- #
 # Disable the boost-cmake feature (BoostConfig.cmake or boost-config.cmake) since
@@ -192,7 +194,7 @@ set(Boost_NO_BOOST_CMAKE ON)
 find_package(Boost 1.64 QUIET)
 if (NOT Boost_FOUND)
  FetchContent_Declare(Boost
- URL https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.bz2)
+ URL https://boostorg.jfrog.io/artifactory/main/release/1.83.0/source/boost_1_83_0.tar.bz2)
  FetchContent_GetProperties(Boost)
  if (NOT Boost_POPULATED)
  FetchContent_Populate(Boost)
@@ -212,7 +214,7 @@ function(add_common_compile_options name)
  target_compile_options(${name} PRIVATE $<$<CONFIG:Debug>:-Og>)
  target_compile_options(${name} PRIVATE $<$<CONFIG:Debug>:-g3>)
  target_compile_options(${name} PRIVATE $<$<CONFIG:Debug>:-Wextra>)
- if (Linux)
+ if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
  target_compile_options(${name} PRIVATE $<$<CONFIG:Debug>:-pg>)
  endif ()
 
@@ -223,7 +225,7 @@ function(add_common_compile_options name)
  # Release with debug info
  target_compile_options(${name} PRIVATE $<$<CONFIG:RelWithDebInfo>:-Ofast>)
  target_compile_options(${name} PRIVATE $<$<CONFIG:RelWithDebInfo>:-g3>)
- if (Linux)
+ if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
  target_compile_options(${name} PRIVATE $<$<CONFIG:RelWithDebInfo>:-pg>)
  endif ()
 endfunction()
@@ -246,20 +248,18 @@ function(common_setup_for_metall_executable name)
  # --------------------
 
  # ----- Compile Definitions ----- #
- foreach(X ${COMPILER_DEFS})
-  target_compile_definitions(${name} PRIVATE ${X})
- endforeach()
+ foreach (X ${COMPILER_DEFS})
+ target_compile_definitions(${name} PRIVATE ${X})
+ endforeach ()
  # --------------------
 
  # ----- CXX17 Filesystem Lib----- #
- # include_cxx_filesystem_library module must be executed first
- if (FOUND_CXX17_FILESYSTEM_LIB)
- if (REQUIRE_LIB_STDCXX_FS)
+ # GNU compilers prior to 9.1 requires linking with stdc++fs
+ if (("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU"))
+ if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.1)
  target_link_libraries(${name} PRIVATE stdc++fs)
- endif()
- elseif()
- target_compile_definitions(${name} PRIVATE "METALL_DISABLE_CXX17_FILESYSTEM_LIB")
- endif()
+ endif ()
+ endif ()
  # --------------------
 
  # ----- Umap----- #
@@ -271,6 +271,37 @@ function(common_setup_for_metall_executable name)
  endif ()
  endif ()
  # --------------------
+
+ # ----- Privateer----- #
+ if (PRIVATEER_ROOT)
+ target_include_directories(${name} PRIVATE ${PRIVATEER_ROOT}/include)
+ if (LIBPRIVATEER)
+ # 1) Privateer Dependencies
+ FIND_PACKAGE(OpenSSL)
+ if (OpenSSL_FOUND)
+ target_link_libraries(${name} PRIVATE OpenSSL::SSL)
+ target_link_libraries(${name} PRIVATE OpenSSL::Crypto)
+ endif ()
+ target_link_libraries(${name} PRIVATE rt)
+ FIND_PACKAGE(OpenMP REQUIRED)
+ if (OpenMP_CXX_FOUND)
+ target_link_libraries(${name} PRIVATE OpenMP::OpenMP_CXX)
+ else ()
+ message(FATAL_ERROR "OpenMP is required to build Metall with Privateer")
+ endif ()
+ if (ZSTD_ROOT)
+ find_library(LIBZSTD NAMES zstd PATHS ${ZSTD_ROOT}/lib)
+ target_include_directories(${name} PRIVATE ${ZSTD_ROOT}/lib)
+ target_link_libraries(${name} PRIVATE ${LIBZSTD})
+ target_compile_definitions(${name} PRIVATE USE_COMPRESSION)
+ endif ()
+
+ # 2) Link Privateer
+ target_link_libraries(${name} PRIVATE ${LIBPRIVATEER})
+ target_compile_definitions(${name} PRIVATE METALL_USE_PRIVATEER)
+ endif ()
+ endif ()
+ # --------------------
 endfunction()
 
 function(add_metall_executable name source)

bench/bfs/run_bfs_bench_metall.cpp

@@ -29,7 +29,7 @@ int main(int argc, char *argv[]) {
  // metall::logger::set_log_level(metall::logger::level::verbose);
 
  metall::manager manager(metall::open_read_only,
- option.graph_file_name_list[0].c_str());
+ option.graph_file_name_list[0]);
  auto adj_list =
  manager.find<adjacency_list_type>(option.graph_key_name.c_str()).first;
 

bench/bfs/run_bfs_bench_metall_multiple.cpp

@@ -33,7 +33,7 @@ int main(int argc, char *argv[]) {
  std::vector<metall::manager *> managers;
  for (const auto &file_name : option.graph_file_name_list) {
  managers.emplace_back(
- new metall::manager(metall::open_read_only, file_name.c_str()));
+ new metall::manager(metall::open_read_only, file_name));
  }
 
  auto adj_list = adjacency_list_type(option.graph_key_name, managers.begin(),

bench/simple_alloc/run_simple_allocation_bench_metall.cpp

@@ -13,7 +13,7 @@
 int main(int argc, char *argv[]) {
  const auto option = simple_alloc_bench::parse_option(argc, argv);
  {
- metall::manager manager(metall::create_only, option.datastore_path.c_str());
+ metall::manager manager(metall::create_only, option.datastore_path);
  simple_alloc_bench::run_bench(option, manager.get_allocator<std::byte>());
  }
  metall::manager::remove(option.datastore_path.c_str());

cmake/check_cxx_filesystem_library.cmake

@@ -0,0 +1,25 @@
+# Checks if the C++17 <filesystem> library is available.
+function(check_cxx_filesystem_library)
+
+ set(FOUND_CXX17_FILESYSTEM_LIB FALSE PARENT_SCOPE)
+ set(REQUIRE_LIB_STDCXX_FS FALSE PARENT_SCOPE)
+
+ # Check if C++17 <filesystem> header files are available
+ # If not, uses our own implementation.
+ include(CheckIncludeFileCXX)
+ CHECK_INCLUDE_FILE_CXX(filesystem FOUND_FILESYSTEM_HEADER)
+ if (NOT FOUND_FILESYSTEM_HEADER)
+ message(FATAL_ERROR "Cannot find the C++17 <filesystem> library.")
+ endif ()
+
+ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # Clang or AppleClang
+ if (${CMAKE_HOST_SYSTEM_NAME} MATCHES "Darwin") # macOS
+ include(get_macos_version)
+ get_macos_version() # Get macOS version
+ message(VERBOSE "Detected macOS version ${MACOS_VERSION}")
+ if (MACOS_VERSION VERSION_LESS 10.15) # macOS < 10.15
+ message(FATAL_ERROR "macOS >= 10.15 is required to use the C++17 <filesystem> library.")
+ endif ()
+ endif ()
+ endif ()
+endfunction()

cmake/setup_mpi.cmake

@@ -1,4 +1,4 @@
-if (MPI_CXX_FOUND)
+if (NOT MPI_CXX_FOUND)
  find_package(MPI)
 endif ()
 

docs/Doxyfile.in

@@ -38,7 +38,7 @@ PROJECT_NAME = "Metall"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER = v0.27
+PROJECT_NUMBER = v0.28
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

docs/readthedocs/basics/getting_started.md

@@ -69,22 +69,11 @@ g++ -std=c++17 your_program.cpp -lstdc++fs -I${BOOST_ROOT}/include -I${METALL_RO
 
 Clang (or Apple clang) could be used instead of GCC to build Metall.
 However, we haven't tested it intensively.
+To run on macOS, Metall requires macOS >= 10.15.
+
 Also, Boost C++ Libraries 1.69 or more may be required
 if one wants to build Metall with Clang + CUDA.
 
-**On macOS >= 10.15 or Linux**
-
 ```bash
-# Remove "-lstdc++fs" option
 clang++ -std=c++17 [tutorial_program.cpp] -I../../include -I${BOOST_ROOT}
-```
-
-**On macOS < 10.15**
-
-The C++17 <filesystem> library is not available on macOS < 10.15.
-One has to stop using C++17 <filesystem> library in Metall.
-If METALL_DISABLE_CXX17_FILESYSTEM_LIB macro is defined, Metall uses its own file system operation implementation.
-
-```bash
-clang++ -std=c++17 [tutorial_program.cpp] -I../../include -I${BOOST_ROOT} -DMETALL_DISABLE_CXX17_FILESYSTEM_LIB
-```
+```

example/CMakeLists.txt

@@ -14,7 +14,7 @@ add_metall_executable(snapshot snapshot.cpp)
 
 add_metall_executable(csr_graph csr_graph.cpp)
 
-add_metall_executable(fallback_allocator_adaptor fallback_allocator_adaptor.cpp)
+add_metall_executable(fallback_allocator fallback_allocator.cpp)
 
 add_metall_executable(datastore_description datastore_description.cpp)
 
@@ -38,6 +38,9 @@ add_metall_executable(logger logger.cpp)
 
 add_metall_executable(concurrent concurrent.cpp)
 
+add_metall_executable(custom_logger custom_logger.cpp)
+target_compile_definitions(custom_logger PRIVATE METALL_LOGGER_EXTERN_C=1)
+
 if (BUILD_C)
  add_c_executable(c_api c_api.c)
  target_link_libraries(c_api PRIVATE metall_c)

example/README.md

@@ -96,9 +96,9 @@ Therefore, the data structures in the examples below are more like how to use ST
 
 ### Fallback Allocator
 
-* [fallback_allocator_adaptor.cpp](fallback_allocator_adaptor.cpp)
+* [fallback_allocator.cpp](fallback_allocator.cpp)
 
- * How to use [fallback_allocator_adaptor](../include/metall/utility/fallback_allocator_adaptor.hpp)
+ * How to use [fallback_allocator](../include/metall/container/fallback_allocator.hpp)
 
 
 ### MPI (experimental implementation)