Merge pull request #644 from dash-project/feat-dart-mpi-unified

Improvements to DART-MPI
dash-project · May 7, 2019 · fb20148 · fb20148
2 parents 23cb4a7 + 8df3801
commit fb20148
Show file tree

Hide file tree

Showing 21 changed files with 107 additions and 97 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -53,8 +53,6 @@ option(ENABLE_DART_LOGGING
        "Specify whether messages from DART should be logged" off)
 option(ENABLE_ASSERTIONS
        "Specify whether runtime assertions should be checked" off)
-option(ENABLE_UNIFIED_MEMORY_MODEL
-       "Specify whether to assume unified memory model" on)
 option(ENABLE_SHARED_WINDOWS
        "Specify whether shared memory features are enabled" on)
 option(ENABLE_DYNAMIC_WINDOWS
@@ -259,8 +257,6 @@ message(INFO "DART log messages:        (ENABLE_DART_LOGGING)            "
         ${ENABLE_DART_LOGGING})
 message(INFO "Runtime assertions:       (ENABLE_ASSERTIONS)              "
         ${ENABLE_ASSERTIONS})
-message(INFO "Unified RMA memory model: (ENABLE_UNIFIED_MEMORY_MODEL)    "
-        ${ENABLE_UNIFIED_MEMORY_MODEL})
 message(INFO "MPI shared windows:       (ENABLE_SHARED_WINDOWS)          "
         ${ENABLE_SHARED_WINDOWS})
 message(INFO "MPI dynamic windows:      (ENABLE_DYNAMIC_WINDOWS)         "

diff --git a/build.analyze.sh b/build.analyze.sh
@@ -104,7 +104,6 @@ mkdir -p $BUILD_DIR/$REPORT_DIR
                         -DENABLE_ASSERTIONS=ON \
                         \
                         -DENABLE_SHARED_WINDOWS=ON \
-                        -DENABLE_UNIFIED_MEMORY_MODEL=ON \
                         -DENABLE_DYNAMIC_WINDOWS=ON \
                         -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \
                         \

diff --git a/build.cov.sh b/build.cov.sh
@@ -66,7 +66,6 @@ rm -Rf $BUILD_DIR/*
                         \
                         -DENABLE_SHARED_WINDOWS=ON \
                         -DENABLE_DYNAMIC_WINDOWS=ON \
-                        -DENABLE_UNIFIED_MEMORY_MODEL=ON \
                         -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \
                         \
                         -DENABLE_LOGGING=ON \

diff --git a/build.debug.sh b/build.debug.sh
@@ -68,7 +68,6 @@ rm -Rf $BUILD_DIR/*
                         \
                         -DENABLE_SHARED_WINDOWS=ON \
                         -DENABLE_DYNAMIC_WINDOWS=ON \
-                        -DENABLE_UNIFIED_MEMORY_MODEL=ON \
                         -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \
                         \
                         -DENABLE_LOGGING=ON \

diff --git a/build.dev.sh b/build.dev.sh
@@ -68,7 +68,6 @@ rm -Rf $BUILD_DIR/*
                         \
                         -DENABLE_SHARED_WINDOWS=ON \
                         -DENABLE_DYNAMIC_WINDOWS=ON \
-                        -DENABLE_UNIFIED_MEMORY_MODEL=ON \
                         -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \
                         \
                         -DENABLE_LOGGING=ON \

diff --git a/build.mic.sh b/build.mic.sh
@@ -57,7 +57,6 @@ rm -Rf $BUILD_DIR/*
                         \
                         -DENABLE_SHARED_WINDOWS=ON \
                         -DENABLE_DYNAMIC_WINDOWS=ON \
-                        -DENABLE_UNIFIED_MEMORY_MODEL=ON \
                         -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \
                         \
                         -DENABLE_LOGGING=OFF \

diff --git a/build.minimal.sh b/build.minimal.sh
@@ -60,7 +60,6 @@ rm -Rf $BUILD_DIR/*
                         \
                         -DENABLE_SHARED_WINDOWS=OFF \
                         -DENABLE_DYNAMIC_WINDOWS=OFF \
-                        -DENABLE_UNIFIED_MEMORY_MODEL=ON \
                         -DENABLE_DEFAULT_INDEX_TYPE_LONG=OFF \
                         \
                         -DENABLE_LOGGING=OFF \

diff --git a/build.nasty.sh b/build.nasty.sh
@@ -73,7 +73,6 @@ rm -Rf $BUILD_DIR/*
                         \
                         -DENABLE_SHARED_WINDOWS=OFF \
                         -DENABLE_DYNAMIC_WINDOWS=OFF \
-                        -DENABLE_UNIFIED_MEMORY_MODEL=ON \
                         -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \
                         \
                         -DENABLE_LOGGING=OFF \

diff --git a/build.sh b/build.sh
@@ -68,7 +68,6 @@ rm -Rf $BUILD_DIR/*
                         \
                         -DENABLE_SHARED_WINDOWS=ON \
                         -DENABLE_DYNAMIC_WINDOWS=ON \
-                        -DENABLE_UNIFIED_MEMORY_MODEL=ON \
                         -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \
                         \
                         -DENABLE_LOGGING=OFF \

diff --git a/dart-impl/base/CMakeLists.txt b/dart-impl/base/CMakeLists.txt
@@ -11,8 +11,6 @@ set(DASH_DART_IF_INCLUDE_DIR ${DASH_DART_IF_INCLUDE_DIR}
     PARENT_SCOPE)
 set(ENABLE_DART_LOGGING ${ENABLE_DART_LOGGING}
     PARENT_SCOPE)
-set(ENABLE_UNIFIED_MEMORY_MODEL ${ENABLE_UNIFIED_MEMORY_MODEL}
-    PARENT_SCOPE)
 set(ENABLE_SHARED_WINDOWS ${ENABLE_SHARED_WINDOWS}
     PARENT_SCOPE)
 set(ENABLE_DEFAULT_INDEX_TYPE_LONG ${ENABLE_DEFAULT_INDEX_TYPE_LONG}
@@ -39,11 +37,6 @@ if (ENABLE_DART_LOGGING)
        "${ADDITIONAL_COMPILE_FLAGS} -DDART_ENABLE_LOGGING")
 endif()
 
-if (ENABLE_UNIFIED_MEMORY_MODEL)
-  set (ADDITIONAL_COMPILE_FLAGS
-       "${ADDITIONAL_COMPILE_FLAGS} -DDART_MPI_ENABLE_UNIFIED_MEMORY_MODEL")
-endif()
-
 # Features compile flags
 #
 if (PAPI_FOUND AND ENABLE_PAPI)

diff --git a/dart-impl/base/src/logging.c b/dart-impl/base/src/logging.c
@@ -1,6 +1,7 @@
 #include <stdarg.h>
 #include <stdio.h>
 #include <string.h>
+#include <strings.h>
 #include <sys/types.h>
 #include <unistd.h>
 #include <time.h>
@@ -67,15 +68,15 @@ env_loglevel()
   if (!log_level_parsed) {
     const char *envstr = getenv(DART_LOGLEVEL_ENVSTR);
     if (envstr) {
-      if (strcmp(envstr, "ERROR") == 0) {
+      if (strcasecmp(envstr, "ERROR") == 0) {
         level = DART_LOGLEVEL_ERROR;
-      } else if (strcmp(envstr, "WARN") == 0) {
+      } else if (strcasecmp(envstr, "WARN") == 0) {
         level = DART_LOGLEVEL_WARN;
-      } else if (strcmp(envstr, "INFO") == 0) {
+      } else if (strcasecmp(envstr, "INFO") == 0) {
         level = DART_LOGLEVEL_INFO;
-      } else if (strcmp(envstr, "DEBUG") == 0) {
+      } else if (strcasecmp(envstr, "DEBUG") == 0) {
         level = DART_LOGLEVEL_DEBUG;
-      } else if (strcmp(envstr, "TRACE") == 0) {
+      } else if (strcasecmp(envstr, "TRACE") == 0) {
         level = DART_LOGLEVEL_TRACE;
       }
     }

diff --git a/dart-impl/mpi/CMakeLists.txt b/dart-impl/mpi/CMakeLists.txt
@@ -21,8 +21,6 @@ set(DASH_DART_IF_INCLUDE_DIR ${DASH_DART_IF_INCLUDE_DIR}
     PARENT_SCOPE)
 set(ENABLE_DART_LOGGING ${ENABLE_DART_LOGGING}
     PARENT_SCOPE)
-set(ENABLE_UNIFIED_MEMORY_MODEL ${ENABLE_UNIFIED_MEMORY_MODEL}
-    PARENT_SCOPE)
 set(ENABLE_SHARED_WINDOWS ${ENABLE_SHARED_WINDOWS}
     PARENT_SCOPE)
 set(ENABLE_DYNAMIC_WINDOWS ${ENABLE_DYNAMIC_WINDOWS}
@@ -87,11 +85,6 @@ if (ENABLE_DART_LOGGING)
        ${ADDITIONAL_COMPILE_FLAGS} -DDART_ENABLE_LOGGING)
 endif()
 
-if (ENABLE_UNIFIED_MEMORY_MODEL)
-  set (ADDITIONAL_COMPILE_FLAGS
-       ${ADDITIONAL_COMPILE_FLAGS} -DDART_MPI_ENABLE_UNIFIED_MEMORY_MODEL)
-endif()
-
 # Features compile flags
 #
 if (PAPI_FOUND AND ENABLE_PAPI)
@@ -144,7 +137,7 @@ if (NOT ENABLE_SHARED_WINDOWS)
   set (ADDITIONAL_COMPILE_FLAGS
        ${ADDITIONAL_COMPILE_FLAGS} -DDART_MPI_DISABLE_SHARED_WINDOWS)
 elseif (NOT ENABLE_DYNAMIC_WINDOWS)
-  message(WARNING 
+  message(WARNING
           "ENABLE_SHARED_WINDOWS enables dynamic windows "
           "(ENABLE_DYNAMIC_WINDOWS)")
   set (ENABLE_DYNAMIC_WINDOWS TRUE)

diff --git a/dart-impl/mpi/include/dash/dart/mpi/dart_globmem_priv.h b/dart-impl/mpi/include/dash/dart/mpi/dart_globmem_priv.h
@@ -7,4 +7,10 @@
 /* Global object for one-sided communication on memory region allocated with 'local allocation'. */
 extern MPI_Win dart_win_local_alloc DART_INTERNAL;
 
+/**
+ * Check that the window support MPI_WIN_UNIFIED, print warning otherwise.
+ * Store the result in \c segment->sync_needed.
+ */
+void dart__mpi__check_memory_model(dart_segment_info_t *segment) DART_INTERNAL;
+
 #endif /* DART__MPI__DART_GLOBMEM_PRIV_H__ */
diff --git a/dart-impl/mpi/include/dash/dart/mpi/dart_segment.h b/dart-impl/mpi/include/dash/dart/mpi/dart_segment.h
@@ -28,6 +28,7 @@ typedef struct
   uint16_t     flags;       /* 16 bit flags */
   dart_segid_t segid;       /* ID of the segment, globally unique in a team */
   bool         is_dynamic;  /* whether this is a shared memory segment */
+  bool         sync_needed; /* whether a call to MPI_WIN_SYNC is needed */
 } dart_segment_info_t;
 
 // forward declaration to make the compiler happy

diff --git a/dart-impl/mpi/src/Makefile b/dart-impl/mpi/src/Makefile
@@ -7,7 +7,6 @@ CFLAGS+=-I.. -I../$(DART_IF)/include -I../include -I../../base/include -L/usr/li
 #CFLAGS+=-DDART_ENABLE_HWLOC
 CFLAGS+=-DDART_MPI_DISABLE_SHARED_WINDOWS
 CFLAGS+=-DDART_MPI_ENABLE_DYNAMIC_WINDOWS
-#CGLAGS+=-DDART_MPI_ENABLE_UNIFIED_MEMORY_MODEL 
 #CFLAGS+=-DDART_DEBUG
 #OPT_FLAGS=-O3
 

diff --git a/dart-impl/mpi/src/dart_communication.c b/dart-impl/mpi/src/dart_communication.c
@@ -70,8 +70,13 @@
   } while (0)
 
 #define CHECK_TYPE_CONSTRAINTS(_src_type, _dst_type, _num_elem)               \
-  CHECK_EQUAL_BASETYPE(_src_type, _dst_type);                                 \
-  CHECK_NUM_ELEM(_src_type, _dst_type, _num_elem);
+  do {                                                                        \
+    CHECK_EQUAL_BASETYPE(_src_type, _dst_type);                               \
+    if (!dart__mpi__datatype_iscontiguous(_src_type) ||                       \
+        !dart__mpi__datatype_iscontiguous(_dst_type)) {                       \
+      CHECK_NUM_ELEM(_src_type, _dst_type, _num_elem);                        \
+    }                                                                         \
+  } while (0)
 
 /**
  * Temporary space allocation:
@@ -294,8 +299,6 @@ dart__mpi__get_complex(
 {
   if (num_reqs != NULL) *num_reqs = 0;
 
-  CHECK_TYPE_CONSTRAINTS(src_type, dst_type, nelem);
-
   MPI_Win win     = seginfo->win;
   char * dest_ptr = (char*) dest;
   offset         += dart_segment_disp(seginfo, team_unit_id);
@@ -418,6 +421,7 @@ dart__mpi__put_basic(
   return DART_OK;
 }
 
+/* slow path for puts of derived types */
 static inline
   dart_ret_t
 dart__mpi__put_complex(
@@ -435,9 +439,6 @@ dart__mpi__put_complex(
   if (flush_required_ptr) *flush_required_ptr = true;
   if (num_reqs) *num_reqs = 0;
 
-  // slow path for derived types
-  CHECK_TYPE_CONSTRAINTS(src_type, dst_type, nelem);
-
   MPI_Win win            = seginfo->win;
   const char * src_ptr   = (const char*) src;
   offset                += dart_segment_disp(seginfo, team_unit_id);
@@ -496,6 +497,8 @@ dart_ret_t dart_get(
   dart_team_unit_t team_unit_id = DART_TEAM_UNIT_ID(gptr.unitid);
   dart_team_t      teamid       = gptr.teamid;
 
+  CHECK_TYPE_CONSTRAINTS(src_type, dst_type, nelem);
+
   dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid);
   if (dart__unlikely(team_data == NULL)) {
     DART_LOG_ERROR("dart_get ! failed: Unknown team %i!", teamid);
@@ -520,7 +523,6 @@ dart_ret_t dart_get(
   if (dart__mpi__datatype_iscontiguous(src_type) &&
       dart__mpi__datatype_iscontiguous(dst_type)) {
     // fast-path for basic types
-    CHECK_EQUAL_BASETYPE(src_type, dst_type);
     ret = dart__mpi__get_basic(team_data, team_unit_id, seginfo, dest,
         offset, nelem, src_type, NULL, NULL);
   } else {
@@ -545,7 +547,7 @@ dart_ret_t dart_put(
   dart_team_unit_t team_unit_id = DART_TEAM_UNIT_ID(gptr.unitid);
   dart_team_t      teamid       = gptr.teamid;
 
-  CHECK_EQUAL_BASETYPE(src_type, dst_type);
+  CHECK_TYPE_CONSTRAINTS(src_type, dst_type, nelem);
 
   dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid);
   if (dart__unlikely(team_data == NULL)) {
@@ -568,7 +570,6 @@ dart_ret_t dart_put(
   if (dart__mpi__datatype_iscontiguous(src_type) &&
       dart__mpi__datatype_iscontiguous(dst_type)) {
     // fast path for basic data types
-    CHECK_EQUAL_BASETYPE(src_type, dst_type);
     ret = dart__mpi__put_basic(team_data, team_unit_id, seginfo, src,
         offset, nelem, src_type,
         NULL, NULL, NULL);
@@ -906,6 +907,8 @@ dart_ret_t dart_get_handle(
 
   *handleptr = DART_HANDLE_NULL;
 
+  CHECK_TYPE_CONSTRAINTS(src_type, dst_type, nelem);
+
   dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid);
   if (dart__unlikely(team_data == NULL)) {
     DART_LOG_ERROR("dart_get_handle ! failed: Unknown team %i!", teamid);
@@ -940,7 +943,6 @@ dart_ret_t dart_get_handle(
   if (dart__mpi__datatype_iscontiguous(src_type) &&
       dart__mpi__datatype_iscontiguous(dst_type)) {
     // fast-path for basic types
-    CHECK_EQUAL_BASETYPE(src_type, dst_type);
     ret = dart__mpi__get_basic(team_data, team_unit_id, seginfo, dest,
         offset, nelem, src_type,
         handle->reqs, &handle->num_reqs);
@@ -978,7 +980,7 @@ dart_ret_t dart_put_handle(
 
   *handleptr = DART_HANDLE_NULL;
 
-  CHECK_EQUAL_BASETYPE(src_type, dst_type);
+  CHECK_TYPE_CONSTRAINTS(src_type, dst_type, nelem);
 
   dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid);
   if (dart__unlikely(team_data == NULL)) {
@@ -1009,7 +1011,6 @@ dart_ret_t dart_put_handle(
   if (dart__mpi__datatype_iscontiguous(src_type) &&
       dart__mpi__datatype_iscontiguous(dst_type)) {
     // fast path for basic data types
-    CHECK_EQUAL_BASETYPE(src_type, dst_type);
     ret = dart__mpi__put_basic(team_data, team_unit_id, seginfo, src,
                                offset, nelem, src_type,
                                handle->reqs,
@@ -1054,7 +1055,7 @@ dart_ret_t dart_put_blocking(
   int16_t           seg_id       = gptr.segid;
   dart_team_t       teamid       = gptr.teamid;
 
-  CHECK_EQUAL_BASETYPE(src_type, dst_type);
+  CHECK_TYPE_CONSTRAINTS(src_type, dst_type, nelem);
 
   dart_team_data_t *team_data = dart_adapt_teamlist_get(gptr.teamid);
   if (dart__unlikely(team_data == NULL)) {
@@ -1083,7 +1084,6 @@ dart_ret_t dart_put_blocking(
   if (dart__mpi__datatype_iscontiguous(src_type) &&
       dart__mpi__datatype_iscontiguous(dst_type)) {
     // fast path for basic data types
-    CHECK_EQUAL_BASETYPE(src_type, dst_type);
     ret = dart__mpi__put_basic(team_data, team_unit_id, seginfo, src,
                                offset, nelem, src_type,
                                NULL, NULL, &needs_flush);
@@ -1118,7 +1118,7 @@ dart_ret_t dart_get_blocking(
   int16_t           seg_id       = gptr.segid;
   dart_team_t       teamid       = gptr.teamid;
 
-  CHECK_EQUAL_BASETYPE(src_type, dst_type);
+  CHECK_TYPE_CONSTRAINTS(src_type, dst_type, nelem);
 
   dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid);
   if (dart__unlikely(team_data == NULL)) {
@@ -1150,7 +1150,6 @@ dart_ret_t dart_get_blocking(
   if (dart__mpi__datatype_iscontiguous(src_type) &&
       dart__mpi__datatype_iscontiguous(dst_type)) {
     // fast-path for basic types
-    CHECK_EQUAL_BASETYPE(src_type, dst_type);
     ret = dart__mpi__get_basic(team_data, team_unit_id, seginfo, dest,
                                offset, nelem, src_type,
                                reqs, &num_reqs);
@@ -1205,9 +1204,12 @@ dart_ret_t dart_flush(
   DART_LOG_TRACE("dart_flush: MPI_Win_flush");
   CHECK_MPI_RET(
     MPI_Win_flush(team_unit_id.id, win), "MPI_Win_flush");
-  DART_LOG_TRACE("dart_flush: MPI_Win_sync");
-  CHECK_MPI_RET(
-    MPI_Win_sync(win), "MPI_Win_sync");
+
+  if (seginfo->sync_needed) {
+    DART_LOG_TRACE("dart_flush: MPI_Win_sync");
+    CHECK_MPI_RET(
+      MPI_Win_sync(win), "MPI_Win_sync");
+  }
 
   // trigger progress
   int flag;
@@ -1250,9 +1252,12 @@ dart_ret_t dart_flush_all(
   DART_LOG_TRACE("dart_flush_all: MPI_Win_flush_all");
   CHECK_MPI_RET(
     MPI_Win_flush_all(win), "MPI_Win_flush");
-  DART_LOG_TRACE("dart_flush_all: MPI_Win_sync");
-  CHECK_MPI_RET(
-    MPI_Win_sync(win), "MPI_Win_sync");
+
+  if (seginfo->sync_needed) {
+    DART_LOG_TRACE("dart_flush_all: MPI_Win_sync");
+    CHECK_MPI_RET(
+      MPI_Win_sync(win), "MPI_Win_sync");
+  }
 
   // trigger progress
   int flag;