diff --git a/CMakeLists.txt b/CMakeLists.txt index bc53c27be..7434b4f29 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,8 +53,6 @@ option(ENABLE_DART_LOGGING "Specify whether messages from DART should be logged" off) option(ENABLE_ASSERTIONS "Specify whether runtime assertions should be checked" off) -option(ENABLE_UNIFIED_MEMORY_MODEL - "Specify whether to assume unified memory model" on) option(ENABLE_SHARED_WINDOWS "Specify whether shared memory features are enabled" on) option(ENABLE_DYNAMIC_WINDOWS @@ -259,8 +257,6 @@ message(INFO "DART log messages: (ENABLE_DART_LOGGING) " ${ENABLE_DART_LOGGING}) message(INFO "Runtime assertions: (ENABLE_ASSERTIONS) " ${ENABLE_ASSERTIONS}) -message(INFO "Unified RMA memory model: (ENABLE_UNIFIED_MEMORY_MODEL) " - ${ENABLE_UNIFIED_MEMORY_MODEL}) message(INFO "MPI shared windows: (ENABLE_SHARED_WINDOWS) " ${ENABLE_SHARED_WINDOWS}) message(INFO "MPI dynamic windows: (ENABLE_DYNAMIC_WINDOWS) " diff --git a/build.analyze.sh b/build.analyze.sh index 87e7ccfab..5e4e02603 100755 --- a/build.analyze.sh +++ b/build.analyze.sh @@ -104,7 +104,6 @@ mkdir -p $BUILD_DIR/$REPORT_DIR -DENABLE_ASSERTIONS=ON \ \ -DENABLE_SHARED_WINDOWS=ON \ - -DENABLE_UNIFIED_MEMORY_MODEL=ON \ -DENABLE_DYNAMIC_WINDOWS=ON \ -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \ \ diff --git a/build.cov.sh b/build.cov.sh index b24cca288..03a2c4e6f 100755 --- a/build.cov.sh +++ b/build.cov.sh @@ -66,7 +66,6 @@ rm -Rf $BUILD_DIR/* \ -DENABLE_SHARED_WINDOWS=ON \ -DENABLE_DYNAMIC_WINDOWS=ON \ - -DENABLE_UNIFIED_MEMORY_MODEL=ON \ -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \ \ -DENABLE_LOGGING=ON \ diff --git a/build.debug.sh b/build.debug.sh index b386da905..8f7320cb1 100755 --- a/build.debug.sh +++ b/build.debug.sh @@ -68,7 +68,6 @@ rm -Rf $BUILD_DIR/* \ -DENABLE_SHARED_WINDOWS=ON \ -DENABLE_DYNAMIC_WINDOWS=ON \ - -DENABLE_UNIFIED_MEMORY_MODEL=ON \ -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \ \ -DENABLE_LOGGING=ON \ diff --git a/build.dev.sh b/build.dev.sh index 2ea323c3c..88fb05269 100755 --- a/build.dev.sh +++ b/build.dev.sh @@ -68,7 +68,6 @@ rm -Rf $BUILD_DIR/* \ -DENABLE_SHARED_WINDOWS=ON \ -DENABLE_DYNAMIC_WINDOWS=ON \ - -DENABLE_UNIFIED_MEMORY_MODEL=ON \ -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \ \ -DENABLE_LOGGING=ON \ diff --git a/build.mic.sh b/build.mic.sh index 5f6770a0a..50b3fade2 100755 --- a/build.mic.sh +++ b/build.mic.sh @@ -57,7 +57,6 @@ rm -Rf $BUILD_DIR/* \ -DENABLE_SHARED_WINDOWS=ON \ -DENABLE_DYNAMIC_WINDOWS=ON \ - -DENABLE_UNIFIED_MEMORY_MODEL=ON \ -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \ \ -DENABLE_LOGGING=OFF \ diff --git a/build.minimal.sh b/build.minimal.sh index 938fd4f06..3d3ec2408 100755 --- a/build.minimal.sh +++ b/build.minimal.sh @@ -60,7 +60,6 @@ rm -Rf $BUILD_DIR/* \ -DENABLE_SHARED_WINDOWS=OFF \ -DENABLE_DYNAMIC_WINDOWS=OFF \ - -DENABLE_UNIFIED_MEMORY_MODEL=ON \ -DENABLE_DEFAULT_INDEX_TYPE_LONG=OFF \ \ -DENABLE_LOGGING=OFF \ diff --git a/build.nasty.sh b/build.nasty.sh index 43236c6b5..8c817b780 100755 --- a/build.nasty.sh +++ b/build.nasty.sh @@ -73,7 +73,6 @@ rm -Rf $BUILD_DIR/* \ -DENABLE_SHARED_WINDOWS=OFF \ -DENABLE_DYNAMIC_WINDOWS=OFF \ - -DENABLE_UNIFIED_MEMORY_MODEL=ON \ -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \ \ -DENABLE_LOGGING=OFF \ diff --git a/build.sh b/build.sh index 9d15ad156..bc44555ca 100755 --- a/build.sh +++ b/build.sh @@ -68,7 +68,6 @@ rm -Rf $BUILD_DIR/* \ -DENABLE_SHARED_WINDOWS=ON \ -DENABLE_DYNAMIC_WINDOWS=ON \ - -DENABLE_UNIFIED_MEMORY_MODEL=ON \ -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \ \ -DENABLE_LOGGING=OFF \ diff --git a/dart-impl/base/CMakeLists.txt b/dart-impl/base/CMakeLists.txt index 9166ee745..36ca80de4 100644 --- a/dart-impl/base/CMakeLists.txt +++ b/dart-impl/base/CMakeLists.txt @@ -11,8 +11,6 @@ set(DASH_DART_IF_INCLUDE_DIR ${DASH_DART_IF_INCLUDE_DIR} PARENT_SCOPE) set(ENABLE_DART_LOGGING ${ENABLE_DART_LOGGING} PARENT_SCOPE) -set(ENABLE_UNIFIED_MEMORY_MODEL ${ENABLE_UNIFIED_MEMORY_MODEL} - PARENT_SCOPE) set(ENABLE_SHARED_WINDOWS ${ENABLE_SHARED_WINDOWS} PARENT_SCOPE) set(ENABLE_DEFAULT_INDEX_TYPE_LONG ${ENABLE_DEFAULT_INDEX_TYPE_LONG} @@ -39,11 +37,6 @@ if (ENABLE_DART_LOGGING) "${ADDITIONAL_COMPILE_FLAGS} -DDART_ENABLE_LOGGING") endif() -if (ENABLE_UNIFIED_MEMORY_MODEL) - set (ADDITIONAL_COMPILE_FLAGS - "${ADDITIONAL_COMPILE_FLAGS} -DDART_MPI_ENABLE_UNIFIED_MEMORY_MODEL") -endif() - # Features compile flags # if (PAPI_FOUND AND ENABLE_PAPI) diff --git a/dart-impl/base/src/logging.c b/dart-impl/base/src/logging.c index 847d81643..48452b84b 100644 --- a/dart-impl/base/src/logging.c +++ b/dart-impl/base/src/logging.c @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -67,15 +68,15 @@ env_loglevel() if (!log_level_parsed) { const char *envstr = getenv(DART_LOGLEVEL_ENVSTR); if (envstr) { - if (strcmp(envstr, "ERROR") == 0) { + if (strcasecmp(envstr, "ERROR") == 0) { level = DART_LOGLEVEL_ERROR; - } else if (strcmp(envstr, "WARN") == 0) { + } else if (strcasecmp(envstr, "WARN") == 0) { level = DART_LOGLEVEL_WARN; - } else if (strcmp(envstr, "INFO") == 0) { + } else if (strcasecmp(envstr, "INFO") == 0) { level = DART_LOGLEVEL_INFO; - } else if (strcmp(envstr, "DEBUG") == 0) { + } else if (strcasecmp(envstr, "DEBUG") == 0) { level = DART_LOGLEVEL_DEBUG; - } else if (strcmp(envstr, "TRACE") == 0) { + } else if (strcasecmp(envstr, "TRACE") == 0) { level = DART_LOGLEVEL_TRACE; } } diff --git a/dart-impl/mpi/CMakeLists.txt b/dart-impl/mpi/CMakeLists.txt index a4d3db993..767807d21 100644 --- a/dart-impl/mpi/CMakeLists.txt +++ b/dart-impl/mpi/CMakeLists.txt @@ -21,8 +21,6 @@ set(DASH_DART_IF_INCLUDE_DIR ${DASH_DART_IF_INCLUDE_DIR} PARENT_SCOPE) set(ENABLE_DART_LOGGING ${ENABLE_DART_LOGGING} PARENT_SCOPE) -set(ENABLE_UNIFIED_MEMORY_MODEL ${ENABLE_UNIFIED_MEMORY_MODEL} - PARENT_SCOPE) set(ENABLE_SHARED_WINDOWS ${ENABLE_SHARED_WINDOWS} PARENT_SCOPE) set(ENABLE_DYNAMIC_WINDOWS ${ENABLE_DYNAMIC_WINDOWS} @@ -87,11 +85,6 @@ if (ENABLE_DART_LOGGING) ${ADDITIONAL_COMPILE_FLAGS} -DDART_ENABLE_LOGGING) endif() -if (ENABLE_UNIFIED_MEMORY_MODEL) - set (ADDITIONAL_COMPILE_FLAGS - ${ADDITIONAL_COMPILE_FLAGS} -DDART_MPI_ENABLE_UNIFIED_MEMORY_MODEL) -endif() - # Features compile flags # if (PAPI_FOUND AND ENABLE_PAPI) @@ -144,7 +137,7 @@ if (NOT ENABLE_SHARED_WINDOWS) set (ADDITIONAL_COMPILE_FLAGS ${ADDITIONAL_COMPILE_FLAGS} -DDART_MPI_DISABLE_SHARED_WINDOWS) elseif (NOT ENABLE_DYNAMIC_WINDOWS) - message(WARNING + message(WARNING "ENABLE_SHARED_WINDOWS enables dynamic windows " "(ENABLE_DYNAMIC_WINDOWS)") set (ENABLE_DYNAMIC_WINDOWS TRUE) diff --git a/dart-impl/mpi/include/dash/dart/mpi/dart_globmem_priv.h b/dart-impl/mpi/include/dash/dart/mpi/dart_globmem_priv.h index ab8e1f4fe..089546db1 100644 --- a/dart-impl/mpi/include/dash/dart/mpi/dart_globmem_priv.h +++ b/dart-impl/mpi/include/dash/dart/mpi/dart_globmem_priv.h @@ -7,4 +7,10 @@ /* Global object for one-sided communication on memory region allocated with 'local allocation'. */ extern MPI_Win dart_win_local_alloc DART_INTERNAL; +/** + * Check that the window support MPI_WIN_UNIFIED, print warning otherwise. + * Store the result in \c segment->sync_needed. + */ +void dart__mpi__check_memory_model(dart_segment_info_t *segment) DART_INTERNAL; + #endif /* DART__MPI__DART_GLOBMEM_PRIV_H__ */ diff --git a/dart-impl/mpi/include/dash/dart/mpi/dart_segment.h b/dart-impl/mpi/include/dash/dart/mpi/dart_segment.h index 8d977c63d..55d566f36 100644 --- a/dart-impl/mpi/include/dash/dart/mpi/dart_segment.h +++ b/dart-impl/mpi/include/dash/dart/mpi/dart_segment.h @@ -28,6 +28,7 @@ typedef struct uint16_t flags; /* 16 bit flags */ dart_segid_t segid; /* ID of the segment, globally unique in a team */ bool is_dynamic; /* whether this is a shared memory segment */ + bool sync_needed; /* whether a call to MPI_WIN_SYNC is needed */ } dart_segment_info_t; // forward declaration to make the compiler happy diff --git a/dart-impl/mpi/src/Makefile b/dart-impl/mpi/src/Makefile index 2b73c3bf5..bf89454d9 100644 --- a/dart-impl/mpi/src/Makefile +++ b/dart-impl/mpi/src/Makefile @@ -7,7 +7,6 @@ CFLAGS+=-I.. -I../$(DART_IF)/include -I../include -I../../base/include -L/usr/li #CFLAGS+=-DDART_ENABLE_HWLOC CFLAGS+=-DDART_MPI_DISABLE_SHARED_WINDOWS CFLAGS+=-DDART_MPI_ENABLE_DYNAMIC_WINDOWS -#CGLAGS+=-DDART_MPI_ENABLE_UNIFIED_MEMORY_MODEL #CFLAGS+=-DDART_DEBUG #OPT_FLAGS=-O3 diff --git a/dart-impl/mpi/src/dart_communication.c b/dart-impl/mpi/src/dart_communication.c index 88ca9bc45..b4da40d73 100644 --- a/dart-impl/mpi/src/dart_communication.c +++ b/dart-impl/mpi/src/dart_communication.c @@ -70,8 +70,13 @@ } while (0) #define CHECK_TYPE_CONSTRAINTS(_src_type, _dst_type, _num_elem) \ - CHECK_EQUAL_BASETYPE(_src_type, _dst_type); \ - CHECK_NUM_ELEM(_src_type, _dst_type, _num_elem); + do { \ + CHECK_EQUAL_BASETYPE(_src_type, _dst_type); \ + if (!dart__mpi__datatype_iscontiguous(_src_type) || \ + !dart__mpi__datatype_iscontiguous(_dst_type)) { \ + CHECK_NUM_ELEM(_src_type, _dst_type, _num_elem); \ + } \ + } while (0) /** * Temporary space allocation: @@ -294,8 +299,6 @@ dart__mpi__get_complex( { if (num_reqs != NULL) *num_reqs = 0; - CHECK_TYPE_CONSTRAINTS(src_type, dst_type, nelem); - MPI_Win win = seginfo->win; char * dest_ptr = (char*) dest; offset += dart_segment_disp(seginfo, team_unit_id); @@ -418,6 +421,7 @@ dart__mpi__put_basic( return DART_OK; } +/* slow path for puts of derived types */ static inline dart_ret_t dart__mpi__put_complex( @@ -435,9 +439,6 @@ dart__mpi__put_complex( if (flush_required_ptr) *flush_required_ptr = true; if (num_reqs) *num_reqs = 0; - // slow path for derived types - CHECK_TYPE_CONSTRAINTS(src_type, dst_type, nelem); - MPI_Win win = seginfo->win; const char * src_ptr = (const char*) src; offset += dart_segment_disp(seginfo, team_unit_id); @@ -496,6 +497,8 @@ dart_ret_t dart_get( dart_team_unit_t team_unit_id = DART_TEAM_UNIT_ID(gptr.unitid); dart_team_t teamid = gptr.teamid; + CHECK_TYPE_CONSTRAINTS(src_type, dst_type, nelem); + dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid); if (dart__unlikely(team_data == NULL)) { DART_LOG_ERROR("dart_get ! failed: Unknown team %i!", teamid); @@ -520,7 +523,6 @@ dart_ret_t dart_get( if (dart__mpi__datatype_iscontiguous(src_type) && dart__mpi__datatype_iscontiguous(dst_type)) { // fast-path for basic types - CHECK_EQUAL_BASETYPE(src_type, dst_type); ret = dart__mpi__get_basic(team_data, team_unit_id, seginfo, dest, offset, nelem, src_type, NULL, NULL); } else { @@ -545,7 +547,7 @@ dart_ret_t dart_put( dart_team_unit_t team_unit_id = DART_TEAM_UNIT_ID(gptr.unitid); dart_team_t teamid = gptr.teamid; - CHECK_EQUAL_BASETYPE(src_type, dst_type); + CHECK_TYPE_CONSTRAINTS(src_type, dst_type, nelem); dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid); if (dart__unlikely(team_data == NULL)) { @@ -568,7 +570,6 @@ dart_ret_t dart_put( if (dart__mpi__datatype_iscontiguous(src_type) && dart__mpi__datatype_iscontiguous(dst_type)) { // fast path for basic data types - CHECK_EQUAL_BASETYPE(src_type, dst_type); ret = dart__mpi__put_basic(team_data, team_unit_id, seginfo, src, offset, nelem, src_type, NULL, NULL, NULL); @@ -906,6 +907,8 @@ dart_ret_t dart_get_handle( *handleptr = DART_HANDLE_NULL; + CHECK_TYPE_CONSTRAINTS(src_type, dst_type, nelem); + dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid); if (dart__unlikely(team_data == NULL)) { DART_LOG_ERROR("dart_get_handle ! failed: Unknown team %i!", teamid); @@ -940,7 +943,6 @@ dart_ret_t dart_get_handle( if (dart__mpi__datatype_iscontiguous(src_type) && dart__mpi__datatype_iscontiguous(dst_type)) { // fast-path for basic types - CHECK_EQUAL_BASETYPE(src_type, dst_type); ret = dart__mpi__get_basic(team_data, team_unit_id, seginfo, dest, offset, nelem, src_type, handle->reqs, &handle->num_reqs); @@ -978,7 +980,7 @@ dart_ret_t dart_put_handle( *handleptr = DART_HANDLE_NULL; - CHECK_EQUAL_BASETYPE(src_type, dst_type); + CHECK_TYPE_CONSTRAINTS(src_type, dst_type, nelem); dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid); if (dart__unlikely(team_data == NULL)) { @@ -1009,7 +1011,6 @@ dart_ret_t dart_put_handle( if (dart__mpi__datatype_iscontiguous(src_type) && dart__mpi__datatype_iscontiguous(dst_type)) { // fast path for basic data types - CHECK_EQUAL_BASETYPE(src_type, dst_type); ret = dart__mpi__put_basic(team_data, team_unit_id, seginfo, src, offset, nelem, src_type, handle->reqs, @@ -1054,7 +1055,7 @@ dart_ret_t dart_put_blocking( int16_t seg_id = gptr.segid; dart_team_t teamid = gptr.teamid; - CHECK_EQUAL_BASETYPE(src_type, dst_type); + CHECK_TYPE_CONSTRAINTS(src_type, dst_type, nelem); dart_team_data_t *team_data = dart_adapt_teamlist_get(gptr.teamid); if (dart__unlikely(team_data == NULL)) { @@ -1083,7 +1084,6 @@ dart_ret_t dart_put_blocking( if (dart__mpi__datatype_iscontiguous(src_type) && dart__mpi__datatype_iscontiguous(dst_type)) { // fast path for basic data types - CHECK_EQUAL_BASETYPE(src_type, dst_type); ret = dart__mpi__put_basic(team_data, team_unit_id, seginfo, src, offset, nelem, src_type, NULL, NULL, &needs_flush); @@ -1118,7 +1118,7 @@ dart_ret_t dart_get_blocking( int16_t seg_id = gptr.segid; dart_team_t teamid = gptr.teamid; - CHECK_EQUAL_BASETYPE(src_type, dst_type); + CHECK_TYPE_CONSTRAINTS(src_type, dst_type, nelem); dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid); if (dart__unlikely(team_data == NULL)) { @@ -1150,7 +1150,6 @@ dart_ret_t dart_get_blocking( if (dart__mpi__datatype_iscontiguous(src_type) && dart__mpi__datatype_iscontiguous(dst_type)) { // fast-path for basic types - CHECK_EQUAL_BASETYPE(src_type, dst_type); ret = dart__mpi__get_basic(team_data, team_unit_id, seginfo, dest, offset, nelem, src_type, reqs, &num_reqs); @@ -1205,9 +1204,12 @@ dart_ret_t dart_flush( DART_LOG_TRACE("dart_flush: MPI_Win_flush"); CHECK_MPI_RET( MPI_Win_flush(team_unit_id.id, win), "MPI_Win_flush"); - DART_LOG_TRACE("dart_flush: MPI_Win_sync"); - CHECK_MPI_RET( - MPI_Win_sync(win), "MPI_Win_sync"); + + if (seginfo->sync_needed) { + DART_LOG_TRACE("dart_flush: MPI_Win_sync"); + CHECK_MPI_RET( + MPI_Win_sync(win), "MPI_Win_sync"); + } // trigger progress int flag; @@ -1250,9 +1252,12 @@ dart_ret_t dart_flush_all( DART_LOG_TRACE("dart_flush_all: MPI_Win_flush_all"); CHECK_MPI_RET( MPI_Win_flush_all(win), "MPI_Win_flush"); - DART_LOG_TRACE("dart_flush_all: MPI_Win_sync"); - CHECK_MPI_RET( - MPI_Win_sync(win), "MPI_Win_sync"); + + if (seginfo->sync_needed) { + DART_LOG_TRACE("dart_flush_all: MPI_Win_sync"); + CHECK_MPI_RET( + MPI_Win_sync(win), "MPI_Win_sync"); + } // trigger progress int flag; diff --git a/dart-impl/mpi/src/dart_globmem.c b/dart-impl/mpi/src/dart_globmem.c index 1b41e1ca1..d6ce7e514 100644 --- a/dart-impl/mpi/src/dart_globmem.c +++ b/dart-impl/mpi/src/dart_globmem.c @@ -166,6 +166,37 @@ dart_ret_t dart_memfree (dart_gptr_t gptr) return DART_OK; } +/** + * Check that the window support MPI_WIN_UNIFIED, print warning otherwise. + */ +void dart__mpi__check_memory_model(dart_segment_info_t *segment) +{ + int mem_model, flag; + MPI_Win_get_attr(segment->win, MPI_WIN_MODEL, &mem_model, &flag); + + DART_ASSERT_MSG(flag != 0, "Failed to query window memory model!"); + + segment->sync_needed = false; + if (mem_model != MPI_WIN_UNIFIED) { + static bool warning_printed = false; + if (!warning_printed) { + dart_global_unit_t myid; + dart_myid(&myid); + if (myid.id == 0) { + DART_LOG_WARN( + "The allocated MPI window does not support the unified memory model. " + ); + DART_LOG_WARN( + "DASH may not be able to guaranteed consistency of local and remote updates." + ); + DART_LOG_WARN("USE AT YOUR OWN RISK!"); + } + warning_printed = true; + } + segment->sync_needed = true; + } +} + #ifdef DART_MPI_ENABLE_DYNAMIC_WINDOWS static dart_ret_t dart_team_memalloc_aligned_dynamic( @@ -352,7 +383,11 @@ dart_team_memalloc_aligned_dynamic( segment->win = team_data->window; segment->selfbaseptr = sub_mem; segment->is_dynamic = true; - + /** + * Following the example 11.21 in the MPI standard v3.1, a sync is necessary + * even in the unified memory model if load/stores are used in shared memory. + */ + segment->sync_needed = true; /* -- Updating infos on gptr -- */ /* Segid equals to dart_memid (always a positive integer), identifies an @@ -401,14 +436,19 @@ dart_team_memalloc_aligned_full( dart_segment_info_t *segment = dart_segment_alloc( &team_data->segdata, DART_SEGMENT_ALLOC); + MPI_Info win_info; + MPI_Info_create(&win_info); + MPI_Info_set(win_info, "same_disp_unit", "true"); if (MPI_Win_allocate( - nbytes, 1, MPI_INFO_NULL, + nbytes, 1, win_info, team_data->comm, &baseptr, &win) != MPI_SUCCESS) { DART_LOG_ERROR("dart_team_memalloc_aligned_full: MPI_Win_allocate failed"); + MPI_Info_free(&win_info); return DART_ERR_OTHER; } + MPI_Info_free(&win_info); - if (MPI_Win_lock_all(0, win) != MPI_SUCCESS) { + if (MPI_Win_lock_all(MPI_MODE_NOCHECK, win) != MPI_SUCCESS) { DART_LOG_ERROR("dart_team_memalloc_aligned_full: MPI_Win_lock_all failed"); return DART_ERR_OTHER; } @@ -430,6 +470,7 @@ dart_team_memalloc_aligned_full( segment->win = win; segment->is_dynamic = false; + dart__mpi__check_memory_model(segment); gptr->segid = segment->segid; gptr->unitid = gptr_unitid; @@ -593,6 +634,8 @@ dart_team_memregister_aligned( segment->selfbaseptr = (char *)addr; segment->flags = 0; + dart__mpi__check_memory_model(segment); + gptr->unitid = gptr_unitid; gptr->segid = segment->segid; gptr->teamid = teamid; @@ -665,6 +708,7 @@ dart_team_memregister( segment->selfbaseptr = (char *)addr; segment->flags = 0; + dart__mpi__check_memory_model(segment); gptr->unitid = gptr_unitid; gptr->segid = segment->segid; diff --git a/dart-impl/mpi/src/dart_initialization.c b/dart-impl/mpi/src/dart_initialization.c index 4fe3863d2..5f8fb19f5 100644 --- a/dart-impl/mpi/src/dart_initialization.c +++ b/dart-impl/mpi/src/dart_initialization.c @@ -32,6 +32,10 @@ dart_ret_t create_local_alloc(dart_team_data_t *team_data) dart_localpool = dart_buddy_new(DART_LOCAL_ALLOC_SIZE); MPI_Win dart_sharedmem_win_local_alloc = MPI_WIN_NULL; char* *dart_sharedmem_local_baseptr_set = NULL; + MPI_Info win_info; + MPI_Info_create(&win_info); + MPI_Info_set(win_info, "same_size", "true"); + MPI_Info_set(win_info, "same_disp_unit", "true"); #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS) @@ -42,8 +46,6 @@ dart_ret_t create_local_alloc(dart_team_data_t *team_data) if (sharedmem_comm != MPI_COMM_NULL) { DART_LOG_DEBUG("dart_init: MPI_Win_allocate_shared(nbytes:%d)", DART_LOCAL_ALLOC_SIZE); - MPI_Info win_info; - MPI_Info_create(&win_info); MPI_Info_set(win_info, "alloc_shared_noncontig", "true"); /* Reserve a free shared memory block for non-collective * global memory allocation. */ @@ -61,7 +63,6 @@ dart_ret_t create_local_alloc(dart_team_data_t *team_data) return DART_ERR_OTHER; } - MPI_Info_free(&win_info); DART_LOG_DEBUG("dart_init: MPI_Win_allocate_shared completed"); @@ -90,13 +91,6 @@ dart_ret_t create_local_alloc(dart_team_data_t *team_data) } } } -#else - MPI_Alloc_mem( - DART_LOCAL_ALLOC_SIZE, - MPI_INFO_NULL, - &dart_mempool_localalloc); -#endif - /* Create a single global win object for dart local * allocation based on the above allocated shared memory. * @@ -105,9 +99,18 @@ dart_ret_t create_local_alloc(dart_team_data_t *team_data) dart_mempool_localalloc, DART_LOCAL_ALLOC_SIZE, sizeof(char), - MPI_INFO_NULL, + win_info, DART_COMM_WORLD, &dart_win_local_alloc); +#else + MPI_Win_allocate( + DART_LOCAL_ALLOC_SIZE, sizeof(char), + win_info, DART_COMM_WORLD, + &dart_mempool_localalloc, + &dart_win_local_alloc); +#endif + + MPI_Info_free(&win_info); /* Start an access epoch on dart_win_local_alloc, and later * on all the units can access the memory region allocated @@ -130,9 +133,11 @@ dart_ret_t create_local_alloc(dart_team_data_t *team_data) segment->win = dart_win_local_alloc; segment->shmwin = dart_sharedmem_win_local_alloc; segment->selfbaseptr = dart_mempool_localalloc; - // addressing in this window is relative, no need to store displacements + // addressing in this window is relative, no need to exchange displacements segment->disp = calloc(team_data->size, sizeof(MPI_Aint)); - segment->is_dynamic = false; + segment->is_dynamic = false; + + dart__mpi__check_memory_model(segment); return DART_OK; } @@ -319,11 +324,6 @@ dart_ret_t dart_exit() /* Has MPI shared windows: */ MPI_Win_free(&seginfo->shmwin); MPI_Comm_free(&(team_data->sharedmem_comm)); -#else - /* No MPI shared windows: */ - if (dart_mempool_localalloc) { - MPI_Free_mem(dart_mempool_localalloc); - } #endif MPI_Win_free(&team_data->window); diff --git a/dart-impl/shmem/CMakeLists.txt b/dart-impl/shmem/CMakeLists.txt index 98b7c1c7e..f11e73b1e 100644 --- a/dart-impl/shmem/CMakeLists.txt +++ b/dart-impl/shmem/CMakeLists.txt @@ -9,8 +9,6 @@ set(ENABLE_LOGGING ${ENABLE_LOGGING} PARENT_SCOPE) set(ENABLE_DART_LOGGING ${ENABLE_DART_LOGGING} PARENT_SCOPE) -set(ENABLE_UNIFIED_MEMORY_MODEL ${ENABLE_UNIFIED_MEMORY_MODEL} - PARENT_SCOPE) # Library name set(DASH_LIBRARY ${DASH_LIBRARY} PARENT_SCOPE) @@ -39,11 +37,6 @@ if (ENABLE_DART_LOGGING) "${ADDITIONAL_COMPILE_FLAGS} -DDART_ENABLE_LOGGING") endif() -if (ENABLE_UNIFIED_MEMORY_MODEL) - set (ADDITIONAL_COMPILE_FLAGS - "${ADDITIONAL_COMPILE_FLAGS} -DDASH_ENABLE_UNIFIED_MEMORY_MODEL") -endif() - ## Build targets # Directories containing the implementation of the library (-I): diff --git a/dash/CMakeLists.txt b/dash/CMakeLists.txt index 39258aa7d..e86df9908 100644 --- a/dash/CMakeLists.txt +++ b/dash/CMakeLists.txt @@ -19,8 +19,6 @@ set(ENABLE_TRACE_LOGGING ${ENABLE_TRACE_LOGGING} PARENT_SCOPE) set(ENABLE_THREADSUPPORT ${ENABLE_THREADSUPPORT} PARENT_SCOPE) -set(ENABLE_UNIFIED_MEMORY_MODEL ${ENABLE_UNIFIED_MEMORY_MODEL} - PARENT_SCOPE) set(ENABLE_DEFAULT_INDEX_TYPE_LONG ${ENABLE_DEFAULT_INDEX_TYPE_LONG} PARENT_SCOPE) set(ENABLE_SHARED_WINDOWS ${ENABLE_SHARED_WINDOWS} @@ -84,10 +82,6 @@ if (ENABLE_THREADSUPPORT) set (ADDITIONAL_COMPILE_FLAGS ${ADDITIONAL_COMPILE_FLAGS} -DDASH_ENABLE_THREADSUPPORT) endif() -if (ENABLE_UNIFIED_MEMORY_MODEL) - set (ADDITIONAL_COMPILE_FLAGS - ${ADDITIONAL_COMPILE_FLAGS} -DDASH_ENABLE_UNIFIED_MEMORY_MODEL) -endif() if (ENABLE_LOGGING) set (ADDITIONAL_COMPILE_FLAGS ${ADDITIONAL_COMPILE_FLAGS} -DDASH_ENABLE_LOGGING) diff --git a/dash/scripts/dash-ci-deploy.sh b/dash/scripts/dash-ci-deploy.sh index 910599876..3b25fec15 100755 --- a/dash/scripts/dash-ci-deploy.sh +++ b/dash/scripts/dash-ci-deploy.sh @@ -70,7 +70,6 @@ if [ "$BUILD_TYPE" = "Release" ]; then -DDART_IMPLEMENTATIONS=mpi \ -DENABLE_ASSERTIONS=OFF \ -DENABLE_SHARED_WINDOWS=ON \ - -DENABLE_UNIFIED_MEMORY_MODEL=ON \ -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \ -DENABLE_LOGGING=OFF \ -DENABLE_TRACE_LOGGING=OFF \ @@ -93,7 +92,6 @@ elif [ "$BUILD_TYPE" = "Debug" ]; then -DDART_IMPLEMENTATIONS=mpi \ -DENABLE_ASSERTIONS=ON\ -DENABLE_SHARED_WINDOWS=ON \ - -DENABLE_UNIFIED_MEMORY_MODEL=ON \ -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \ -DENABLE_LOGGING=OFF \ -DENABLE_TEST_LOGGING=OFF \ @@ -117,7 +115,6 @@ elif [ "$BUILD_TYPE" = "Development" ]; then -DDART_IMPLEMENTATIONS=mpi \ -DENABLE_ASSERTIONS=ON \ -DENABLE_SHARED_WINDOWS=ON \ - -DENABLE_UNIFIED_MEMORY_MODEL=ON \ -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \ -DENABLE_LOGGING=ON \ -DENABLE_TRACE_LOGGING=ON \ @@ -142,7 +139,6 @@ elif [ "$BUILD_TYPE" = "Minimal" ]; then -DENABLE_LT_OPTIMIZATION=OFF \ -DENABLE_ASSERTIONS=OFF \ -DENABLE_SHARED_WINDOWS=OFF \ - -DENABLE_UNIFIED_MEMORY_MODEL=ON \ -DENABLE_DEFAULT_INDEX_TYPE_LONG=OFF \ -DENABLE_LOGGING=OFF \ -DENABLE_TRACE_LOGGING=OFF \ @@ -171,7 +167,6 @@ elif [ "$BUILD_TYPE" = "Nasty" ]; then -DDART_IMPLEMENTATIONS=mpi \ -DENABLE_ASSERTIONS=ON \ -DENABLE_SHARED_WINDOWS=OFF \ - -DENABLE_UNIFIED_MEMORY_MODEL=ON \ -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \ -DENABLE_LOGGING=OFF \ -DENABLE_TRACE_LOGGING=OFF \ @@ -196,7 +191,6 @@ elif [ "$BUILD_TYPE" = "Coverage-Shared" ]; then -DENABLE_ASSERTIONS=ON \ -DENABLE_SHARED_WINDOWS=ON \ -DENABLE_DYNAMIC_WINDOWS=ON \ - -DENABLE_UNIFIED_MEMORY_MODEL=ON \ -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \ -DENABLE_LOGGING=ON \ -DENABLE_TRACE_LOGGING=OFF \ @@ -227,7 +221,6 @@ elif [ "$BUILD_TYPE" = "Coverage-NoShared" ]; then -DENABLE_ASSERTIONS=ON \ -DENABLE_SHARED_WINDOWS=OFF \ -DENABLE_DYNAMIC_WINDOWS=OFF \ - -DENABLE_UNIFIED_MEMORY_MODEL=ON \ -DENABLE_DEFAULT_INDEX_TYPE_LONG=ON \ -DENABLE_LOGGING=ON \ -DENABLE_TRACE_LOGGING=OFF \