Make segment IDs unique again!

CHANGELOG.md

@@ -66,8 +66,16 @@ Features:
 - Made global memory allocation and communication operations aware of the underlying 
   data type to improve stability and performance. 
 
+- Made DART global pointer globally unique to allow copying of global pointer between
+  members of the team that allocated the global memory. Note that a global now 
+  contains unit IDs relative to the team that allocated the memory instead of 
+  global unit IDs. 
+
 - Extended use of `const` specifier in DART communication interface 
 
+- Introduced typed unit IDs to safely distinguish between global IDs 
+  (`dart_global_unit_t`) and IDs that are relative to a team (`dart_team_unit_t`).
+
 - Added interface component `dart_locality` implementing topology discovery
   and hierarchical locality description
 
@@ -102,7 +110,10 @@ Features:
         - `dart__base__locality__unit`
 
 Fixes:
-- Added clarification that DART currently does not provide thread-safe access. 
+- Added clarification which DART functionality provides thread-safe access. 
+  DART functions can be considered thread-safe as long as they do not operate
+  on the same data structures. In particular, thread-concurrent (collective) 
+  operations on the same team are not guaranteed to be safe. 
 
 # DASH 0.2.0 (2016-03-03)
 

CMakeExt/CompilerFlags.cmake

@@ -207,14 +207,14 @@ set(CMAKE_C_FLAGS_DEBUG
 set(CMAKE_C_FLAGS_DEBUG
     "${CMAKE_C_FLAGS_DEBUG} ${CC_REPORT_FLAG} ${CC_WARN_FLAG}")
 set(CMAKE_C_FLAGS_DEBUG
-    "${CMAKE_C_FLAGS_DEBUG} -Ofast -DDASH_DEBUG ${CC_GDB_FLAG}")
+    "${CMAKE_C_FLAGS_DEBUG} -O0 -DDASH_DEBUG ${CC_GDB_FLAG}")
 
 set(CMAKE_CXX_FLAGS_DEBUG
     "${CMAKE_CXX_FLAGS_DEBUG} ${CXX_STD_FLAG} ${CXX_OMP_FLAG}")
 set(CMAKE_CXX_FLAGS_DEBUG
     "${CMAKE_CXX_FLAGS_DEBUG} ${CC_REPORT_FLAG} ${CXX_WARN_FLAG}")
 set(CMAKE_CXX_FLAGS_DEBUG
-    "${CMAKE_CXX_FLAGS_DEBUG} -Ofast -DDASH_DEBUG ${CXX_GDB_FLAG}")
+    "${CMAKE_CXX_FLAGS_DEBUG} -O0 -DDASH_DEBUG ${CXX_GDB_FLAG}")
 
 
 set(CMAKE_C_FLAGS_RELEASE

dart-if/v3.2/include/dash/dart/if/dart_globmem.h

@@ -57,10 +57,10 @@ extern "C" {
  PROPOSAL: use 128 bit global pointers with the following layout:
 
 
- 0         1         2         3         4         5         6
- 0123456789012345678901234567890123456789012345678901234567890123
- |------<32 bit unit id>--------|-<segment id>--|--flags/resv---|
- |-----------<either a virtual address or an offset>------------|
+ 0       1       2       3       4       5       6       7
+ 0123456701234567012345670123456701234567012345670123456701234567
+ |----<24 bit unit id>---|-flags-|-<segment id>--|---<team id>--|
+ |-----------<64 bit virtual address or offset>-----------------|
 
   */
 
@@ -71,13 +71,18 @@ extern "C" {
  */
 typedef struct
 {
-  /** The unit holding the memory element */
-  dart_unit_t unitid;
-  /** The segment ID of the allocation */
-  int16_t     segid;
+  /**
+   * The unit holding the memory element.
+   * The ID is relative to the team identified by \c teamid.
+   */
+  dart_unit_t  unitid : 24;
   /** Reserved */
-  uint16_t    flags;
-  /** Absolute address or relative offset */
+  unsigned int flags  :  8;
+  /** The segment ID of the allocation. */
+  int16_t     segid;
+  /** The team associated with the allocation. */
+  int16_t     teamid;
+  /** Absolute address or relative offset. */
   union
   {
     uint64_t offset;
@@ -90,37 +95,48 @@ typedef struct
  * \ingroup DartGlobMem
  */
 #ifdef __cplusplus
-#define DART_GPTR_NULL (dart_gptr_t { -1, 0, 0, { 0 } })
+#define DART_GPTR_NULL (dart_gptr_t { -1, 0, 0, DART_TEAM_NULL, { 0 } })
 #else
 #define DART_GPTR_NULL \
-((dart_gptr_t)({ .unitid = -1, \
-                 .segid  =  0, \
-                 .flags  =  0, \
-                 .addr_or_offs.offset = 0 }))
+(dart_gptr_t){ .unitid = -1, \
+               .flags  =  0, \
+               .segid  =  0, \
+               .teamid  =  DART_TEAM_NULL, \
+               .addr_or_offs.offset = 0 }
 #endif
 
 /**
  * Test for NULL global pointer
  *
  * \ingroup DartGlobMem
  */
-#define DART_GPTR_ISNULL(gptr_)			\
-  (gptr_.unitid<0 && gptr_.segid==0 &&		\
-   gptr_.flags==0 && gptr_.addr_or_offs.addr==0)
+#define DART_GPTR_ISNULL(gptr_)         \
+  (gptr_.unitid<0 && gptr_.segid==0 &&  \
+   gptr_.teamid==DART_TEAM_NULL   &&    \
+   gptr_.addr_or_offs.addr==0)
 
 /**
  * Compare two global pointers
  *
  * \ingroup DartGlobMem
  */
-#define DART_GPTR_EQUAL(gptr1_, gptr2_ )		\
-  ((gptr1_.unitid == gptr2_.unitid) &&			\
-   (gptr1_.segid == gptr2_.segid) &&			\
-   (gptr1_.flags == gptr2_.flags) &&			\
-   (gptr1_.addr_or_offs.offset ==			\
+#define DART_GPTR_EQUAL(gptr1_, gptr2_ )    \
+  ((gptr1_.unitid == gptr2_.unitid) &&      \
+   (gptr1_.segid  == gptr2_.segid)  &&      \
+   (gptr1_.teamid == gptr2_.teamid) &&      \
+   (gptr1_.addr_or_offs.offset ==           \
     gptr2_.addr_or_offs.offset) )
 
 
+/**
+ * Segment ID identifying unaligned allocations.
+ *
+ * \sa dart_memalloc
+ * \sa dart_memfree
+ */
+#define DART_SEGMENT_LOCAL ((int16_t)0)
+
+
 /**
  * Get the local memory address for the specified global pointer
  * gptr. I.e., if the global pointer has affinity to the local unit,
@@ -161,7 +177,7 @@ dart_ret_t dart_gptr_setaddr(dart_gptr_t *gptr, void *addr);
  * \threadsafe
  * \ingroup DartGlobMem
  */
-dart_ret_t dart_gptr_incaddr(dart_gptr_t *gptr, int32_t offs);
+dart_ret_t dart_gptr_incaddr(dart_gptr_t *gptr, int64_t offs);
 
 /**
  * Set the unit information for the specified global pointer.
@@ -174,7 +190,38 @@ dart_ret_t dart_gptr_incaddr(dart_gptr_t *gptr, int32_t offs);
  * \threadsafe
  * \ingroup DartGlobMem
  */
-dart_ret_t dart_gptr_setunit(dart_gptr_t *gptr, dart_global_unit_t unit);
+dart_ret_t dart_gptr_setunit(dart_gptr_t *gptr, dart_team_unit_t unit);
+
+/**
+ * Get the flags field for the segment specified by the global pointer.
+ *
+ * \param gptr Global Pointer describing a segment.
+ * \param unit The flags to get for segment in \c gptr
+ *
+ * \return \c DART_OK on success, any other of \ref dart_ret_t otherwise.
+ *
+ * \threadsafe
+ * \ingroup DartGlobMem
+ */
+dart_ret_t dart_gptr_getflags(dart_gptr_t gptr, uint16_t *flags);
+
+
+/**
+ * Set the flags field for the segment specified by the global pointer.
+ * The flags are stored in the segment's meta data. The lower 8 bit of
+ * the flags are stored in the \c .flags field of the \c gptr for
+ * fast access. The remaining flags can be queried through
+ * \ref dart_gptr_getflags.
+ *
+ * \param gptr Global Pointer describing a segment.
+ * \param unit The flags to set for segment in \c gptr
+ *
+ * \return \c DART_OK on success, any other of \ref dart_ret_t otherwise.
+ *
+ * \threadsafe
+ * \ingroup DartGlobMem
+ */
+dart_ret_t dart_gptr_setflags(dart_gptr_t *gptr, uint16_t flags);
 
 /**
  * Allocates memory for \c nelem elements of type \c dtype in the global
@@ -265,7 +312,6 @@ dart_ret_t dart_team_memalloc_aligned(
  * \ingroup DartGlobMem
  */
 dart_ret_t dart_team_memfree(
-  dart_team_t teamid,
   dart_gptr_t gptr);
 
 /**
@@ -331,7 +377,7 @@ dart_ret_t dart_team_memregister(
  * \threadsafe_none
  * \ingroup DartGlobMem
  */
-dart_ret_t dart_team_memderegister(dart_team_t teamid, dart_gptr_t gptr);
+dart_ret_t dart_team_memderegister(dart_gptr_t gptr);
 
 
 /** \cond DART_HIDDEN_SYMBOLS */

dart-if/v3.2/include/dash/dart/if/dart_types.h

@@ -213,7 +213,7 @@ dart_global_unit_t dart_create_global_unit(dart_unit_t unit)
  * Data type for storing a team ID
  * \ingroup DartTypes
  */
-typedef int32_t dart_team_t;
+typedef int16_t dart_team_t;
 
 /**
  * Undefined team ID.

dart-impl/mpi/include/dash/dart/mpi/dart_globmem_priv.h

@@ -7,11 +7,10 @@
   do {                                                      \
     gptr_.unitid = gptrt_.unitid;                           \
     gptr_.segid  = gptrt_.segid;                            \
-    gptr_.flags  = gptrt_.flags;                            \
+    gptr_.teamid = gptrt_.teamid;                           \
     gptr_.addr_or_offs.offset = gptrt_.addr_or_offs.offset; \
   } while(0)
 
-
 /* Global object for one-sided communication on memory region allocated with 'local allocation'. */
 extern MPI_Win dart_win_local_alloc;
 #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)

dart-impl/mpi/include/dash/dart/mpi/dart_mem.h

@@ -12,8 +12,8 @@
 #include <string.h>
 #include <inttypes.h>
 
+// forward declaration
 struct dart_buddy;
-
 extern char* dart_mempool_localalloc;
 extern struct dart_buddy* dart_localpool;
 

dart-impl/mpi/include/dash/dart/mpi/dart_segment.h

@@ -13,6 +13,8 @@
 
 typedef int16_t dart_segid_t;
 
+#define DART_SEGMENT_HASH_SIZE 256
+
 typedef struct
 {
   dart_segid_t seg_id; /* seg_id determines a global pointer uniquely */
@@ -21,28 +23,32 @@ typedef struct
   char      ** baseptr;
   char       * selfbaseptr;
   MPI_Win      win;
+  uint16_t     flags;
 } dart_segment_info_t;
 
+// forward declaration to make the compiler happy
+typedef struct dart_seghash_elem dart_seghash_elem_t;
+
+typedef struct {
+
+  dart_seghash_elem_t* hashtab[DART_SEGMENT_HASH_SIZE];
+  dart_team_t team_id;
+
+  // TODO: Add free-segment-ID list here
+
+} dart_segmentdata_t;
+
 
 /**
  * @brief Initialize the segment data hash table.
  */
-dart_ret_t dart_segment_init();
+dart_ret_t dart_segment_init(dart_segmentdata_t *segdata, dart_team_t teamid);
 
 /**
  * @brief Allocates a new segment data struct. May be served from a freelist.
  */
-dart_ret_t dart_segment_alloc(dart_segid_t segid, uint16_t team_idx);
+dart_ret_t dart_segment_alloc(dart_segmentdata_t *segdata, const dart_segment_info_t *item);
 
-/**
- * @brief Returns the registered team index for the segment ID.
- */
-dart_ret_t dart_segment_get_teamidx(dart_segid_t segid, uint16_t *team_idx);
-
-/**
- * @brief Add segment information to the segment hash table.
- */
-dart_ret_t dart_segment_add_info(const dart_segment_info_t *item);
 
 #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
 /** @brief Query the shared memory window object associated with the specified seg_id.
@@ -53,17 +59,19 @@ dart_ret_t dart_segment_add_info(const dart_segment_info_t *item);
  *  @retval non-negative integer Search successfully.
  *  @retval negative integer Failure.
  */
-dart_ret_t dart_segment_get_win(int16_t seg_id, MPI_Win * win);
+dart_ret_t dart_segment_get_win(dart_segmentdata_t *segdata, int16_t seg_id, MPI_Win * win);
 
 dart_ret_t dart_segment_get_baseptr(
+  dart_segmentdata_t   * segdata,
   int16_t                seg_id,
   dart_team_unit_t       rel_unitid,
   char               **  baseptr_s);
 #endif
 
 dart_ret_t dart_segment_get_selfbaseptr(
-  int16_t    seg_id,
-  char   **  baseptr);
+  dart_segmentdata_t * segdata,
+  int16_t              seg_id,
+  char              ** baseptr);
 
 /** @brief Query the address of the memory location of the specified rel_unit in specified team.
  *
@@ -73,28 +81,41 @@ dart_ret_t dart_segment_get_selfbaseptr(
  *
  *  @retval ditto
  */
-dart_ret_t dart_segment_get_disp(int16_t             seg_id,
-                                 dart_team_unit_t    rel_unitid,
-                                 MPI_Aint          * disp_s);
+dart_ret_t dart_segment_get_disp(dart_segmentdata_t * segdata,
+                                 int16_t              seg_id,
+                                 dart_team_unit_t     rel_unitid,
+                                 MPI_Aint           * disp_s);
 
 /** @brief Query the length of the global memory block indicated by the specified seg_id.
  *
  *  @retval ditto
  */
 dart_ret_t dart_segment_get_size(
-  int16_t   seg_id,
-  size_t  * size);
+  dart_segmentdata_t * segdata,
+  int16_t              seg_id,
+  size_t             * size);
+
+dart_ret_t dart_segment_get_flags(
+  dart_segmentdata_t * segdata,
+  int16_t              seg_id,
+  uint16_t           * flags);
+
+dart_ret_t dart_segment_set_flags(
+  dart_segmentdata_t * segdata,
+  int16_t              seg_id,
+  uint16_t             flags);
 
 /**
  * @brief Deallocates the segment identified by the segment ID.
  */
-dart_ret_t dart_segment_free(dart_segid_t segid);
+dart_ret_t dart_segment_free(dart_segmentdata_t * segdata,
+                             dart_segid_t         segid);
 
 
 /**
  * @brief Clear the segment data hash table.
  */
-dart_ret_t dart_segment_fini();
+dart_ret_t dart_segment_fini(dart_segmentdata_t *segdata);
 
 
 #endif /* DART_SEGMENT_H_ */