Make segment IDs unique again!

CMakeExt/CompilerFlags.cmake

@@ -196,9 +196,9 @@ set(CMAKE_CXX_FLAGS_RELEASE
     "${CMAKE_CXX_FLAGS_RELEASE} ${CXX_ENV_SETUP_FLAGS}")
 
 set(CMAKE_C_FLAGS_DEBUG
-    "${CMAKE_C_FLAGS_DEBUG} ${CC_STD_FLAG} ${CXX_OMP_FLAG} ${CC_REPORT_FLAG} ${CC_WARN_FLAG} -Ofast -DDASH_DEBUG ${CC_GDB_FLAG}")
+    "${CMAKE_C_FLAGS_DEBUG} ${CC_STD_FLAG} ${CXX_OMP_FLAG} ${CC_REPORT_FLAG} ${CC_WARN_FLAG} -O0 -DDASH_DEBUG ${CC_GDB_FLAG}")
 set(CMAKE_CXX_FLAGS_DEBUG
-    "${CMAKE_CXX_FLAGS_DEBUG} ${CXX_STD_FLAG} ${CXX_OMP_FLAG} ${CC_REPORT_FLAG} ${CXX_WARN_FLAG} -Ofast -DDASH_DEBUG ${CXX_GDB_FLAG}")
+    "${CMAKE_CXX_FLAGS_DEBUG} ${CXX_STD_FLAG} ${CXX_OMP_FLAG} ${CC_REPORT_FLAG} ${CXX_WARN_FLAG} -O0 -DDASH_DEBUG ${CXX_GDB_FLAG}")
 
 set(CMAKE_C_FLAGS_RELEASE
   "${CMAKE_C_FLAGS_RELEASE} ${CC_STD_FLAG} ${CXX_OMP_FLAG} ${CXX_LTO_FLAG} ${CC_REPORT_FLAG} ${CC_WARN_FLAG} -Ofast -DDASH_RELEASE")

dart-if/v3.2/include/dash/dart/if/dart_globmem.h

@@ -76,7 +76,7 @@ typedef struct
   /** The segment ID of the allocation */
   int16_t     segid;
   /** Reserved */
-  uint16_t    flags;
+  int16_t     teamid;
   /** Absolute address or relative offset */
   union
   {
@@ -90,13 +90,13 @@ typedef struct
  * \ingroup DartGlobMem
  */
 #ifdef __cplusplus
-#define DART_GPTR_NULL (dart_gptr_t { -1, 0, 0, { 0 } })
+#define DART_GPTR_NULL (dart_gptr_t { -1, 0, DART_TEAM_NULL, { 0 } })
 #else
 #define DART_GPTR_NULL \
-((dart_gptr_t)({ .unitid = -1, \
-                 .segid  =  0, \
-                 .flags  =  0, \
-                 .addr_or_offs.offset = 0 }))
+(dart_gptr_t){ .unitid = -1, \
+               .segid  =  0, \
+               .teamid  =  DART_TEAM_NULL, \
+               .addr_or_offs.offset = 0 }
 #endif
 
 /**
@@ -106,7 +106,8 @@ typedef struct
  */
 #define DART_GPTR_ISNULL(gptr_)			\
   (gptr_.unitid<0 && gptr_.segid==0 &&		\
-   gptr_.flags==0 && gptr_.addr_or_offs.addr==0)
+   gptr_.teamid==DART_TEAM_NULL   &&    \
+   gptr_.addr_or_offs.addr==0)
 
 /**
  * Compare two global pointers
@@ -115,12 +116,21 @@ typedef struct
  */
 #define DART_GPTR_EQUAL(gptr1_, gptr2_ )		\
   ((gptr1_.unitid == gptr2_.unitid) &&			\
-   (gptr1_.segid == gptr2_.segid) &&			\
-   (gptr1_.flags == gptr2_.flags) &&			\
+   (gptr1_.segid  == gptr2_.segid)  &&			\
+   (gptr1_.teamid == gptr2_.teamid) &&			\
    (gptr1_.addr_or_offs.offset ==			\
     gptr2_.addr_or_offs.offset) )
 
 
+/**
+ * Segment ID identifying unaligned allocations.
+ *
+ * \sa dart_memalloc
+ * \sa dart_memfree
+ */
+#define DART_SEGMENT_LOCAL ((int16_t)0)
+
+
 /**
  * Get the local memory address for the specified global pointer
  * gptr. I.e., if the global pointer has affinity to the local unit,
@@ -161,7 +171,7 @@ dart_ret_t dart_gptr_setaddr(dart_gptr_t *gptr, void *addr);
  * \threadsafe
  * \ingroup DartGlobMem
  */
-dart_ret_t dart_gptr_incaddr(dart_gptr_t *gptr, int32_t offs);
+dart_ret_t dart_gptr_incaddr(dart_gptr_t *gptr, int64_t offs);
 
 /**
  * Set the unit information for the specified global pointer.
@@ -176,6 +186,33 @@ dart_ret_t dart_gptr_incaddr(dart_gptr_t *gptr, int32_t offs);
  */
 dart_ret_t dart_gptr_setunit(dart_gptr_t *gptr, dart_global_unit_t unit);
 
+/**
+ * Get the flags field for the segment specified by the global pointer.
+ *
+ * \param gptr Global Pointer describing a segment.
+ * \param unit The flags to get for segment in \c gptr
+ *
+ * \return \c DART_OK on success, any other of \ref dart_ret_t otherwise.
+ *
+ * \threadsafe
+ * \ingroup DartGlobMem
+ */
+dart_ret_t dart_gptr_getflags(dart_gptr_t gptr, uint16_t *flags);
+
+
+/**
+ * Set the flags field for the segment specified by the global pointer.
+ *
+ * \param gptr Global Pointer describing a segment.
+ * \param unit The flags to set for segment in \c gptr
+ *
+ * \return \c DART_OK on success, any other of \ref dart_ret_t otherwise.
+ *
+ * \threadsafe
+ * \ingroup DartGlobMem
+ */
+dart_ret_t dart_gptr_setflags(dart_gptr_t gptr, uint16_t flags);
+
 /**
  * Allocates memory for \c nelem elements of type \c dtype in the global
  * address space of the calling unit and returns a global pointer to it.
@@ -264,7 +301,6 @@ dart_ret_t dart_team_memalloc_aligned(
  * \ingroup DartGlobMem
  */
 dart_ret_t dart_team_memfree(
-  dart_team_t teamid,
   dart_gptr_t gptr);
 
 /**
@@ -330,7 +366,7 @@ dart_ret_t dart_team_memregister(
  * \threadsafe_none
  * \ingroup DartGlobMem
  */
-dart_ret_t dart_team_memderegister(dart_team_t teamid, dart_gptr_t gptr);
+dart_ret_t dart_team_memderegister(dart_gptr_t gptr);
 
 
 /** \cond DART_HIDDEN_SYMBOLS */

dart-if/v3.2/include/dash/dart/if/dart_types.h

@@ -207,7 +207,7 @@ dart_global_unit_t dart_create_global_unit(dart_unit_t unit)
  * Data type for storing a team ID
  * \ingroup DartTypes
  */
-typedef int32_t dart_team_t;
+typedef int16_t dart_team_t;
 
 /**
  * Undefined team ID.

dart-impl/mpi/include/dash/dart/mpi/dart_globmem_priv.h

@@ -7,11 +7,10 @@
   do {                                                      \
     gptr_.unitid = gptrt_.unitid;                           \
     gptr_.segid  = gptrt_.segid;                            \
-    gptr_.flags  = gptrt_.flags;                            \
+    gptr_.teamid = gptrt_.teamid;                           \
     gptr_.addr_or_offs.offset = gptrt_.addr_or_offs.offset; \
   } while(0)
 
-
 /* Global object for one-sided communication on memory region allocated with 'local allocation'. */
 extern MPI_Win dart_win_local_alloc;
 #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)

dart-impl/mpi/include/dash/dart/mpi/dart_segment.h

@@ -13,6 +13,8 @@
 
 typedef int16_t dart_segid_t;
 
+#define DART_SEGMENT_HASH_SIZE 256
+
 typedef struct
 {
   dart_segid_t seg_id; /* seg_id determines a global pointer uniquely */
@@ -21,28 +23,38 @@ typedef struct
   char      ** baseptr;
   char       * selfbaseptr;
   MPI_Win      win;
+  uint16_t     flags;
 } dart_segment_info_t;
 
+// forward declaration to make the compiler happy
+typedef struct dart_seghash_elem dart_seghash_elem_t;
+
+struct dart_seghash_elem {
+  dart_seghash_elem_t *next;
+  dart_segment_info_t  data;
+  int32_t              seg_id; // use int32_t internally to signal "invalid segment id"
+};
+
+typedef struct {
+
+  dart_seghash_elem_t* hashtab[DART_SEGMENT_HASH_SIZE];
+  dart_team_t team_id;
+
+  // TODO: Add free-segment-ID list here
+
+} dart_segmentdata_t;
+
 
 /**
  * @brief Initialize the segment data hash table.
  */
-dart_ret_t dart_segment_init();
+dart_ret_t dart_segment_init(dart_segmentdata_t *segdata, dart_team_t teamid);
 
 /**
  * @brief Allocates a new segment data struct. May be served from a freelist.
  */
-dart_ret_t dart_segment_alloc(dart_segid_t segid, uint16_t team_idx);
+dart_ret_t dart_segment_alloc(dart_segmentdata_t *segdata, const dart_segment_info_t *item);
 
-/**
- * @brief Returns the registered team index for the segment ID.
- */
-dart_ret_t dart_segment_get_teamidx(dart_segid_t segid, uint16_t *team_idx);
-
-/**
- * @brief Add segment information to the segment hash table.
- */
-dart_ret_t dart_segment_add_info(const dart_segment_info_t *item);
 
 #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
 /** @brief Query the shared memory window object associated with the specified seg_id.
@@ -53,17 +65,19 @@ dart_ret_t dart_segment_add_info(const dart_segment_info_t *item);
  *  @retval non-negative integer Search successfully.
  *  @retval negative integer Failure.
  */
-dart_ret_t dart_segment_get_win(int16_t seg_id, MPI_Win * win);
+dart_ret_t dart_segment_get_win(dart_segmentdata_t *segdata, int16_t seg_id, MPI_Win * win);
 
 dart_ret_t dart_segment_get_baseptr(
+  dart_segmentdata_t   * segdata,
   int16_t                seg_id,
   dart_team_unit_t       rel_unitid,
   char               **  baseptr_s);
 #endif
 
 dart_ret_t dart_segment_get_selfbaseptr(
-  int16_t    seg_id,
-  char   **  baseptr);
+  dart_segmentdata_t * segdata,
+  int16_t              seg_id,
+  char              ** baseptr);
 
 /** @brief Query the address of the memory location of the specified rel_unit in specified team.
  *
@@ -73,28 +87,41 @@ dart_ret_t dart_segment_get_selfbaseptr(
  *
  *  @retval ditto
  */
-dart_ret_t dart_segment_get_disp(int16_t             seg_id,
-                                 dart_team_unit_t    rel_unitid,
-                                 MPI_Aint          * disp_s);
+dart_ret_t dart_segment_get_disp(dart_segmentdata_t * segdata,
+                                 int16_t              seg_id,
+                                 dart_team_unit_t     rel_unitid,
+                                 MPI_Aint           * disp_s);
 
 /** @brief Query the length of the global memory block indicated by the specified seg_id.
  *
  *  @retval ditto
  */
 dart_ret_t dart_segment_get_size(
-  int16_t   seg_id,
-  size_t  * size);
+  dart_segmentdata_t * segdata,
+  int16_t              seg_id,
+  size_t             * size);
+
+dart_ret_t dart_segment_get_flags(
+  dart_segmentdata_t * segdata,
+  int16_t              seg_id,
+  uint16_t           * flags);
+
+dart_ret_t dart_segment_set_flags(
+  dart_segmentdata_t * segdata,
+  int16_t              seg_id,
+  uint16_t             flags);
 
 /**
  * @brief Deallocates the segment identified by the segment ID.
  */
-dart_ret_t dart_segment_free(dart_segid_t segid);
+dart_ret_t dart_segment_free(dart_segmentdata_t * segdata,
+                             dart_segid_t         segid);
 
 
 /**
  * @brief Clear the segment data hash table.
  */
-dart_ret_t dart_segment_fini();
+dart_ret_t dart_segment_fini(dart_segmentdata_t *segdata);
 
 
 #endif /* DART_SEGMENT_H_ */

dart-impl/mpi/include/dash/dart/mpi/dart_team_private.h

@@ -141,6 +141,7 @@
 #include <mpi.h>
 #include <dash/dart/base/logging.h>
 #include <dash/dart/mpi/dart_mem.h>
+#include <dash/dart/mpi/dart_segment.h>
 
 extern dart_team_t dart_next_availteamid;
 
@@ -159,6 +160,20 @@ typedef struct dart_team_data {
    */
   MPI_Win window;
 
+  dart_segmentdata_t segdata;
+
+  /**
+   * For DART collective allocation/free: offset in the returned gptr
+   * represents the displacement relative to the beginning of sub-memory
+   * spanned by a DART collective allocation.
+   * For DART local allocation/free: offset in the returned gptr represents
+   * the displacement relative to the base address of memory region reserved
+   * for the dart local allocation/free (see dart_buddy_allocator).
+   * Local allocations are identified by Segment ID DART_SEGMENT_LOCAL.
+   */
+  int16_t dart_memid;
+  int16_t dart_registermemid;
+
 #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
   /**
    * @brief Store the sub-communicator with regard to certain node, where the units can