fabric: Add fi_hmem_attr to fi_info structure

FI_HMEM is only an on/off capability bit but there are more
specific capabilities. Add fi_hmem_attr to fi_info to provide
better control over:
- whether device API calls are permitted in libfabric, except for
FI_HMEM initialization.
- whether peer to peer transfers should be used.
- whether optimized memcpy for device mempry should be used.

This change results in the ABI needing to be updated.  A new
compatibility set of interfaces is the result.

Signed-off-by: Jessie Yang <[email protected]>

Author: jiaxiyan

include/ofi_abi.h

@@ -111,7 +111,7 @@ extern "C" {
  * name appended with the ABI version that it is compatible with.
  */
 
-#define CURRENT_ABI "FABRIC_1.7"
+#define CURRENT_ABI "FABRIC_1.8"
 
 #if  HAVE_ALIAS_ATTRIBUTE == 1
 #define DEFAULT_SYMVER_PRE(a) a##_

include/ofi_util.h

@@ -1113,6 +1113,7 @@ int ofi_check_rx_attr(const struct fi_provider *prov,
 int ofi_check_tx_attr(const struct fi_provider *prov,
 		      const struct fi_tx_attr *prov_attr,
 		      const struct fi_tx_attr *user_attr, uint64_t info_mode);
+int ofi_check_hmem_attr(const struct fi_provider *prov, const struct fi_info *user_info);
 int ofi_check_attr_subset(const struct fi_provider *prov,
 			uint64_t base_caps, uint64_t requested_caps);
 int ofi_prov_check_info(const struct util_prov *util_prov,

include/rdma/fabric.h

@@ -360,6 +360,22 @@ enum {
 	FI_TC_NETWORK_CTRL,
 };
 
+enum fi_hmem_iface {
+	FI_HMEM_SYSTEM	= 0,
+	FI_HMEM_CUDA,
+	FI_HMEM_ROCR,
+	FI_HMEM_ZE,
+	FI_HMEM_NEURON,
+	FI_HMEM_SYNAPSEAI,
+};
+
+enum fi_hmem_attr_opt {
+	FI_HMEM_ATTR_ENABLED,
+	FI_HMEM_ATTR_REQUIRED,
+	FI_HMEM_ATTR_PREFERRED,
+	FI_HMEM_ATTR_DISABLED
+};
+
 static inline uint32_t fi_tc_dscp_set(uint8_t dscp)
 {
 	return ((uint32_t) dscp) | FI_TC_DSCP;
@@ -465,6 +481,14 @@ struct fi_fabric_attr {
 	uint32_t		api_version;
 };
 
+struct fi_hmem_attr {
+	enum fi_hmem_iface		iface;
+	enum fi_hmem_attr_opt		api_permitted;
+	enum fi_hmem_attr_opt		use_p2p;
+	enum fi_hmem_attr_opt		use_dev_reg_copy;
+	struct fi_hmem_attr		*next;
+};
+
 struct fi_info {
 	struct fi_info		*next;
 	uint64_t		caps;
@@ -480,6 +504,7 @@ struct fi_info {
 	struct fi_ep_attr	*ep_attr;
 	struct fi_domain_attr	*domain_attr;
 	struct fi_fabric_attr	*fabric_attr;
+	struct fi_hmem_attr	*hmem_attr;
 	struct fid_nic		*nic;
 };
 
@@ -771,6 +796,7 @@ enum fi_type {
 	FI_TYPE_MR_ATTR,
 	FI_TYPE_CNTR_ATTR,
 	FI_TYPE_CQ_ERR_ENTRY,
+	FI_TYPE_HMEM_ATTR,
 };
 
 char *fi_tostr(const void *data, enum fi_type datatype);

include/rdma/fi_domain.h

@@ -128,15 +128,6 @@ struct fid_mr {
 	uint64_t		key;
 };
 
-enum fi_hmem_iface {
-	FI_HMEM_SYSTEM	= 0,
-	FI_HMEM_CUDA,
-	FI_HMEM_ROCR,
-	FI_HMEM_ZE,
-	FI_HMEM_NEURON,
-	FI_HMEM_SYNAPSEAI,
-};
-
 static inline int fi_hmem_ze_device(int driver_index, int device_index)
 {
 	return driver_index << 16 | device_index;

man/fabric.7.md

@@ -447,6 +447,14 @@ attributes:
 *fi_domain_attr*
 : Added max_ep_auth_key
 
+## ABI 1.8
+ABI version starting with libfabric 2.0. Expanded the following structure.
+
+*fi_info*
+: The fi_info structure was expanded to reference a new fabric object,
+  fi_hmem_attr.  When available, the fi_hmem_attr references a new set of 
+  attributes related to heterogeneous memory.
+
 # SEE ALSO
 
 [`fi_info`(1)](fi_info.1.html),

man/fi_fabric.3.md

@@ -177,6 +177,9 @@ datatype or field value.
 *FI_TYPE_LOG_SUBSYS*
 : enum fi_log_subsys
 
+*FI_TYPE_HMEM_ATTR*
+: struct fi_hmem_attr
+
 fi_tostr() will return a pointer to an internal libfabric buffer that
 should not be modified, and will be overwritten the next time
 fi_tostr() is invoked.  fi_tostr() is not thread safe.

man/fi_getinfo.3.md

@@ -142,6 +142,7 @@ struct fi_info {
 	struct fi_ep_attr     *ep_attr;
 	struct fi_domain_attr *domain_attr;
 	struct fi_fabric_attr *fabric_attr;
+	struct fi_hmem_attr   *hmem_attr;
 	struct fid_nic        *nic;
 };
 ```
@@ -242,6 +243,13 @@ struct fi_info {
   output, the actual fabric attributes that can be provided will be
   returned.  See [`fi_fabric`(3)](fi_fabric.3.html) for details.
 
+*hmem_attr - heterogeneous memory attributes*
+: Optionally supplied HMEM attributes.  HMEM attributes may be
+  specified and returned as part of fi_getinfo.  When provided as
+  hints, requested values of struct fi_hmem_attr should be set.  On
+  output, the actual HMEM attributes that can be provided will be
+  returned.
+
 *nic - network interface details*
 : Optional attributes related to the hardware NIC associated with
   the specified fabric, domain, and endpoint data.  This field is

man/fi_info.1.md

@@ -200,6 +200,11 @@ fi_info:
         prov_name: verbs;ofi_rxd
         prov_version: 116.0
         api_version: 1.16
+    fi_hmem_attr:
+        iface:
+        api_permitted:
+        use_p2p:
+        use_dev_reg_copy:
     nic:
         fi_device_attr:
             name: mlx5_0

prov/util/src/util_attr.c

@@ -1002,6 +1002,17 @@ int ofi_check_tx_attr(const struct fi_provider *prov,
 	return 0;
 }
 
+int ofi_check_hmem_attr(const struct fi_provider *prov,
+			const struct fi_info *user_info)
+{
+	if (!(user_info->caps & FI_HMEM)) {
+		FI_INFO(prov, FI_LOG_CORE, "FI_HMEM not set\n");
+		return -FI_ENODATA;
+	}
+
+	return 0;
+}
+
 /* Use if there are multiple fi_info in the provider:
  * check provider's info */
 int ofi_prov_check_info(const struct util_prov *util_prov,
@@ -1152,6 +1163,13 @@ int ofi_check_info(const struct util_prov *util_prov,
 		if (ret)
 			return ret;
 	}
+
+	if (user_info->hmem_attr) {
+		ret = ofi_check_hmem_attr(prov, user_info);
+		if (ret)
+			return ret;
+	}
+
 	return 0;
 }
 
@@ -1271,6 +1289,24 @@ static void fi_alter_tx_attr(struct fi_tx_attr *attr,
 		attr->rma_iov_limit = hints->rma_iov_limit;
 }
 
+static void fi_alter_hmem_attr(struct fi_hmem_attr *attr,
+			     const struct fi_hmem_attr *hints)
+{
+	if (!hints)
+		return;
+
+	if (hints->iface)
+		attr->iface = hints->iface;
+	if (hints->api_permitted)
+		attr->api_permitted = hints->api_permitted;
+	if (hints->use_p2p)
+		attr->use_p2p = hints->use_p2p;
+	if (hints->use_dev_reg_copy)
+		attr->use_dev_reg_copy = hints->use_dev_reg_copy;
+	if (hints->next)
+		attr->next = hints->next;
+}
+
 static uint64_t ofi_get_info_caps(const struct fi_info *prov_info,
 				  const struct fi_info *user_info,
 				  uint32_t api_version)
@@ -1336,5 +1372,6 @@ void ofi_alter_info(struct fi_info *info, const struct fi_info *hints,
 				 info->caps);
 		fi_alter_tx_attr(info->tx_attr, hints ? hints->tx_attr : NULL,
 				 info->caps);
+		fi_alter_hmem_attr(info->hmem_attr, hints ? hints->hmem_attr : NULL);
 	}
 }

src/abi_1_0.c

@@ -305,6 +305,40 @@ struct fi_info_1_7 {
         struct fid_nic_1_7        *nic;
 };
 
+struct fi_hmem_attr_1_8 {
+	enum fi_hmem_iface iface;
+	enum fi_hmem_attr_opt api_permitted;
+	enum fi_hmem_attr_opt use_p2p;
+	enum fi_hmem_attr_opt use_dev_reg_copy;
+	struct fi_hmem_attr *next;
+};
+
+#define fi_tx_attr_1_8 fi_tx_attr_1_7
+#define fi_rx_attr_1_8 fi_rx_attr_1_7
+#define fi_ep_attr_1_8 fi_ep_attr_1_7
+#define fi_domain_attr_1_8 fi_domain_attr_1_7
+#define fi_fabric_attr_1_8 fi_fabric_attr_1_7
+#define fid_nic_1_8 fid_nic_1_7
+
+struct fi_info_1_8 {
+        struct fi_info            *next;
+        uint64_t                  caps;
+        uint64_t                  mode;
+        uint32_t                  addr_format;
+        size_t                    src_addrlen;
+        size_t                    dest_addrlen;
+        void                      *src_addr;
+        void                      *dest_addr;
+        fid_t                     handle;
+        struct fi_tx_attr_1_8     *tx_attr;
+        struct fi_rx_attr_1_8     *rx_attr;
+        struct fi_ep_attr_1_8     *ep_attr;
+        struct fi_domain_attr_1_8 *domain_attr;
+        struct fi_fabric_attr_1_8 *fabric_attr;
+        struct fi_hmem_attr_1_8   *hmem_attr;
+        struct fid_nic_1_8        *nic;
+};
+
 #define ofi_dup_attr(dst, src)				\
 	do {						\
 		dst = calloc(1, sizeof(*dst));		\

src/fabric.c

@@ -1041,6 +1041,7 @@ __attribute__((visibility ("default"),EXTERNALLY_VISIBLE))
 void DEFAULT_SYMVER_PRE(fi_freeinfo)(struct fi_info *info)
 {
 	struct fi_info *next;
+	struct fi_hmem_attr *next_hmem_attr;
 
 	for (; info; info = next) {
 		next = info->next;
@@ -1063,6 +1064,12 @@ void DEFAULT_SYMVER_PRE(fi_freeinfo)(struct fi_info *info)
 			free(info->fabric_attr->prov_name);
 			free(info->fabric_attr);
 		}
+		if (info->hmem_attr) {
+			for (; info->hmem_attr; info->hmem_attr = next_hmem_attr) {
+				next_hmem_attr = info->hmem_attr->next;
+				free(info->hmem_attr);
+			}
+		}
 		if (info->nic &&
 		    FI_CHECK_OP(info->nic->fid.ops, struct fi_ops, close)) {
 			fi_close(&info->nic->fid);
@@ -1303,6 +1310,21 @@ static int ofi_layering_ok(const struct fi_provider *provider,
 	return !strcasecmp(provider->name, prov_name);
 }
 
+static struct fi_hmem_attr *dup_hmem_attr(const struct fi_hmem_attr *hmem_attr)
+{
+	if (!hmem_attr)
+		return NULL;
+
+	struct fi_hmem_attr *dup =
+		(struct fi_hmem_attr *) mem_dup(hmem_attr, sizeof(*hmem_attr));
+	if (!dup)
+		return NULL;
+
+	dup->next = dup_hmem_attr(hmem_attr->next);
+
+	return dup;
+}
+
 __attribute__((visibility ("default"),EXTERNALLY_VISIBLE))
 int DEFAULT_SYMVER_PRE(fi_getinfo)(uint32_t version, const char *node,
 		const char *service, uint64_t flags,
@@ -1423,8 +1445,9 @@ struct fi_info *ofi_allocinfo_internal(void)
 	info->ep_attr = calloc(1, sizeof(*info->ep_attr));
 	info->domain_attr = calloc(1, sizeof(*info->domain_attr));
 	info->fabric_attr = calloc(1, sizeof(*info->fabric_attr));
-	if (!info->tx_attr|| !info->rx_attr || !info->ep_attr ||
-	    !info->domain_attr || !info->fabric_attr)
+	info->hmem_attr = calloc(1, sizeof(*info->hmem_attr));
+	if (!info->tx_attr || !info->rx_attr || !info->ep_attr ||
+	    !info->domain_attr || !info->fabric_attr || !info->hmem_attr)
 		goto err;
 
 	return info;
@@ -1454,6 +1477,7 @@ struct fi_info *DEFAULT_SYMVER_PRE(fi_dupinfo)(const struct fi_info *info)
 	dup->ep_attr = NULL;
 	dup->domain_attr = NULL;
 	dup->fabric_attr = NULL;
+	dup->hmem_attr = NULL;
 	dup->next = NULL;
 
 	if (info->src_addr != NULL) {
@@ -1526,7 +1550,11 @@ struct fi_info *DEFAULT_SYMVER_PRE(fi_dupinfo)(const struct fi_info *info)
 				goto fail;
 		}
 	}
-
+	if (info->hmem_attr) {
+		dup->hmem_attr = dup_hmem_attr(info->hmem_attr);
+		if (dup->hmem_attr == NULL)
+			goto fail;
+	}
 	if (info->nic) {
 		ret = fi_control(&info->nic->fid, FI_DUP, &dup->nic);
 		if (ret && ret != -FI_ENOSYS)