prov/efa: Make efa_hmem_info a global variable

Currently efa_hmem_info is part of efa_domain and created for every
efa domain. hmem_info init involves several operations like device
memory allocation / free, and trial ibv reg mr, which is expensive
and can potentially cause more memory usage. Make efa_hmem_info
a global variable and call it only once per process.

Signed-off-by: Jessie Yang <[email protected]>

Author: jiaxiyan

prov/efa/src/efa_domain.c

@@ -297,13 +297,6 @@ int efa_domain_open(struct fid_fabric *fabric_fid, struct fi_info *info,
 		goto err_free;
 	}
 
-	err = efa_domain_hmem_info_init_all(efa_domain);
-	if (err) {
-		ret = err;
-		EFA_WARN(FI_LOG_DOMAIN, "Failed to check hmem support status. err: %d\n", ret);
-		goto err_free;
-	}
-
 	dlist_insert_tail(&efa_domain->list_entry, &g_efa_domain_list);
 	return 0;
 

prov/efa/src/efa_domain.h

@@ -22,7 +22,6 @@ struct efa_domain {
 	struct ofi_mr_cache	*cache;
 	struct efa_qp		**qp_table;
 	size_t			qp_table_sz_m1;
-	struct efa_hmem_info	hmem_info[OFI_HMEM_MAX];
 	size_t			mtu_size;
 	size_t			addrlen;
 	bool 			mr_local;

prov/efa/src/efa_hmem.c

@@ -5,16 +5,18 @@
 #include "efa_hmem.h"
 #include "rdm/efa_rdm_pkt_type.h"
 
+struct efa_hmem_info g_efa_hmem_info[OFI_HMEM_MAX];
+
 #if HAVE_CUDA || HAVE_NEURON
-static size_t efa_max_eager_msg_size_with_largest_header(struct efa_domain *efa_domain) {
+static size_t efa_max_eager_msg_size_with_largest_header() {
 	int mtu_size;
 
-	mtu_size = efa_domain->device->rdm_info->ep_attr->max_msg_size;
+	mtu_size = g_device_list[0].rdm_info->ep_attr->max_msg_size;
 
 	return mtu_size - efa_rdm_pkt_type_get_max_hdr_size();
 }
 #else
-static size_t efa_max_eager_msg_size_with_largest_header(struct efa_domain *efa_domain) {
+static size_t efa_max_eager_msg_size_with_largest_header() {
 	return 0;
 }
 #endif
@@ -23,14 +25,13 @@ static size_t efa_max_eager_msg_size_with_largest_header(struct efa_domain *efa_
  * @brief  Initialize the various protocol thresholds tracked in efa_hmem_info
  *         according to the given FI_HMEM interface.
  *
- * @param[in,out]  efa_domain  Pointer to struct efa_domain
  * @param[in]      iface       The FI_HMEM interface to initialize
  *
  * @return  0
  */
-static int efa_domain_hmem_info_init_protocol_thresholds(struct efa_domain *efa_domain, enum fi_hmem_iface iface)
+static int efa_domain_hmem_info_init_protocol_thresholds(enum fi_hmem_iface iface)
 {
-	struct efa_hmem_info *info = &efa_domain->hmem_info[iface];
+	struct efa_hmem_info *info = g_efa_hmem_info[iface];
 	size_t tmp_value;
 
 	/* Fall back to FI_HMEM_SYSTEM initialization logic when p2p is
@@ -53,8 +54,8 @@ static int efa_domain_hmem_info_init_protocol_thresholds(struct efa_domain *efa_
 	case FI_HMEM_CUDA:
 		info->runt_size = EFA_DEFAULT_RUNT_SIZE;
 		info->max_medium_msg_size = 0;
-		info->min_read_msg_size = efa_max_eager_msg_size_with_largest_header(efa_domain) + 1;
-		info->min_read_write_size = efa_max_eager_msg_size_with_largest_header(efa_domain) + 1;
+		info->min_read_msg_size = efa_max_eager_msg_size_with_largest_header() + 1;
+		info->min_read_write_size = efa_max_eager_msg_size_with_largest_header() + 1;
 		fi_param_get_size_t(&efa_prov, "runt_size", &info->runt_size);
 		fi_param_get_size_t(&efa_prov, "inter_min_read_message_size", &info->min_read_msg_size);
 		fi_param_get_size_t(&efa_prov, "inter_min_read_write_size", &info->min_read_write_size);
@@ -68,8 +69,8 @@ static int efa_domain_hmem_info_init_protocol_thresholds(struct efa_domain *efa_
 	case FI_HMEM_NEURON:
 		info->runt_size = EFA_NEURON_RUNT_SIZE;
 		info->max_medium_msg_size = 0;
-		info->min_read_msg_size = efa_max_eager_msg_size_with_largest_header(efa_domain) + 1;
-		info->min_read_write_size = efa_max_eager_msg_size_with_largest_header(efa_domain) + 1;
+		info->min_read_msg_size = efa_max_eager_msg_size_with_largest_header() + 1;
+		info->min_read_write_size = efa_max_eager_msg_size_with_largest_header() + 1;
 		fi_param_get_size_t(&efa_prov, "runt_size", &info->runt_size);
 		fi_param_get_size_t(&efa_prov, "inter_min_read_message_size", &info->min_read_msg_size);
 		fi_param_get_size_t(&efa_prov, "inter_min_read_write_size", &info->min_read_write_size);
@@ -105,7 +106,7 @@ static int efa_domain_hmem_info_init_protocol_thresholds(struct efa_domain *efa_
 	return 0;
 }
 
-static inline void efa_domain_hmem_info_check_p2p_support_cuda(struct efa_hmem_info *info) {
+static inline void efa_hmem_info_check_p2p_support_cuda(struct efa_hmem_info *info) {
 #if HAVE_CUDA
 	cudaError_t cuda_ret;
 	void *ptr = NULL;
@@ -168,7 +169,7 @@ static inline void efa_domain_hmem_info_check_p2p_support_cuda(struct efa_hmem_i
 	return;
 }
 
-static inline void efa_domain_hmem_info_check_p2p_support_neuron(struct efa_hmem_info *info) {
+static inline void efa_hmem_info_check_p2p_support_neuron(struct efa_hmem_info *info) {
 #if HAVE_NEURON
 	struct ibv_mr *ibv_mr = NULL;
 	int ibv_access = IBV_ACCESS_LOCAL_WRITE;
@@ -239,13 +240,12 @@ static inline void efa_domain_hmem_info_check_p2p_support_neuron(struct efa_hmem
 /**
  * @brief Initialize the efa_hmem_info state for iface
  *
- * @param[in,out]  efa_domain  Pointer to struct efa_domain
  * @param[in]      iface       HMEM interface
  */
 static void
-efa_domain_hmem_info_init_iface(struct efa_domain *efa_domain, enum fi_hmem_iface iface)
+efa_hmem_info_init_iface(enum fi_hmem_iface iface)
 {
-	struct efa_hmem_info *info = &efa_domain->hmem_info[iface];
+	struct efa_hmem_info *info = g_efa_hmem_info[iface];
 
 	if (!ofi_hmem_is_initialized(iface)) {
 		EFA_INFO(FI_LOG_DOMAIN, "%s is not initialized\n",
@@ -262,17 +262,16 @@ efa_domain_hmem_info_init_iface(struct efa_domain *efa_domain, enum fi_hmem_ifac
 	}
 
 	info->initialized = true;
-	info->p2p_disabled_by_user = (iface == FI_HMEM_SYSTEM) ? false : ofi_hmem_p2p_disabled();
 
 	if (iface == FI_HMEM_SYNAPSEAI || iface == FI_HMEM_SYSTEM) {
 		info->p2p_supported_by_device = true;
-	} else if (info->p2p_disabled_by_user) {
+	} else if (ofi_hmem_p2p_disabled()) {
 		info->p2p_supported_by_device = false;
 	} else {
 		if (iface == FI_HMEM_CUDA)
-			efa_domain_hmem_info_check_p2p_support_cuda(info);
+			efa_hmem_info_check_p2p_support_cuda(info);
 		if (iface == FI_HMEM_NEURON)
-			efa_domain_hmem_info_check_p2p_support_neuron(info);
+			efa_hmem_info_check_p2p_support_neuron(info);
 		if (!info->p2p_supported_by_device)
 			EFA_INFO(FI_LOG_DOMAIN, "%s P2P support is not available.\n", fi_tostr(&iface, FI_TYPE_HMEM_IFACE));
 	}
@@ -283,20 +282,18 @@ efa_domain_hmem_info_init_iface(struct efa_domain *efa_domain, enum fi_hmem_ifac
 	* memory, therefore p2p is not required.
 	*/
 	if (iface == FI_HMEM_CUDA &&
-	    FI_VERSION_GE(efa_domain->util_domain.fabric->fabric_fid.api_version, FI_VERSION(1, 18)))
+	    FI_VERSION_GE(g_device_list[0].rdm_info->fabric_attr.api_version, FI_VERSION(1, 18)))
 		info->p2p_required_by_impl = !hmem_ops[iface].initialized;
 	if (iface == FI_HMEM_SYSTEM)
 		info->p2p_required_by_impl = false;
 
-	efa_domain_hmem_info_init_protocol_thresholds(efa_domain, iface);
+	efa_domain_hmem_info_init_protocol_thresholds(iface);
 }
 
 /**
  * @brief   Validate an FI_OPT_FI_HMEM_P2P (FI_OPT_ENDPOINT) option for a
  *          specified HMEM interface.
- *          Also update hmem_info[iface]->p2p_disabled_by_user accordingly.
  *
- * @param[in,out]  domain   The efa_domain struct which contains an efa_hmem_info array
  * @param[in]      iface    The fi_hmem_iface enum of the FI_HMEM interface to validate
  * @param[in]      p2p_opt  The P2P option to validate
  *
@@ -305,9 +302,9 @@ efa_domain_hmem_info_init_iface(struct efa_domain *efa_domain, enum fi_hmem_ifac
  *         -FI_ENODATA if the given HMEM interface was not initialized
  *         -FI_EINVAL if p2p_opt is not a valid FI_OPT_FI_HMEM_P2P option
  */
-int efa_domain_hmem_validate_p2p_opt(struct efa_domain *efa_domain, enum fi_hmem_iface iface, int p2p_opt)
+int efa_domain_hmem_validate_p2p_opt(enum fi_hmem_iface iface, int p2p_opt)
 {
-	struct efa_hmem_info *info = &efa_domain->hmem_info[iface];
+	struct efa_hmem_info *info = g_efa_hmem_info[iface];
 
 	if (OFI_UNLIKELY(!info->initialized))
 		return -FI_ENODATA;
@@ -317,7 +314,6 @@ int efa_domain_hmem_validate_p2p_opt(struct efa_domain *efa_domain, enum fi_hmem
 		if (OFI_UNLIKELY(ofi_hmem_p2p_disabled()) || !info->p2p_supported_by_device)
 			return -FI_EOPNOTSUPP;
 
-		info->p2p_disabled_by_user = false;
 		return 0;
 	/*
 	 * According to fi_setopt() document:
@@ -334,43 +330,39 @@ int efa_domain_hmem_validate_p2p_opt(struct efa_domain *efa_domain, enum fi_hmem
 		if (OFI_UNLIKELY(ofi_hmem_p2p_disabled()))
 			return -FI_EOPNOTSUPP;
 
-		info->p2p_disabled_by_user = false;
 		return 0;
 
 	case FI_HMEM_P2P_DISABLED:
 		if (info->p2p_required_by_impl)
 			return -FI_EOPNOTSUPP;
 
-		info->p2p_disabled_by_user = true;
 		return 0;
 	}
 
 	return -FI_EINVAL;
 }
 
 /**
- * @brief Initialize the hmem_info structs for
+ * @brief Initialize the g_efa_hmem_info structs for
  * all of the HMEM devices. The device hmem_info
  * struct will be used to determine which efa transfer
  * protocol should be selected.
  *
- * @param[in,out]  efa_domain  Pointer to struct efa_domain to be initialized
- *
  * @return  0 on success
  *          negative libfabric error code on an unexpected error
  */
-int efa_domain_hmem_info_init_all(struct efa_domain *efa_domain)
+int efa_hmem_info_initialize()
 {
 	int ret = 0, i = 0;
 
 	if(g_device_cnt <= 0) {
 		return -FI_ENODEV;
 	}
 
-	memset(efa_domain->hmem_info, 0, OFI_HMEM_MAX * sizeof(struct efa_hmem_info));
+	memset(g_efa_hmem_info, 0, OFI_HMEM_MAX * sizeof(struct efa_hmem_info));
 
 	EFA_HMEM_IFACE_FOREACH(i) {
-		efa_domain_hmem_info_init_iface(efa_domain, efa_hmem_ifaces[i]);
+		efa_hmem_info_init_iface(efa_hmem_ifaces[i]);
 	}
 
 	return ret;

prov/efa/src/efa_hmem.h

@@ -23,7 +23,6 @@ static const enum fi_hmem_iface efa_hmem_ifaces[] = {
 
 struct efa_hmem_info {
 	bool initialized; 	/* do we support it at all */
-	bool p2p_disabled_by_user;	/* Did the user disable p2p via FI_OPT_FI_HMEM_P2P? */
 	bool p2p_required_by_impl;	/* Is p2p required for this interface? */
 	bool p2p_supported_by_device;	/* do we support p2p with this device */
 
@@ -33,10 +32,12 @@ struct efa_hmem_info {
 	size_t min_read_write_size;
 };
 
+extern struct efa_hmem_info	g_efa_hmem_info[OFI_HMEM_MAX];
+
 struct efa_domain;
 
-int efa_domain_hmem_validate_p2p_opt(struct efa_domain *efa_domain, enum fi_hmem_iface iface, int p2p_opt);
-int efa_domain_hmem_info_init_all(struct efa_domain *efa_domain);
+int efa_domain_hmem_validate_p2p_opt(enum fi_hmem_iface iface, int p2p_opt);
+int efa_hmem_info_initialize();
 
 /**
  * @brief Copy data from a hmem device to a system buffer

prov/efa/src/efa_mr.c

@@ -192,7 +192,7 @@ static int efa_mr_hmem_setup(struct efa_mr *efa_mr,
 	}
 
 	if (efa_mr->domain->util_domain.info_domain_caps & FI_HMEM) {
-		if (efa_mr->domain->hmem_info[attr->iface].initialized) {
+		if (g_efa_hmem_info[attr->iface].initialized) {
 			efa_mr->peer.iface = attr->iface;
 		} else {
 			EFA_WARN(FI_LOG_MR,
@@ -813,7 +813,7 @@ static int efa_mr_reg_impl(struct efa_mr *efa_mr, uint64_t flags, const void *at
 	 * For FI_HMEM_CUDA iface when p2p is unavailable, skip ibv_reg_mr() and
 	 * generate proprietary mr_fid key.
 	 */
-	if (mr_attr.iface == FI_HMEM_CUDA && !efa_mr->domain->hmem_info[FI_HMEM_CUDA].p2p_supported_by_device) {
+	if (mr_attr.iface == FI_HMEM_CUDA && !g_efa_hmem_info[FI_HMEM_CUDA].p2p_supported_by_device) {
 		efa_mr->mr_fid.key = efa_mr_cuda_non_p2p_keygen();
 	} else {
 		efa_mr->ibv_mr = efa_mr_reg_ibv_mr(efa_mr, &mr_attr, fi_ibv_access, flags);

prov/efa/src/efa_prov.c

@@ -164,6 +164,10 @@ EFA_INI
 	if (err)
 		goto err_free;
 
+	err = efa_hmem_info_initialize();
+	if (err)
+		goto err_free;
+
 	dlist_init(&g_efa_domain_list);
 
 	return &efa_prov;

prov/efa/src/rdm/efa_rdm_ep.h

@@ -307,7 +307,7 @@ int efa_rdm_ep_use_p2p(struct efa_rdm_ep *efa_rdm_ep, struct efa_mr *efa_mr)
 	if (!efa_mr || efa_mr->peer.iface == FI_HMEM_SYSTEM)
 		return 1;
 
-	if (efa_rdm_ep_domain(efa_rdm_ep)->hmem_info[efa_mr->peer.iface].p2p_supported_by_device)
+	if (g_efa_hmem_info[efa_mr->peer.iface].p2p_supported_by_device)
 		return (efa_rdm_ep->hmem_p2p_opt != FI_HMEM_P2P_DISABLED);
 
 	if (efa_rdm_ep->hmem_p2p_opt == FI_HMEM_P2P_REQUIRED) {

prov/efa/src/rdm/efa_rdm_ep_fiops.c

@@ -483,9 +483,10 @@ void efa_rdm_ep_set_use_zcpy_rx(struct efa_rdm_ep *ep)
 
 	/* Zero-copy receive requires P2P support. Disable it if any initialized HMEM iface does not support P2P. */
 	for (iface = FI_HMEM_SYSTEM; iface < OFI_HMEM_MAX; ++iface) {
-		hmem_info = &ep->base_ep.domain->hmem_info[iface];
+		hmem_info = g_efa_hmem_info[iface];
 		if (hmem_info->initialized &&
-		    !hmem_info->p2p_disabled_by_user &&
+			!ofi_hmem_p2p_disabled() &&
+		    ep->hmem_p2p_opt != FI_HMEM_P2P_DISABLED &&
 		    !hmem_info->p2p_supported_by_device) {
 			EFA_INFO(FI_LOG_EP_CTRL,
 			         "%s does not support P2P, zero-copy receive "
@@ -615,9 +616,9 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info,
 	 * tighter requirements for the default p2p opt
 	 */
 	EFA_HMEM_IFACE_FOREACH_NON_SYSTEM(i) {
-		if (efa_rdm_ep->base_ep.domain->hmem_info[efa_hmem_ifaces[i]].initialized &&
-			efa_rdm_ep->base_ep.domain->hmem_info[efa_hmem_ifaces[i]].p2p_supported_by_device) {
-			efa_rdm_ep->hmem_p2p_opt = efa_rdm_ep->base_ep.domain->hmem_info[efa_hmem_ifaces[i]].p2p_required_by_impl
+		if (g_efa_hmem_info[efa_hmem_ifaces[i]].initialized &&
+			g_efa_hmem_info[efa_hmem_ifaces[i]].p2p_supported_by_device) {
+			efa_rdm_ep->hmem_p2p_opt = g_efa_hmem_info[efa_hmem_ifaces[i]].p2p_required_by_impl
 				? FI_HMEM_P2P_REQUIRED
 				: FI_HMEM_P2P_PREFERRED;
 			break;
@@ -1413,7 +1414,7 @@ static int efa_rdm_ep_set_fi_hmem_p2p_opt(struct efa_rdm_ep *efa_rdm_ep, int opt
 	 * tighter restrictions on valid p2p options.
 	 */
 	EFA_HMEM_IFACE_FOREACH_NON_SYSTEM(i) {
-		err = efa_domain_hmem_validate_p2p_opt(efa_rdm_ep_domain(efa_rdm_ep), efa_hmem_ifaces[i], opt);
+		err = efa_domain_hmem_validate_p2p_opt(efa_hmem_ifaces[i], opt);
 		if (err == -FI_ENODATA)
 			continue;
 
@@ -1449,7 +1450,7 @@ static int efa_rdm_ep_set_cuda_api_permitted(struct efa_rdm_ep *ep, bool cuda_ap
 	/* CUDA memory can be supported by using either peer to peer or CUDA API. If neither is
 	 * available, we cannot support CUDA memory
 	 */
-	if (!efa_rdm_ep_domain(ep)->hmem_info[FI_HMEM_CUDA].p2p_supported_by_device)
+	if (!g_efa_hmem_info[FI_HMEM_CUDA].p2p_supported_by_device)
 		return -FI_EOPNOTSUPP;
 
 	ep->cuda_api_permitted = false;

prov/efa/src/rdm/efa_rdm_msg.c

@@ -60,15 +60,13 @@ int efa_rdm_msg_select_rtm(struct efa_rdm_ep *efa_rdm_ep, struct efa_rdm_ope *tx
 	int tagged;
 	int eager_rtm, medium_rtm, longcts_rtm, readbase_rtm, iface;
 	size_t eager_rtm_max_data_size;
-	struct efa_hmem_info *hmem_info;
 	bool delivery_complete_requested;
 
 	assert(txe->op == ofi_op_msg || txe->op == ofi_op_tagged);
 	tagged = (txe->op == ofi_op_tagged);
 	assert(tagged == 0 || tagged == 1);
 
 	iface = txe->desc[0] ? ((struct efa_mr*) txe->desc[0])->peer.iface : FI_HMEM_SYSTEM;
-	hmem_info = efa_rdm_ep_domain(efa_rdm_ep)->hmem_info;
 
 	if (txe->fi_flags & FI_INJECT || efa_both_support_zero_hdr_data_transfer(efa_rdm_ep, txe->peer))
 		delivery_complete_requested = false;
@@ -89,15 +87,15 @@ int efa_rdm_msg_select_rtm(struct efa_rdm_ep *efa_rdm_ep, struct efa_rdm_ope *tx
 	readbase_rtm = efa_rdm_peer_select_readbase_rtm(txe->peer, efa_rdm_ep, txe);
 
 	if (use_p2p && 
-		txe->total_len >= hmem_info[iface].min_read_msg_size &&
+		txe->total_len >= g_efa_hmem_info[iface].min_read_msg_size &&
 		efa_rdm_interop_rdma_read(efa_rdm_ep, txe->peer) &&
 		(txe->desc[0] || efa_is_cache_available(efa_rdm_ep_domain(efa_rdm_ep))))
 		return readbase_rtm;
 
 	if (txe->total_len <= eager_rtm_max_data_size)
 		return eager_rtm;
 
-	if (txe->total_len <= hmem_info[iface].max_medium_msg_size)
+	if (txe->total_len <= g_efa_hmem_info[iface].max_medium_msg_size)
 		return medium_rtm;
 
 	return longcts_rtm;

prov/efa/src/rdm/efa_rdm_peer.c

@@ -330,18 +330,16 @@ void efa_rdm_peer_proc_pending_items_in_robuf(struct efa_rdm_peer *peer, struct
 size_t efa_rdm_peer_get_runt_size(struct efa_rdm_peer *peer,
 				  struct efa_rdm_ep *ep, struct efa_rdm_ope *ope)
 {
-	struct efa_hmem_info *hmem_info;
 	size_t runt_size;
 	size_t memory_alignment;
 	int iface;
 
-	hmem_info = efa_rdm_ep_domain(ep)->hmem_info;
 	iface = ope->desc[0] ? ((struct efa_mr*) ope->desc[0])->peer.iface : FI_HMEM_SYSTEM;
 
-	if (hmem_info[iface].runt_size < peer->num_runt_bytes_in_flight)
+	if (g_efa_hmem_info[iface].runt_size < peer->num_runt_bytes_in_flight)
 		return 0;
 
-	runt_size = MIN(hmem_info[iface].runt_size - peer->num_runt_bytes_in_flight, ope->total_len);
+	runt_size = MIN(g_efa_hmem_info[iface].runt_size - peer->num_runt_bytes_in_flight, ope->total_len);
 	memory_alignment = efa_rdm_ep_get_memory_alignment(ep, iface);
 	/*
 	 * runt size must be aligned because:

prov/efa/src/rdm/efa_rdm_rma.c

@@ -400,7 +400,7 @@ ssize_t efa_rdm_rma_post_write(struct efa_rdm_ep *ep, struct efa_rdm_ope *txe)
 	iface = txe->desc[0] ? ((struct efa_mr*) txe->desc[0])->peer.iface : FI_HMEM_SYSTEM;
 
 	if (use_p2p && 
-		txe->total_len >= efa_rdm_ep_domain(ep)->hmem_info[iface].min_read_write_size &&
+		txe->total_len >= g_efa_hmem_info[iface].min_read_write_size &&
 		efa_rdm_interop_rdma_read(ep, txe->peer) &&
 		(txe->desc[0] || efa_is_cache_available(efa_rdm_ep_domain(ep)))) {
 		err = efa_rdm_ope_post_send(txe, EFA_RDM_LONGREAD_RTW_PKT);