prov/efa: Use tclass to prioritize the messages from an ep

To prioritize the messages from a given endpoint, user can specify ` fi_info->tx_attr->tclass = FI_TC_LOW_LATENCY` in the fi_endpoint() call to set the service level in rdma-core. All other tclass values will be ignored. Signed-off-by: Jessie Yang <[email protected]>
jiaxiyan · Sep 25, 2024 · b2cfcce · b2cfcce
1 parent 16a4445
commit b2cfcce
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 5 deletions.
diff --git a/man/fi_efa.7.md b/man/fi_efa.7.md
@@ -205,6 +205,8 @@ struct fi_efa_mr_attr {
 **query_mr()** returns 0 on success, or the value of errno on failure
 (which indicates the failure reason).
 
+# Traffic Class (tclass) in EFA
+To prioritize the messages from a given endpoint, user can specify `fi_info->tx_attr->tclass = FI_TC_LOW_LATENCY` in the fi_endpoint() call to set the service level in rdma-core. All other tclass values will be ignored.
 
 # RUNTIME PARAMETERS
 

diff --git a/prov/efa/src/efa_base_ep.c b/prov/efa/src/efa_base_ep.c
@@ -167,7 +167,7 @@ static int efa_base_ep_modify_qp_rst2rts(struct efa_base_ep *base_ep,
  * @param init_attr_ex ibv_qp_init_attr_ex
  * @return int 0 on success, negative integer on failure
  */
-int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex)
+int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex, uint32_t tclass)
 {
 	struct efadv_qp_init_attr efa_attr = { 0 };
 
@@ -185,6 +185,10 @@ int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex)
 			efa_attr.flags |= EFADV_QP_FLAGS_UNSOLICITED_WRITE_RECV;
 #endif
 		efa_attr.driver_qp_type = EFADV_QP_DRIVER_TYPE_SRD;
+
+		if (tclass == FI_TC_LOW_LATENCY)
+			efa_attr.sl = tclass;
+
 		(*qp)->ibv_qp = efadv_create_qp_ex(
 			init_attr_ex->pd->context, init_attr_ex, &efa_attr,
 			sizeof(struct efadv_qp_init_attr));
@@ -206,7 +210,7 @@ int efa_base_ep_create_qp(struct efa_base_ep *base_ep,
 {
 	int ret;
 
-	ret = efa_qp_create(&base_ep->qp, init_attr_ex);
+	ret = efa_qp_create(&base_ep->qp, init_attr_ex, base_ep->info->tx_attr->tclass);
 	if (ret)
 		return ret;
 

diff --git a/prov/efa/src/efa_base_ep.h b/prov/efa/src/efa_base_ep.h
@@ -72,7 +72,7 @@ int efa_base_ep_construct(struct efa_base_ep *base_ep,
 
 int efa_base_ep_getname(fid_t fid, void *addr, size_t *addrlen);
 
-int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex);
+int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex, uint32_t tclass);
 
 void efa_qp_destruct(struct efa_qp *qp);
 

diff --git a/prov/efa/src/rdm/efa_rdm_ep_fiops.c b/prov/efa/src/rdm/efa_rdm_ep_fiops.c
@@ -105,7 +105,7 @@ int efa_rdm_ep_create_base_ep_ibv_qp(struct efa_rdm_ep *ep)
 	 * without any headers.
 	 */
 	if (ep->use_zcpy_rx) {
-		ret = efa_qp_create(&ep->base_ep.user_recv_qp, &attr_ex);
+		ret = efa_qp_create(&ep->base_ep.user_recv_qp, &attr_ex, ep->base_ep.info->tx_attr->tclass);
 		if (ret) {
 			efa_base_ep_destruct_qp(&ep->base_ep);
 			return ret;
@@ -1646,7 +1646,7 @@ int efa_rdm_ep_check_qp_in_order_aligned_128_bytes(struct efa_rdm_ep *ep,
 	/* Create a dummy qp for query only */
 	efa_rdm_ep_construct_ibv_qp_init_attr_ex(ep, &attr_ex, ibv_cq_ex, ibv_cq_ex);
 
-	ret = efa_qp_create(&qp, &attr_ex);
+	ret = efa_qp_create(&qp, &attr_ex, ep->base_ep.info->tx_attr->tclass);
 	if (ret)
 		goto out;