From 5ddcc252799992d5cc851cd14a9a2779984cefe1 Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Wed, 28 Aug 2024 15:59:20 +0300 Subject: [PATCH 01/14] UCP: Enable ppln protos with cuda buffers by default --- src/ucp/core/ucp_context.c | 14 ++++--- src/ucp/core/ucp_context.h | 4 +- src/ucp/rndv/proto_rndv.h | 1 + src/ucp/rndv/rndv.c | 25 +++++++++--- src/ucp/rndv/rndv_get.c | 38 +++++++++++-------- src/ucp/rndv/rndv_mtype.inl | 29 +++++++------- src/ucp/rndv/rndv_put.c | 39 +++++++++++++------ src/ucp/rndv/rndv_rkey_ptr.c | 7 ++-- src/ucp/rndv/rndv_rtr.c | 73 +++++++++++++++++++++--------------- 9 files changed, 145 insertions(+), 85 deletions(-) diff --git a/src/ucp/core/ucp_context.c b/src/ucp/core/ucp_context.c index 7007499cc3a..8aabfa4cccb 100644 --- a/src/ucp/core/ucp_context.c +++ b/src/ucp/core/ucp_context.c @@ -340,11 +340,15 @@ static ucs_config_field_t ucp_context_config_table[] = { "and the resulting performance.", ucs_offsetof(ucp_context_config_t, estimated_num_ppn), UCS_CONFIG_TYPE_ULUNITS}, - {"RNDV_FRAG_MEM_TYPE", "host", - "Memory type of fragments used for RNDV pipeline protocol.\n" - "Allowed memory types is one of: host, cuda, rocm, ze-host, ze-device", - ucs_offsetof(ucp_context_config_t, rndv_frag_mem_type), - UCS_CONFIG_TYPE_ENUM(ucs_memory_type_names)}, + {"RNDV_FRAG_MEM_TYPE", NULL, "", + ucs_offsetof(ucp_context_config_t, rndv_frag_mem_types), + UCS_CONFIG_TYPE_BITMAP(ucs_memory_type_names)}, + + {"RNDV_FRAG_MEM_TYPES", "host,cuda", + "Memory types of fragments used for RNDV pipeline protocol.\n" + "Allowed memory types are: host, cuda, rocm, ze-host, ze-device", + ucs_offsetof(ucp_context_config_t, rndv_frag_mem_types), + UCS_CONFIG_TYPE_BITMAP(ucs_memory_type_names)}, {"RNDV_PIPELINE_SEND_THRESH", "inf", "RNDV size threshold to enable sender side pipeline for mem type", diff --git a/src/ucp/core/ucp_context.h b/src/ucp/core/ucp_context.h index 1395b9704a3..49ab5880539 100644 --- a/src/ucp/core/ucp_context.h +++ b/src/ucp/core/ucp_context.h @@ -78,8 +78,8 @@ typedef struct ucp_context_config { size_t rndv_frag_size[UCS_MEMORY_TYPE_LAST]; /** Number of RNDV pipeline fragments per allocation */ size_t rndv_num_frags[UCS_MEMORY_TYPE_LAST]; - /** Memory type of fragments used for RNDV pipeline protocol */ - ucs_memory_type_t rndv_frag_mem_type; + /** Memory types of fragments used for RNDV pipeline protocol */ + uint64_t rndv_frag_mem_types; /** RNDV pipeline send threshold */ size_t rndv_pipeline_send_thresh; /** Enabling 2-stage pipeline rndv protocol */ diff --git a/src/ucp/rndv/proto_rndv.h b/src/ucp/rndv/proto_rndv.h index 221bc2c3fc4..88ec9fbc64e 100644 --- a/src/ucp/rndv/proto_rndv.h +++ b/src/ucp/rndv/proto_rndv.h @@ -68,6 +68,7 @@ typedef struct { */ typedef struct { ucp_proto_rndv_ack_priv_t super; + ucs_memory_type_t frag_mem_type; /* Multi-lane common part. Must be the last field, see @ref ucp_proto_multi_priv_t */ diff --git a/src/ucp/rndv/rndv.c b/src/ucp/rndv/rndv.c index 903bac92bfc..7550f8d7a42 100644 --- a/src/ucp/rndv/rndv.c +++ b/src/ucp/rndv/rndv.c @@ -21,6 +21,21 @@ #include +static UCS_F_ALWAYS_INLINE int +ucp_rndv_frag_memtype(ucp_context_t *context) +{ + if (context->config.ext.rndv_frag_mem_types == 0) { + return UCS_MEMORY_TYPE_HOST; + } + + /* Just one fragment memory type can be specified for proto v1, so take the + * first one from the map. Anyway for proto v1 UCX_RNDV_FRAG_MEM_TYPE is + * supposed to be used. + */ + + return ucs_ffs64(context->config.ext.rndv_frag_mem_types); +} + static UCS_F_ALWAYS_INLINE int ucp_rndv_memtype_direct_support(ucp_context_h context, size_t reg_length, uint64_t reg_mem_types) @@ -70,7 +85,7 @@ static int ucp_rndv_is_recv_pipeline_needed(ucp_request_t *rndv_req, const ucp_ep_config_t *ep_config = ucp_ep_config(rndv_req->send.ep); ucp_context_h context = rndv_req->send.ep->worker->context; int found = 0; - ucs_memory_type_t frag_mem_type = context->config.ext.rndv_frag_mem_type; + ucs_memory_type_t frag_mem_type = ucp_rndv_frag_memtype(context); ucp_md_index_t md_index; uct_md_attr_v2_t *md_attr; uint64_t mem_types; @@ -1170,7 +1185,7 @@ static void ucp_rndv_send_frag_get_mem_type(ucp_request_t *sreq, size_t length, uct_completion_callback_t comp_cb) { ucp_worker_h worker = sreq->send.ep->worker; - ucs_memory_type_t frag_mem_type = worker->context->config.ext.rndv_frag_mem_type; + ucs_memory_type_t frag_mem_type = ucp_rndv_frag_memtype(worker->context); ucp_request_t *freq; ucp_mem_desc_t *mdesc; @@ -1254,7 +1269,7 @@ ucp_rndv_recv_start_get_pipeline(ucp_worker_h worker, ucp_request_t *rndv_req, size_t frag_size; /* use ucp_rkey_packed_mem_type(rkey_buffer) with non-host fragments */ - frag_mem_type = context->config.ext.rndv_frag_mem_type; + frag_mem_type = ucp_rndv_frag_memtype(context); frag_size = context->config.ext.rndv_frag_size[frag_mem_type]; min_zcopy = config->rndv.get_zcopy.min; @@ -1337,7 +1352,7 @@ static void ucp_rndv_send_frag_rtr(ucp_worker_h worker, ucp_request_t *rndv_req, ucp_trace_req(rreq, "using rndv pipeline protocol rndv_req %p", rndv_req); offset = 0; - frag_mem_type = worker->context->config.ext.rndv_frag_mem_type; + frag_mem_type = ucp_rndv_frag_memtype(worker->context); max_frag_size = worker->context->config.ext.rndv_frag_size[frag_mem_type]; num_frags = ucs_div_round_up(rndv_rts_hdr->size, max_frag_size); @@ -2069,7 +2084,7 @@ static ucs_status_t ucp_rndv_send_start_put_pipeline(ucp_request_t *sreq, ucp_worker_h worker = sreq->send.ep->worker; ucp_context_h context = worker->context; size_t rndv_base_offset = rndv_rtr_hdr->offset; - ucs_memory_type_t frag_mem_type = context->config.ext.rndv_frag_mem_type; + ucs_memory_type_t frag_mem_type = ucp_rndv_frag_memtype(context); size_t rndv_size = ucs_min(rndv_rtr_hdr->size, sreq->send.length); const uct_md_attr_v2_t *md_attr; diff --git a/src/ucp/rndv/rndv_get.c b/src/ucp/rndv/rndv_get.c index 14d37814a08..a77d920fc4c 100644 --- a/src/ucp/rndv/rndv_get.c +++ b/src/ucp/rndv/rndv_get.c @@ -25,7 +25,8 @@ ucp_proto_rndv_get_common_probe(const ucp_proto_init_params_t *init_params, uint64_t rndv_modes, size_t max_length, uct_ep_operation_t memtype_op, unsigned flags, ucp_md_map_t initial_reg_md_map, - int support_ppln) + int support_ppln, + ucs_memory_type_t frag_mem_type) { ucp_context_t *context = init_params->worker->context; ucp_proto_multi_init_params_t params = { @@ -76,6 +77,7 @@ ucp_proto_rndv_get_common_probe(const ucp_proto_init_params_t *init_params, return; } + rpriv.frag_mem_type = frag_mem_type; priv_size = UCP_PROTO_MULTI_EXTENDED_PRIV_SIZE(&rpriv, mpriv); ucp_proto_common_add_proto(¶ms.super, &caps, &rpriv, priv_size); } @@ -125,7 +127,7 @@ ucp_proto_rndv_get_zcopy_probe(const ucp_proto_init_params_t *init_params) UCT_EP_OP_LAST, UCP_PROTO_COMMON_INIT_FLAG_SEND_ZCOPY | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, - 0, 0); + 0, 0, UCS_MEMORY_TYPE_HOST); } static void @@ -284,11 +286,11 @@ static ucs_status_t ucp_proto_rndv_get_mtype_fetch_progress(uct_pending_req_t *uct_req) { ucp_request_t *req = ucs_container_of(uct_req, ucp_request_t, send.uct); - const ucp_proto_rndv_bulk_priv_t *rpriv; + const ucp_proto_rndv_bulk_priv_t *rpriv = req->send.proto_config->priv; ucs_status_t status; if (!(req->flags & UCP_REQUEST_FLAG_PROTO_INITIALIZED)) { - status = ucp_proto_rndv_mtype_request_init(req); + status = ucp_proto_rndv_mtype_request_init(req, rpriv->frag_mem_type); if (status != UCS_OK) { ucp_proto_request_abort(req, status); return UCS_OK; @@ -300,8 +302,6 @@ ucp_proto_rndv_get_mtype_fetch_progress(uct_pending_req_t *uct_req) req->flags |= UCP_REQUEST_FLAG_PROTO_INITIALIZED; } - /* coverity[tainted_data_downcast] */ - rpriv = req->send.proto_config->priv; return ucp_proto_multi_progress(req, &rpriv->mpriv, ucp_proto_rndv_get_mtype_send_func, ucp_request_invoke_uct_completion_success, @@ -311,27 +311,35 @@ ucp_proto_rndv_get_mtype_fetch_progress(uct_pending_req_t *uct_req) static void ucp_proto_rndv_get_mtype_probe(const ucp_proto_init_params_t *init_params) { + ucp_context_t *context = init_params->worker->context; ucp_md_map_t mdesc_md_map; ucs_status_t status; size_t frag_size; + ucs_memory_type_t frag_mem_type; - status = ucp_proto_rndv_mtype_init(init_params, &mdesc_md_map, &frag_size); - if (status != UCS_OK) { - return; - } + ucs_for_each_bit(frag_mem_type, context->config.ext.rndv_frag_mem_types) { + status = ucp_proto_rndv_mtype_init(init_params, frag_mem_type, + &mdesc_md_map, &frag_size); + if (status != UCS_OK) { + return; + } - ucp_proto_rndv_get_common_probe(init_params, - UCS_BIT(UCP_RNDV_MODE_GET_PIPELINE), - frag_size, UCT_EP_OP_PUT_ZCOPY, 0, - mdesc_md_map, 1); + ucp_proto_rndv_get_common_probe(init_params, + UCS_BIT(UCP_RNDV_MODE_GET_PIPELINE), + frag_size, UCT_EP_OP_PUT_ZCOPY, 0, + mdesc_md_map, 1, frag_mem_type); + } } static void ucp_proto_rndv_get_mtype_query(const ucp_proto_query_params_t *params, ucp_proto_query_attr_t *attr) { + const ucp_proto_rndv_bulk_priv_t *rpriv = params->priv; + ucp_proto_rndv_bulk_query(params, attr); - ucp_proto_rndv_mtype_query_desc(params, attr, UCP_PROTO_RNDV_GET_DESC); + ucp_proto_rndv_mtype_query_desc(params, rpriv->frag_mem_type, attr, + UCP_PROTO_RNDV_GET_DESC); } static ucs_status_t ucp_proto_rndv_get_mtype_reset(ucp_request_t *req) diff --git a/src/ucp/rndv/rndv_mtype.inl b/src/ucp/rndv/rndv_mtype.inl index 55e021a83e9..b28e657678f 100644 --- a/src/ucp/rndv/rndv_mtype.inl +++ b/src/ucp/rndv/rndv_mtype.inl @@ -13,11 +13,9 @@ static ucp_ep_h ucp_proto_rndv_mtype_ep(ucp_worker_t *worker, + ucs_memory_type_t frag_mem_type, ucs_memory_type_t buf_mem_type) { - ucs_memory_type_t frag_mem_type = - worker->context->config.ext.rndv_frag_mem_type; - if (worker->mem_type_ep[buf_mem_type] != NULL) { return worker->mem_type_ep[buf_mem_type]; } @@ -27,15 +25,15 @@ static ucp_ep_h ucp_proto_rndv_mtype_ep(ucp_worker_t *worker, static UCS_F_ALWAYS_INLINE ucs_status_t ucp_proto_rndv_mtype_init(const ucp_proto_init_params_t *init_params, + ucs_memory_type_t frag_mem_type, ucp_md_map_t *mdesc_md_map_p, size_t *frag_size_p) { - ucp_worker_h worker = init_params->worker; - ucp_context_h context = worker->context; - ucs_memory_type_t mem_type = init_params->select_param->mem_type; - ucs_memory_type_t frag_mem_type = context->config.ext.rndv_frag_mem_type; + ucp_worker_h worker = init_params->worker; + ucp_context_h context = worker->context; + ucs_memory_type_t mem_type = init_params->select_param->mem_type; if ((init_params->select_param->dt_class != UCP_DATATYPE_CONTIG) || - (ucp_proto_rndv_mtype_ep(worker, mem_type) == NULL) || + (ucp_proto_rndv_mtype_ep(worker, frag_mem_type, mem_type) == NULL) || !ucp_proto_init_check_op(init_params, UCP_PROTO_RNDV_OP_ID_MASK)) { return UCS_ERR_UNSUPPORTED; } @@ -47,11 +45,10 @@ ucp_proto_rndv_mtype_init(const ucp_proto_init_params_t *init_params, } static UCS_F_ALWAYS_INLINE ucs_status_t -ucp_proto_rndv_mtype_request_init(ucp_request_t *req) +ucp_proto_rndv_mtype_request_init(ucp_request_t *req, + ucs_memory_type_t frag_mem_type) { - ucp_worker_h worker = req->send.ep->worker; - ucs_memory_type_t frag_mem_type = - worker->context->config.ext.rndv_frag_mem_type; + ucp_worker_h worker = req->send.ep->worker; req->send.rndv.mdesc = ucp_rndv_mpool_get(worker, frag_mem_type, UCS_SYS_DEVICE_ID_UNKNOWN); @@ -82,6 +79,7 @@ static ucp_ep_h ucp_proto_rndv_req_mtype_ep(ucp_request_t *req) ucp_ep_h mem_type_ep; mem_type_ep = ucp_proto_rndv_mtype_ep(req->send.ep->worker, + req->send.rndv.mdesc->memh->mem_type, req->send.state.dt_iter.mem_info.type); ucs_assert(mem_type_ep != NULL); @@ -142,7 +140,7 @@ static UCS_F_ALWAYS_INLINE ucs_status_t ucp_proto_rndv_mtype_copy( ucp_trace_req(req, "buffer %p copy-%s %p %s-%s using memtype-ep %p lane[%d]", buffer, mode, req->send.state.dt_iter.type.contig.buffer, ucs_memory_type_names[req->send.state.dt_iter.mem_info.type], - ucs_memory_type_names[context->config.ext.rndv_frag_mem_type], + ucs_memory_type_names[req->send.rndv.mdesc->memh->mem_type], mtype_ep, lane); ucp_proto_completion_init(&req->send.state.uct_comp, comp_func); @@ -169,6 +167,7 @@ static UCS_F_ALWAYS_INLINE ucs_status_t ucp_proto_rndv_mtype_copy( static UCS_F_ALWAYS_INLINE void ucp_proto_rndv_mtype_query_desc(const ucp_proto_query_params_t *params, + ucs_memory_type_t frag_mem_type, ucp_proto_query_attr_t *attr, const char *xfer_desc) { @@ -176,6 +175,7 @@ ucp_proto_rndv_mtype_query_desc(const ucp_proto_query_params_t *params, ucp_context_h context = params->worker->context; ucs_memory_type_t mem_type = params->select_param->mem_type; ucp_ep_h mtype_ep = ucp_proto_rndv_mtype_ep(params->worker, + frag_mem_type, mem_type); ucp_lane_index_t lane; ucp_rsc_index_t rsc_index; @@ -196,6 +196,9 @@ ucp_proto_rndv_mtype_query_desc(const ucp_proto_query_params_t *params, if (ucp_proto_select_op_id(params->select_param) == UCP_OP_ID_RNDV_RECV) { ucs_string_buffer_appendf(&strb, ", %s", tl_name); } + + ucs_string_buffer_appendf(&strb, ", frag %s", + ucs_memory_type_names[frag_mem_type]); } #endif diff --git a/src/ucp/rndv/rndv_put.c b/src/ucp/rndv/rndv_put.c index 547a0ac9252..fecee38c2fb 100644 --- a/src/ucp/rndv/rndv_put.c +++ b/src/ucp/rndv/rndv_put.c @@ -227,7 +227,8 @@ ucp_proto_rndv_put_common_probe(const ucp_proto_init_params_t *init_params, uct_ep_operation_t memtype_op, unsigned flags, ucp_md_map_t initial_reg_md_map, uct_completion_callback_t comp_cb, - int support_ppln, uint8_t stat_counter) + int support_ppln, uint8_t stat_counter, + ucs_memory_type_t frag_mem_type) { const size_t atp_size = sizeof(ucp_rndv_ack_hdr_t); ucp_context_t *context = init_params->worker->context; @@ -345,8 +346,9 @@ ucp_proto_rndv_put_common_probe(const ucp_proto_init_params_t *init_params, if (send_atp) { ucs_assert(rpriv.atp_map != 0); } - rpriv.atp_num_lanes = ucs_popcount(rpriv.atp_map); - rpriv.stat_counter = stat_counter; + rpriv.atp_num_lanes = ucs_popcount(rpriv.atp_map); + rpriv.stat_counter = stat_counter; + rpriv.bulk.frag_mem_type = frag_mem_type; priv_size = UCP_PROTO_MULTI_EXTENDED_PRIV_SIZE(&rpriv, bulk.mpriv); ucp_proto_common_add_proto(¶ms.super, &caps, &rpriv, priv_size); @@ -417,7 +419,7 @@ ucp_proto_rndv_put_zcopy_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_SEND_ZCOPY | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, 0, ucp_proto_rndv_put_zcopy_completion, 0, - UCP_WORKER_STAT_RNDV_PUT_ZCOPY); + UCP_WORKER_STAT_RNDV_PUT_ZCOPY, UCS_MEMORY_TYPE_HOST); } static void @@ -500,11 +502,12 @@ static ucs_status_t ucp_proto_rndv_put_mtype_copy_progress(uct_pending_req_t *uct_req) { ucp_request_t *req = ucs_container_of(uct_req, ucp_request_t, send.uct); + const ucp_proto_rndv_put_priv_t *rpriv = req->send.proto_config->priv; ucs_status_t status; ucs_assert(!(req->flags & UCP_REQUEST_FLAG_PROTO_INITIALIZED)); - status = ucp_proto_rndv_mtype_request_init(req); + status = ucp_proto_rndv_mtype_request_init(req, rpriv->bulk.frag_mem_type); if (status != UCS_OK) { ucp_proto_request_abort(req, status); return UCS_OK; @@ -563,8 +566,19 @@ ucp_proto_rndv_put_mtype_probe(const ucp_proto_init_params_t *init_params) ucp_md_map_t mdesc_md_map; ucs_status_t status; size_t frag_size; + ucs_memory_type_t frag_mem_type; - status = ucp_proto_rndv_mtype_init(init_params, &mdesc_md_map, &frag_size); + if (init_params->rkey_config_key == NULL) { + /* FIXME: maybe can initialize proto with all available types if no + * rkey in RTR. + */ + frag_mem_type = UCS_MEMORY_TYPE_HOST; + } else { + frag_mem_type = init_params->rkey_config_key->mem_type; + } + + status = ucp_proto_rndv_mtype_init(init_params, frag_mem_type, + &mdesc_md_map, &frag_size); if (status != UCS_OK) { return; } @@ -575,21 +589,22 @@ ucp_proto_rndv_put_mtype_probe(const ucp_proto_init_params_t *init_params) comp_cb = ucp_proto_rndv_put_mtype_completion; } - ucp_proto_rndv_put_common_probe(init_params, - UCS_BIT(UCP_RNDV_MODE_PUT_PIPELINE), - frag_size, UCT_EP_OP_GET_ZCOPY, 0, - mdesc_md_map, comp_cb, 1, - UCP_WORKER_STAT_RNDV_PUT_MTYPE_ZCOPY); + ucp_proto_rndv_put_common_probe( + init_params, UCS_BIT(UCP_RNDV_MODE_PUT_PIPELINE), frag_size, + UCT_EP_OP_GET_ZCOPY, 0, mdesc_md_map, comp_cb, 1, + UCP_WORKER_STAT_RNDV_PUT_MTYPE_ZCOPY, frag_mem_type); } static void ucp_proto_rndv_put_mtype_query(const ucp_proto_query_params_t *params, ucp_proto_query_attr_t *attr) { + const ucp_proto_rndv_put_priv_t *rpriv = params->priv; const char *put_desc; put_desc = ucp_proto_rndv_put_common_query(params, attr); - ucp_proto_rndv_mtype_query_desc(params, attr, put_desc); + ucp_proto_rndv_mtype_query_desc(params, rpriv->bulk.frag_mem_type, attr, + put_desc); } ucp_proto_t ucp_rndv_put_mtype_proto = { diff --git a/src/ucp/rndv/rndv_rkey_ptr.c b/src/ucp/rndv/rndv_rkey_ptr.c index 882de12629c..4e3d68d6f20 100644 --- a/src/ucp/rndv/rndv_rkey_ptr.c +++ b/src/ucp/rndv/rndv_rkey_ptr.c @@ -240,8 +240,9 @@ ucp_proto_rndv_rkey_ptr_mtype_probe(const ucp_proto_init_params_t *init_params) return; } - status = ucp_proto_rndv_mtype_init(init_params, &mdesc_md_map, - ¶ms.super.max_length); + /* 2-stage ppln protocols work with host staging buffers only */ + status = ucp_proto_rndv_mtype_init(init_params, UCS_MEMORY_TYPE_HOST, + &mdesc_md_map, ¶ms.super.max_length); if (status != UCS_OK) { return; } @@ -348,7 +349,7 @@ ucp_proto_rndv_rkey_ptr_mtype_query(const ucp_proto_query_params_t *params, const char *desc = UCP_PROTO_RNDV_RKEY_PTR_DESC; ucp_rndv_rkey_ptr_query_common(params, attr); - ucp_proto_rndv_mtype_query_desc(params, attr, desc); + ucp_proto_rndv_mtype_query_desc(params, UCS_MEMORY_TYPE_HOST, attr, desc); } ucp_proto_t ucp_rndv_rkey_ptr_mtype_proto = { diff --git a/src/ucp/rndv/rndv_rtr.c b/src/ucp/rndv/rndv_rtr.c index 51c64b958f0..17889dcaa67 100644 --- a/src/ucp/rndv/rndv_rtr.c +++ b/src/ucp/rndv/rndv_rtr.c @@ -32,6 +32,11 @@ typedef struct { ucp_proto_rndv_rtr_data_received_cb_t data_received; } ucp_proto_rndv_rtr_priv_t; +typedef struct { + ucp_proto_rndv_rtr_priv_t super; + ucs_memory_type_t frag_mem_type; +} ucp_proto_rndv_rtr_mtype_priv_t; + static UCS_F_ALWAYS_INLINE void ucp_proto_rtr_common_request_init(ucp_request_t *req) { @@ -341,10 +346,11 @@ ucp_proto_rndv_rtr_mtype_data_received(ucp_request_t *req, int in_buffer) static ucs_status_t ucp_proto_rndv_rtr_mtype_progress(uct_pending_req_t *self) { ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.uct); + const ucp_proto_rndv_rtr_mtype_priv_t *rpriv = req->send.proto_config->priv; ucs_status_t status; if (!(req->flags & UCP_REQUEST_FLAG_PROTO_INITIALIZED)) { - status = ucp_proto_rndv_mtype_request_init(req); + status = ucp_proto_rndv_mtype_request_init(req, rpriv->frag_mem_type); if (status != UCS_OK) { ucp_proto_request_abort(req, status); return UCS_OK; @@ -386,11 +392,11 @@ ucp_proto_rndv_rtr_mtype_probe(const ucp_proto_init_params_t *init_params) .remote_op_id = UCP_OP_ID_RNDV_SEND, .lane = ucp_proto_rndv_find_ctrl_lane(init_params), .perf_bias = 0.0, - .mem_info.type = context->config.ext.rndv_frag_mem_type, .mem_info.sys_dev = UCS_SYS_DEVICE_ID_UNKNOWN, .ctrl_msg_name = UCP_PROTO_RNDV_RTR_NAME, }; - ucp_proto_rndv_rtr_priv_t rpriv; + ucs_memory_type_t frag_mem_type; + ucp_proto_rndv_rtr_mtype_priv_t rpriv; ucp_md_map_t dummy_md_map; ucp_md_index_t md_index; ucs_status_t status; @@ -400,49 +406,56 @@ ucp_proto_rndv_rtr_mtype_probe(const ucp_proto_init_params_t *init_params) return; } - status = ucp_proto_rndv_mtype_init(init_params, &dummy_md_map, - ¶ms.super.max_length); - if (status != UCS_OK) { - return; - } + ucs_for_each_bit(frag_mem_type, context->config.ext.rndv_frag_mem_types) { + status = ucp_proto_rndv_mtype_init(init_params, frag_mem_type, + &dummy_md_map, + ¶ms.super.max_length); + if (status != UCS_OK) { + return; + } - status = ucp_proto_init_buffer_copy_time( - init_params->worker, "rtr/mtype unpack", params.mem_info.type, - init_params->select_param->mem_type, UCT_EP_OP_PUT_ZCOPY, - ¶ms.unpack_time, ¶ms.unpack_perf_node); - if (status != UCS_OK) { - return; - } + params.mem_info.type = frag_mem_type; - status = ucp_mm_get_alloc_md_index(context, &md_index, - params.mem_info.type); - if ((status != UCS_OK) || (md_index == UCP_NULL_RESOURCE)) { - params.md_map = 0; - } else { - params.md_map = UCS_BIT(md_index); - } + status = ucp_proto_init_buffer_copy_time( + init_params->worker, "rtr/mtype unpack", frag_mem_type, + init_params->select_param->mem_type, UCT_EP_OP_PUT_ZCOPY, + ¶ms.unpack_time, ¶ms.unpack_perf_node); + if (status != UCS_OK) { + return; + } + + status = ucp_mm_get_alloc_md_index(context, &md_index, frag_mem_type); + if ((status != UCS_OK) || (md_index == UCP_NULL_RESOURCE)) { + params.md_map = 0; + } else { + params.md_map = UCS_BIT(md_index); + } - rpriv.pack_cb = ucp_proto_rndv_rtr_mtype_pack; - rpriv.data_received = ucp_proto_rndv_rtr_mtype_data_received; + rpriv.super.pack_cb = ucp_proto_rndv_rtr_mtype_pack; + rpriv.super.data_received = ucp_proto_rndv_rtr_mtype_data_received; + rpriv.frag_mem_type = frag_mem_type; - ucp_proto_rndv_ctrl_probe(¶ms, &rpriv, sizeof(rpriv)); - ucp_proto_perf_node_deref(¶ms.unpack_perf_node); + ucp_proto_rndv_ctrl_probe(¶ms, &rpriv, sizeof(rpriv)); + ucp_proto_perf_node_deref(¶ms.unpack_perf_node); + } } static void ucp_proto_rndv_rtr_mtype_query(const ucp_proto_query_params_t *params, ucp_proto_query_attr_t *attr) { - const ucp_proto_rndv_ctrl_priv_t *rpriv = params->priv; + const ucp_proto_rndv_rtr_mtype_priv_t *rpriv = params->priv; ucp_proto_query_attr_t remote_attr; - ucp_proto_config_query(params->worker, &rpriv->remote_proto_config, + ucp_proto_config_query(params->worker, + &rpriv->super.super.remote_proto_config, params->msg_length, &remote_attr); attr->is_estimation = 1; attr->max_msg_length = remote_attr.max_msg_length; - attr->lane_map = UCS_BIT(rpriv->lane); - ucp_proto_rndv_mtype_query_desc(params, attr, remote_attr.desc); + attr->lane_map = UCS_BIT(rpriv->super.super.lane); + ucp_proto_rndv_mtype_query_desc(params, rpriv->frag_mem_type, attr, + remote_attr.desc); ucs_strncpy_safe(attr->config, remote_attr.config, sizeof(attr->config)); } From e4ca90e7fd267ae2f3522bc2c69129ccedbd87f5 Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Wed, 11 Sep 2024 10:23:34 +0300 Subject: [PATCH 02/14] UCP: Update mtype copy estimations --- src/ucp/proto/proto_init.c | 21 ++++++++++++++++++--- src/ucp/proto/proto_select.h | 2 ++ src/ucp/rndv/rndv_get.c | 1 + src/ucp/rndv/rndv_mtype.inl | 10 ++++++---- src/ucp/rndv/rndv_put.c | 1 + src/ucp/rndv/rndv_rkey_ptr.c | 2 +- src/ucp/rndv/rndv_rtr.c | 4 ++++ 7 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/ucp/proto/proto_init.c b/src/ucp/proto/proto_init.c index 741e6489dad..c8ab972efbc 100644 --- a/src/ucp/proto/proto_init.c +++ b/src/ucp/proto/proto_init.c @@ -475,6 +475,7 @@ ucp_proto_common_init_send_perf(const ucp_proto_common_init_params_t *params, { ucp_proto_perf_node_t *child_perf_node; ucs_linear_func_t send_overhead; + ucs_memory_type_t remote_mem_type; ucs_status_t status; send_perf->node = ucp_proto_perf_node_new_data("send-ovrh", ""); @@ -494,10 +495,21 @@ ucp_proto_common_init_send_perf(const ucp_proto_common_init_params_t *params, send_overhead = UCS_LINEAR_FUNC_ZERO; } else { ucs_assert(reg_md_map == 0); + if ((params->flags & UCP_PROTO_COMMON_INIT_FLAG_REMOTE_ACCESS) && + (params->super.rkey_config_key != NULL)) { + remote_mem_type = params->super.rkey_config_key->mem_type; + } else { + remote_mem_type = UCS_MEMORY_TYPE_HOST; + } + ucs_print("%s |TX init send selp memt %s rkey mt %s config key %p", + ucp_proto_id_field(params->super.proto_id, name), + ucs_memory_type_names[params->super.select_param->mem_type], + ucs_memory_type_names[remote_mem_type], + params->super.rkey_config_key); status = ucp_proto_init_buffer_copy_time( - params->super.worker, "send copy", UCS_MEMORY_TYPE_HOST, - params->super.select_param->mem_type, params->memtype_op, - &send_overhead, &child_perf_node); + params->super.worker, "send copy", + params->super.select_param->mem_type, remote_mem_type, + params->memtype_op, &send_overhead, &child_perf_node); if (status != UCS_OK) { ucp_proto_perf_node_deref(&send_perf->node); return status; @@ -598,8 +610,11 @@ ucp_proto_common_init_recv_perf(const ucp_proto_common_init_params_t *params, if (params->super.rkey_config_key == NULL) { /* Assume same memory type as sender */ recv_mem_type = select_param->mem_type; + ucs_print("%s |RX selp memt %s, recvmt the same",ucp_proto_id_field(params->super.proto_id, name), ucs_memory_type_names[select_param->mem_type]) ; } else { recv_mem_type = params->super.rkey_config_key->mem_type; + ucs_print("%s|RX selp memt %s, recvmt %s (from rkey)", + ucp_proto_id_field(params->super.proto_id, name), ucs_memory_type_names[select_param->mem_type], ucs_memory_type_names[params->super.rkey_config_key->mem_type]) ; } /* Silence cppcheck */ diff --git a/src/ucp/proto/proto_select.h b/src/ucp/proto/proto_select.h index 9441c3c0044..c3371d4a33f 100644 --- a/src/ucp/proto/proto_select.h +++ b/src/ucp/proto/proto_select.h @@ -82,6 +82,8 @@ struct ucp_proto_select_param { uint8_t sys_dev; /* Reply buffer system device */ } UCS_S_PACKED reply; + uint8_t mem_type_flags; + /* Align struct size to uint64_t */ uint8_t padding[2]; diff --git a/src/ucp/rndv/rndv_get.c b/src/ucp/rndv/rndv_get.c index a77d920fc4c..fc850c30095 100644 --- a/src/ucp/rndv/rndv_get.c +++ b/src/ucp/rndv/rndv_get.c @@ -277,6 +277,7 @@ ucp_proto_rndv_get_mtype_fetch_completion(uct_completion_t *uct_comp) ucp_proto_rndv_mtype_copy(req, req->send.rndv.mdesc->ptr, ucp_proto_rndv_mtype_get_req_memh(req), + req->send.rndv.mdesc->memh->mem_type, uct_ep_put_zcopy, ucp_proto_rndv_get_mtype_unpack_completion, "out to"); diff --git a/src/ucp/rndv/rndv_mtype.inl b/src/ucp/rndv/rndv_mtype.inl index b28e657678f..18911e50879 100644 --- a/src/ucp/rndv/rndv_mtype.inl +++ b/src/ucp/rndv/rndv_mtype.inl @@ -128,10 +128,12 @@ ucp_proto_rndv_mtype_next_iov(ucp_request_t *req, static UCS_F_ALWAYS_INLINE ucs_status_t ucp_proto_rndv_mtype_copy( ucp_request_t *req, void *buffer, uct_mem_h memh, - uct_ep_put_zcopy_func_t copy_func, uct_completion_callback_t comp_func, - const char *mode) + ucs_memory_type_t frag_mem_type, uct_ep_put_zcopy_func_t copy_func, + uct_completion_callback_t comp_func, const char *mode) { - ucp_ep_h mtype_ep = ucp_proto_rndv_req_mtype_ep(req); + ucp_ep_h mtype_ep = ucp_proto_rndv_mtype_ep( + req->send.ep->worker, frag_mem_type, + req->send.state.dt_iter.mem_info.type); ucp_lane_index_t lane = ucp_ep_config(mtype_ep)->key.rma_bw_lanes[0]; ucp_context_t UCS_V_UNUSED *context = req->send.ep->worker->context; ucs_status_t status; @@ -140,7 +142,7 @@ static UCS_F_ALWAYS_INLINE ucs_status_t ucp_proto_rndv_mtype_copy( ucp_trace_req(req, "buffer %p copy-%s %p %s-%s using memtype-ep %p lane[%d]", buffer, mode, req->send.state.dt_iter.type.contig.buffer, ucs_memory_type_names[req->send.state.dt_iter.mem_info.type], - ucs_memory_type_names[req->send.rndv.mdesc->memh->mem_type], + ucs_memory_type_names[frag_mem_type], mtype_ep, lane); ucp_proto_completion_init(&req->send.state.uct_comp, comp_func); diff --git a/src/ucp/rndv/rndv_put.c b/src/ucp/rndv/rndv_put.c index fecee38c2fb..4fddf4a8216 100644 --- a/src/ucp/rndv/rndv_put.c +++ b/src/ucp/rndv/rndv_put.c @@ -517,6 +517,7 @@ ucp_proto_rndv_put_mtype_copy_progress(uct_pending_req_t *uct_req) req->flags |= UCP_REQUEST_FLAG_PROTO_INITIALIZED; ucp_proto_rndv_mtype_copy(req, req->send.rndv.mdesc->ptr, ucp_proto_rndv_mtype_get_req_memh(req), + req->send.rndv.mdesc->memh->mem_type, uct_ep_get_zcopy, ucp_proto_rndv_put_mtype_pack_completion, "in from"); diff --git a/src/ucp/rndv/rndv_rkey_ptr.c b/src/ucp/rndv/rndv_rkey_ptr.c index 4e3d68d6f20..3c69cf9dad5 100644 --- a/src/ucp/rndv/rndv_rkey_ptr.c +++ b/src/ucp/rndv/rndv_rkey_ptr.c @@ -323,7 +323,7 @@ ucp_proto_rndv_rkey_ptr_mtype_copy_progress(uct_pending_req_t *uct_req) req->flags |= UCP_REQUEST_FLAG_PROTO_INITIALIZED; ucp_proto_rndv_mtype_copy(req, ppln_data->local_ptr, ppln_data->uct_memh, - uct_ep_get_zcopy, + UCS_MEMORY_TYPE_HOST, uct_ep_get_zcopy, ucp_proto_rndv_rkey_ptr_mtype_copy_completion, "in from"); diff --git a/src/ucp/rndv/rndv_rtr.c b/src/ucp/rndv/rndv_rtr.c index 17889dcaa67..e84eb1cdb0c 100644 --- a/src/ucp/rndv/rndv_rtr.c +++ b/src/ucp/rndv/rndv_rtr.c @@ -337,6 +337,7 @@ ucp_proto_rndv_rtr_mtype_data_received(ucp_request_t *req, int in_buffer) the remote address we published - the rendezvous fragment */ ucp_proto_rndv_mtype_copy(req, req->send.rndv.mdesc->ptr, ucp_proto_rndv_mtype_get_req_memh(req), + req->send.rndv.mdesc->memh->mem_type, uct_ep_put_zcopy, ucp_proto_rndv_rtr_mtype_copy_completion, "out to"); @@ -416,6 +417,9 @@ ucp_proto_rndv_rtr_mtype_probe(const ucp_proto_init_params_t *init_params) params.mem_info.type = frag_mem_type; + ucs_print("RTR mtype | localmt %s, remote %s", + ucs_memory_type_names[frag_mem_type], + ucs_memory_type_names[init_params->select_param->mem_type]); status = ucp_proto_init_buffer_copy_time( init_params->worker, "rtr/mtype unpack", frag_mem_type, init_params->select_param->mem_type, UCT_EP_OP_PUT_ZCOPY, From 87019a3570fcdfae991f8eccbf51b9d011c68cae Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Sun, 15 Sep 2024 20:19:36 +0300 Subject: [PATCH 03/14] GTEST: Fix rebase comp error --- src/ucp/rndv/rndv.c | 1 - test/gtest/ucp/ucp_test.cc | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/ucp/rndv/rndv.c b/src/ucp/rndv/rndv.c index 7550f8d7a42..90dba6f211e 100644 --- a/src/ucp/rndv/rndv.c +++ b/src/ucp/rndv/rndv.c @@ -32,7 +32,6 @@ ucp_rndv_frag_memtype(ucp_context_t *context) * first one from the map. Anyway for proto v1 UCX_RNDV_FRAG_MEM_TYPE is * supposed to be used. */ - return ucs_ffs64(context->config.ext.rndv_frag_mem_types); } diff --git a/test/gtest/ucp/ucp_test.cc b/test/gtest/ucp/ucp_test.cc index 8fd0c7e1316..f2e746df777 100644 --- a/test/gtest/ucp/ucp_test.cc +++ b/test/gtest/ucp/ucp_test.cc @@ -1184,8 +1184,11 @@ bool ucp_test_base::entity::has_lane_with_caps(uint64_t caps) const bool ucp_test_base::entity::is_rndv_put_ppln_supported() const { - const auto config = ucp_ep_config(ep()); - ucs_memory_type_t mem_type = ucph()->config.ext.rndv_frag_mem_type; + const auto config = ucp_ep_config(ep()); + ucs_memory_type_t mem_type; + + mem_type = (ucs_memory_type_t)ucs_ffs64( + ucph()->config.ext.rndv_frag_mem_types); for (auto i = 0; i < config->key.num_lanes; ++i) { const auto lane = config->key.rma_bw_lanes[i]; From f0784d390dfe9b91167a8731b8a4746769dddebd Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Tue, 17 Sep 2024 13:36:40 +0300 Subject: [PATCH 04/14] UCP: Fix ppln/cuda for D-D and D-H cases --- src/ucp/proto/proto_common.h | 1 + src/ucp/rndv/proto_rndv.c | 3 ++- src/ucp/rndv/rndv_rtr.c | 5 +++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ucp/proto/proto_common.h b/src/ucp/proto/proto_common.h index 01bc5aee64a..6492904530b 100644 --- a/src/ucp/proto/proto_common.h +++ b/src/ucp/proto/proto_common.h @@ -61,6 +61,7 @@ typedef enum { /* Supports starting the request when its datatype iterator offset is > 0 */ UCP_PROTO_COMMON_INIT_FLAG_RESUME = UCS_BIT(10), + UCP_PROTO_COMMON_KEEP_MD_MAP = UCS_BIT(11) } ucp_proto_common_init_flags_t; diff --git a/src/ucp/rndv/proto_rndv.c b/src/ucp/rndv/proto_rndv.c index 687ed4053ba..bdf3b37288c 100644 --- a/src/ucp/rndv/proto_rndv.c +++ b/src/ucp/rndv/proto_rndv.c @@ -268,7 +268,8 @@ ucp_proto_rndv_ctrl_init_priv(const ucp_proto_rndv_ctrl_init_params_t *params, /* Use only memory domains for which the unpacking of the remote key was * successful */ - if (init_params->rkey_config_key != NULL) { + if ((init_params->rkey_config_key != NULL) && + !(params->super.flags & UCP_PROTO_COMMON_KEEP_MD_MAP)) { rpriv->md_map &= ~init_params->rkey_config_key->unreachable_md_map; } diff --git a/src/ucp/rndv/rndv_rtr.c b/src/ucp/rndv/rndv_rtr.c index e84eb1cdb0c..19d533ce627 100644 --- a/src/ucp/rndv/rndv_rtr.c +++ b/src/ucp/rndv/rndv_rtr.c @@ -388,7 +388,8 @@ ucp_proto_rndv_rtr_mtype_probe(const ucp_proto_init_params_t *init_params) .super.send_op = UCT_EP_OP_AM_BCOPY, .super.memtype_op = UCT_EP_OP_LAST, .super.flags = UCP_PROTO_COMMON_INIT_FLAG_RESPONSE | - UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, + UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING | + UCP_PROTO_COMMON_KEEP_MD_MAP, .super.exclude_map = 0, .remote_op_id = UCP_OP_ID_RNDV_SEND, .lane = ucp_proto_rndv_find_ctrl_lane(init_params), @@ -412,7 +413,7 @@ ucp_proto_rndv_rtr_mtype_probe(const ucp_proto_init_params_t *init_params) &dummy_md_map, ¶ms.super.max_length); if (status != UCS_OK) { - return; + continue; } params.mem_info.type = frag_mem_type; From c1f065ed30465c57c97637c3c9328011d5cfaf73 Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Tue, 17 Sep 2024 13:40:39 +0300 Subject: [PATCH 05/14] UCP: Remove ucs_prints --- src/ucp/proto/proto_init.c | 10 ++-------- src/ucp/rndv/rndv_rtr.c | 3 --- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/ucp/proto/proto_init.c b/src/ucp/proto/proto_init.c index c8ab972efbc..c3da5194ec3 100644 --- a/src/ucp/proto/proto_init.c +++ b/src/ucp/proto/proto_init.c @@ -495,17 +495,14 @@ ucp_proto_common_init_send_perf(const ucp_proto_common_init_params_t *params, send_overhead = UCS_LINEAR_FUNC_ZERO; } else { ucs_assert(reg_md_map == 0); + if ((params->flags & UCP_PROTO_COMMON_INIT_FLAG_REMOTE_ACCESS) && (params->super.rkey_config_key != NULL)) { remote_mem_type = params->super.rkey_config_key->mem_type; } else { remote_mem_type = UCS_MEMORY_TYPE_HOST; } - ucs_print("%s |TX init send selp memt %s rkey mt %s config key %p", - ucp_proto_id_field(params->super.proto_id, name), - ucs_memory_type_names[params->super.select_param->mem_type], - ucs_memory_type_names[remote_mem_type], - params->super.rkey_config_key); + status = ucp_proto_init_buffer_copy_time( params->super.worker, "send copy", params->super.select_param->mem_type, remote_mem_type, @@ -610,11 +607,8 @@ ucp_proto_common_init_recv_perf(const ucp_proto_common_init_params_t *params, if (params->super.rkey_config_key == NULL) { /* Assume same memory type as sender */ recv_mem_type = select_param->mem_type; - ucs_print("%s |RX selp memt %s, recvmt the same",ucp_proto_id_field(params->super.proto_id, name), ucs_memory_type_names[select_param->mem_type]) ; } else { recv_mem_type = params->super.rkey_config_key->mem_type; - ucs_print("%s|RX selp memt %s, recvmt %s (from rkey)", - ucp_proto_id_field(params->super.proto_id, name), ucs_memory_type_names[select_param->mem_type], ucs_memory_type_names[params->super.rkey_config_key->mem_type]) ; } /* Silence cppcheck */ diff --git a/src/ucp/rndv/rndv_rtr.c b/src/ucp/rndv/rndv_rtr.c index 19d533ce627..b64bac5f25d 100644 --- a/src/ucp/rndv/rndv_rtr.c +++ b/src/ucp/rndv/rndv_rtr.c @@ -418,9 +418,6 @@ ucp_proto_rndv_rtr_mtype_probe(const ucp_proto_init_params_t *init_params) params.mem_info.type = frag_mem_type; - ucs_print("RTR mtype | localmt %s, remote %s", - ucs_memory_type_names[frag_mem_type], - ucs_memory_type_names[init_params->select_param->mem_type]); status = ucp_proto_init_buffer_copy_time( init_params->worker, "rtr/mtype unpack", frag_mem_type, init_params->select_param->mem_type, UCT_EP_OP_PUT_ZCOPY, From 267fdeea837f47dfab2196ce40a92af4d8f65b60 Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Wed, 18 Sep 2024 09:28:57 +0300 Subject: [PATCH 06/14] UCP: Fix ppln with cuda + minor fixes --- src/ucp/proto/proto_common.c | 23 ++++++++++++----------- src/ucp/proto/proto_common.h | 12 +++++++----- src/ucp/proto/proto_multi.c | 11 +++++++---- src/ucp/proto/proto_multi.h | 1 + src/ucp/proto/proto_select.h | 2 -- src/ucp/proto/proto_single.c | 3 ++- src/ucp/rndv/proto_rndv.c | 15 +++++++-------- src/ucp/rndv/rndv_get.c | 14 +++++++++----- src/ucp/rndv/rndv_put.c | 13 +++++++------ src/ucp/rndv/rndv_rtr.c | 2 +- 10 files changed, 53 insertions(+), 43 deletions(-) diff --git a/src/ucp/proto/proto_common.c b/src/ucp/proto/proto_common.c index b93e01dc1e4..cd4771e8648 100644 --- a/src/ucp/proto/proto_common.c +++ b/src/ucp/proto/proto_common.c @@ -409,7 +409,8 @@ ucp_lane_index_t ucp_proto_common_find_lanes(const ucp_proto_init_params_t *params, uct_ep_operation_t memtype_op, unsigned flags, ptrdiff_t max_iov_offs, size_t min_iov, - ucp_lane_type_t lane_type, uint64_t tl_cap_flags, + ucp_lane_type_t lane_type, + ucs_memory_type_t mem_type, uint64_t tl_cap_flags, ucp_lane_index_t max_lanes, ucp_lane_map_t exclude_map, ucp_lane_index_t *lanes) { @@ -494,25 +495,24 @@ ucp_proto_common_find_lanes(const ucp_proto_init_params_t *params, } /* Check memory registration capabilities for zero-copy case */ - if (flags & UCP_PROTO_COMMON_INIT_FLAG_SEND_ZCOPY) { + if (flags & (UCP_PROTO_COMMON_INIT_FLAG_SEND_ZCOPY | + UCP_PROTO_COMMON_INIT_FLAG_MTYPE_ZCOPY)) { if (md_attr->flags & UCT_MD_FLAG_NEED_MEMH) { /* Memory domain must support registration on the relevant * memory type */ - if (!(context->reg_md_map[select_param->mem_type] & - UCS_BIT(md_index))) { + if (!(context->reg_md_map[mem_type] & UCS_BIT(md_index))) { ucs_trace("%s: md %s cannot register %s memory", lane_desc, context->tl_mds[md_index].rsc.md_name, - ucs_memory_type_names[select_param->mem_type]); + ucs_memory_type_names[mem_type]); continue; } - } else if (!(md_attr->access_mem_types & - UCS_BIT(select_param->mem_type))) { + } else if (!(md_attr->access_mem_types & UCS_BIT(mem_type))) { /* * Memory domain which does not require a registration for zero * copy operation must be able to access the relevant memory type */ ucs_trace("%s: no access to mem type %s", lane_desc, - ucs_memory_type_names[select_param->mem_type]); + ucs_memory_type_names[mem_type]); continue; } } @@ -598,8 +598,9 @@ ucp_proto_common_reg_md_map(const ucp_proto_common_init_params_t *params, ucp_lane_index_t ucp_proto_common_find_lanes_with_min_frag( const ucp_proto_common_init_params_t *params, ucp_lane_type_t lane_type, - uint64_t tl_cap_flags, ucp_lane_index_t max_lanes, - ucp_lane_map_t exclude_map, ucp_lane_index_t *lanes) + ucs_memory_type_t mem_type, uint64_t tl_cap_flags, + ucp_lane_index_t max_lanes, ucp_lane_map_t exclude_map, + ucp_lane_index_t *lanes) { ucp_lane_index_t lane_index, lane, num_lanes, num_valid_lanes; const uct_iface_attr_t *iface_attr; @@ -608,7 +609,7 @@ ucp_lane_index_t ucp_proto_common_find_lanes_with_min_frag( num_lanes = ucp_proto_common_find_lanes(¶ms->super, params->memtype_op, params->flags, params->max_iov_offs, params->min_iov, lane_type, - tl_cap_flags, max_lanes, + mem_type, tl_cap_flags, max_lanes, exclude_map, lanes); num_valid_lanes = 0; diff --git a/src/ucp/proto/proto_common.h b/src/ucp/proto/proto_common.h index 6492904530b..10aacc07a6d 100644 --- a/src/ucp/proto/proto_common.h +++ b/src/ucp/proto/proto_common.h @@ -61,7 +61,8 @@ typedef enum { /* Supports starting the request when its datatype iterator offset is > 0 */ UCP_PROTO_COMMON_INIT_FLAG_RESUME = UCS_BIT(10), - UCP_PROTO_COMMON_KEEP_MD_MAP = UCS_BIT(11) + UCP_PROTO_COMMON_INIT_FLAG_KEEP_MD_MAP = UCS_BIT(11), + UCP_PROTO_COMMON_INIT_FLAG_MTYPE_ZCOPY = UCS_BIT(12) } ucp_proto_common_init_flags_t; @@ -248,16 +249,17 @@ ucp_proto_common_get_lane_perf(const ucp_proto_common_init_params_t *params, /* @return number of lanes found */ ucp_lane_index_t ucp_proto_common_find_lanes_with_min_frag( const ucp_proto_common_init_params_t *params, ucp_lane_type_t lane_type, - uint64_t tl_cap_flags, ucp_lane_index_t max_lanes, - ucp_lane_map_t exclude_map, ucp_lane_index_t *lanes); + ucs_memory_type_t mem_type, uint64_t tl_cap_flags, + ucp_lane_index_t max_lanes, ucp_lane_map_t exclude_map, + ucp_lane_index_t *lanes); ucp_lane_index_t ucp_proto_common_find_lanes(const ucp_proto_init_params_t *params, uct_ep_operation_t memtype_op, unsigned flags, ptrdiff_t max_iov_offs, size_t min_iov, - ucp_lane_type_t lane_type, uint64_t tl_cap_flags, - ucp_lane_index_t max_lanes, + ucp_lane_type_t lane_type, ucs_memory_type_t mem_type, + uint64_t tl_cap_flags, ucp_lane_index_t max_lanes, ucp_lane_map_t exclude_map, ucp_lane_index_t *lanes); diff --git a/src/ucp/proto/proto_multi.c b/src/ucp/proto/proto_multi.c index 0689866c2a3..3bbdfd69982 100644 --- a/src/ucp/proto/proto_multi.c +++ b/src/ucp/proto/proto_multi.c @@ -22,6 +22,7 @@ ucs_status_t ucp_proto_multi_init(const ucp_proto_multi_init_params_t *params, ucp_proto_caps_t *caps, + ucs_memory_type_t mem_type, ucp_proto_multi_priv_t *mpriv) { ucp_context_h context = params->super.super.worker->context; @@ -55,8 +56,8 @@ ucs_status_t ucp_proto_multi_init(const ucp_proto_multi_init_params_t *params, /* Find first lane */ num_lanes = ucp_proto_common_find_lanes_with_min_frag( - ¶ms->super, params->first.lane_type, params->first.tl_cap_flags, - 1, 0, lanes); + ¶ms->super, params->first.lane_type, mem_type, + params->first.tl_cap_flags, 1, 0, lanes); if (num_lanes == 0) { ucs_trace("no lanes for %s", ucp_proto_id_field(params->super.super.proto_id, name)); @@ -65,7 +66,7 @@ ucs_status_t ucp_proto_multi_init(const ucp_proto_multi_init_params_t *params, /* Find rest of the lanes */ num_lanes += ucp_proto_common_find_lanes_with_min_frag( - ¶ms->super, params->middle.lane_type, + ¶ms->super, params->middle.lane_type, mem_type, params->middle.tl_cap_flags, UCP_PROTO_MAX_LANES - 1, UCS_BIT(lanes[0]), lanes + 1); @@ -264,7 +265,9 @@ void ucp_proto_multi_probe(const ucp_proto_multi_init_params_t *params) ucp_proto_caps_t caps; ucs_status_t status; - status = ucp_proto_multi_init(params, &caps, &mpriv); + status = ucp_proto_multi_init(params, &caps, + params->super.super.select_param->mem_type, + &mpriv); if (status != UCS_OK) { return; } diff --git a/src/ucp/proto/proto_multi.h b/src/ucp/proto/proto_multi.h index 8e6dff043c7..828ceeac272 100644 --- a/src/ucp/proto/proto_multi.h +++ b/src/ucp/proto/proto_multi.h @@ -160,6 +160,7 @@ typedef ucs_status_t (*ucp_proto_multi_lane_send_func_t)(ucp_request_t *req, ucs_status_t ucp_proto_multi_init(const ucp_proto_multi_init_params_t *params, ucp_proto_caps_t *caps, + ucs_memory_type_t mem_type, ucp_proto_multi_priv_t *mpriv); diff --git a/src/ucp/proto/proto_select.h b/src/ucp/proto/proto_select.h index c3371d4a33f..9441c3c0044 100644 --- a/src/ucp/proto/proto_select.h +++ b/src/ucp/proto/proto_select.h @@ -82,8 +82,6 @@ struct ucp_proto_select_param { uint8_t sys_dev; /* Reply buffer system device */ } UCS_S_PACKED reply; - uint8_t mem_type_flags; - /* Align struct size to uint64_t */ uint8_t padding[2]; diff --git a/src/ucp/proto/proto_single.c b/src/ucp/proto/proto_single.c index ddd754e86a2..07cb74eb7ff 100644 --- a/src/ucp/proto/proto_single.c +++ b/src/ucp/proto/proto_single.c @@ -30,7 +30,8 @@ ucs_status_t ucp_proto_single_init(const ucp_proto_single_init_params_t *params, ucs_status_t status; num_lanes = ucp_proto_common_find_lanes_with_min_frag( - ¶ms->super, params->lane_type, params->tl_cap_flags, 1, + ¶ms->super, params->lane_type, + params->super.super.select_param->mem_type, params->tl_cap_flags, 1, params->super.exclude_map, &lane); if (num_lanes == 0) { ucs_trace("no lanes for %s", diff --git a/src/ucp/rndv/proto_rndv.c b/src/ucp/rndv/proto_rndv.c index bdf3b37288c..524fedcc715 100644 --- a/src/ucp/rndv/proto_rndv.c +++ b/src/ucp/rndv/proto_rndv.c @@ -269,7 +269,7 @@ ucp_proto_rndv_ctrl_init_priv(const ucp_proto_rndv_ctrl_init_params_t *params, /* Use only memory domains for which the unpacking of the remote key was * successful */ if ((init_params->rkey_config_key != NULL) && - !(params->super.flags & UCP_PROTO_COMMON_KEEP_MD_MAP)) { + !(params->super.flags & UCP_PROTO_COMMON_INIT_FLAG_KEEP_MD_MAP)) { rpriv->md_map &= ~init_params->rkey_config_key->unreachable_md_map; } @@ -519,12 +519,10 @@ ucp_proto_rndv_find_ctrl_lane(const ucp_proto_init_params_t *params) { ucp_lane_index_t lane, num_lanes; - num_lanes = ucp_proto_common_find_lanes(params, UCT_EP_OP_LAST, - UCP_PROTO_COMMON_INIT_FLAG_HDR_ONLY, - UCP_PROTO_COMMON_OFFSET_INVALID, 1, - UCP_LANE_TYPE_AM, - UCT_IFACE_FLAG_AM_BCOPY, 1, 0, - &lane); + num_lanes = ucp_proto_common_find_lanes( + params, UCT_EP_OP_LAST, UCP_PROTO_COMMON_INIT_FLAG_HDR_ONLY, + UCP_PROTO_COMMON_OFFSET_INVALID, 1, UCP_LANE_TYPE_AM, + UCS_MEMORY_TYPE_HOST, UCT_IFACE_FLAG_AM_BCOPY, 1, 0, &lane); if (num_lanes == 0) { ucs_debug("no active message lane for %s", ucp_proto_id_field(params->proto_id, name)); @@ -709,7 +707,8 @@ ucp_proto_rndv_bulk_init(const ucp_proto_multi_init_params_t *init_params, ucp_proto_caps_t multi_caps; ucs_status_t status; - status = ucp_proto_multi_init(init_params, &multi_caps, mpriv); + status = ucp_proto_multi_init(init_params, &multi_caps, + rpriv->frag_mem_type, mpriv); if (status != UCS_OK) { return status; } diff --git a/src/ucp/rndv/rndv_get.c b/src/ucp/rndv/rndv_get.c index fc850c30095..4f26e320772 100644 --- a/src/ucp/rndv/rndv_get.c +++ b/src/ucp/rndv/rndv_get.c @@ -71,13 +71,13 @@ ucp_proto_rndv_get_common_probe(const ucp_proto_init_params_t *init_params, return; } + rpriv.frag_mem_type = frag_mem_type; status = ucp_proto_rndv_bulk_init(¶ms, UCP_PROTO_RNDV_GET_DESC, UCP_PROTO_RNDV_ATS_NAME, &rpriv, &caps); if (status != UCS_OK) { return; } - rpriv.frag_mem_type = frag_mem_type; priv_size = UCP_PROTO_MULTI_EXTENDED_PRIV_SIZE(&rpriv, mpriv); ucp_proto_common_add_proto(¶ms.super, &caps, &rpriv, priv_size); } @@ -127,7 +127,7 @@ ucp_proto_rndv_get_zcopy_probe(const ucp_proto_init_params_t *init_params) UCT_EP_OP_LAST, UCP_PROTO_COMMON_INIT_FLAG_SEND_ZCOPY | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, - 0, 0, UCS_MEMORY_TYPE_HOST); + 0, 0, init_params->select_param->mem_type); } static void @@ -287,9 +287,12 @@ static ucs_status_t ucp_proto_rndv_get_mtype_fetch_progress(uct_pending_req_t *uct_req) { ucp_request_t *req = ucs_container_of(uct_req, ucp_request_t, send.uct); - const ucp_proto_rndv_bulk_priv_t *rpriv = req->send.proto_config->priv; + const ucp_proto_rndv_bulk_priv_t *rpriv; ucs_status_t status; + /* coverity[tainted_data_downcast] */ + rpriv = req->send.proto_config->priv; + if (!(req->flags & UCP_REQUEST_FLAG_PROTO_INITIALIZED)) { status = ucp_proto_rndv_mtype_request_init(req, rpriv->frag_mem_type); if (status != UCS_OK) { @@ -322,12 +325,13 @@ ucp_proto_rndv_get_mtype_probe(const ucp_proto_init_params_t *init_params) status = ucp_proto_rndv_mtype_init(init_params, frag_mem_type, &mdesc_md_map, &frag_size); if (status != UCS_OK) { - return; + continue; } ucp_proto_rndv_get_common_probe(init_params, UCS_BIT(UCP_RNDV_MODE_GET_PIPELINE), - frag_size, UCT_EP_OP_PUT_ZCOPY, 0, + frag_size, UCT_EP_OP_PUT_ZCOPY, + UCP_PROTO_COMMON_INIT_FLAG_MTYPE_ZCOPY, mdesc_md_map, 1, frag_mem_type); } } diff --git a/src/ucp/rndv/rndv_put.c b/src/ucp/rndv/rndv_put.c index 4fddf4a8216..639a77885e0 100644 --- a/src/ucp/rndv/rndv_put.c +++ b/src/ucp/rndv/rndv_put.c @@ -278,6 +278,7 @@ ucp_proto_rndv_put_common_probe(const ucp_proto_init_params_t *init_params, return; } + rpriv.bulk.frag_mem_type = frag_mem_type; status = ucp_proto_rndv_bulk_init(¶ms, UCP_PROTO_RNDV_PUT_DESC, UCP_PROTO_RNDV_ATP_NAME, &rpriv.bulk, &caps); @@ -346,9 +347,8 @@ ucp_proto_rndv_put_common_probe(const ucp_proto_init_params_t *init_params, if (send_atp) { ucs_assert(rpriv.atp_map != 0); } - rpriv.atp_num_lanes = ucs_popcount(rpriv.atp_map); - rpriv.stat_counter = stat_counter; - rpriv.bulk.frag_mem_type = frag_mem_type; + rpriv.atp_num_lanes = ucs_popcount(rpriv.atp_map); + rpriv.stat_counter = stat_counter; priv_size = UCP_PROTO_MULTI_EXTENDED_PRIV_SIZE(&rpriv, bulk.mpriv); ucp_proto_common_add_proto(¶ms.super, &caps, &rpriv, priv_size); @@ -419,7 +419,7 @@ ucp_proto_rndv_put_zcopy_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_SEND_ZCOPY | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, 0, ucp_proto_rndv_put_zcopy_completion, 0, - UCP_WORKER_STAT_RNDV_PUT_ZCOPY, UCS_MEMORY_TYPE_HOST); + UCP_WORKER_STAT_RNDV_PUT_ZCOPY, init_params->select_param->mem_type); } static void @@ -592,8 +592,9 @@ ucp_proto_rndv_put_mtype_probe(const ucp_proto_init_params_t *init_params) ucp_proto_rndv_put_common_probe( init_params, UCS_BIT(UCP_RNDV_MODE_PUT_PIPELINE), frag_size, - UCT_EP_OP_GET_ZCOPY, 0, mdesc_md_map, comp_cb, 1, - UCP_WORKER_STAT_RNDV_PUT_MTYPE_ZCOPY, frag_mem_type); + UCT_EP_OP_GET_ZCOPY, UCP_PROTO_COMMON_INIT_FLAG_MTYPE_ZCOPY, + mdesc_md_map, comp_cb, 1, UCP_WORKER_STAT_RNDV_PUT_MTYPE_ZCOPY, + frag_mem_type); } static void diff --git a/src/ucp/rndv/rndv_rtr.c b/src/ucp/rndv/rndv_rtr.c index b64bac5f25d..d6508a9a4cb 100644 --- a/src/ucp/rndv/rndv_rtr.c +++ b/src/ucp/rndv/rndv_rtr.c @@ -389,7 +389,7 @@ ucp_proto_rndv_rtr_mtype_probe(const ucp_proto_init_params_t *init_params) .super.memtype_op = UCT_EP_OP_LAST, .super.flags = UCP_PROTO_COMMON_INIT_FLAG_RESPONSE | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING | - UCP_PROTO_COMMON_KEEP_MD_MAP, + UCP_PROTO_COMMON_INIT_FLAG_KEEP_MD_MAP, .super.exclude_map = 0, .remote_op_id = UCP_OP_ID_RNDV_SEND, .lane = ucp_proto_rndv_find_ctrl_lane(init_params), From 44455a4c577337819a4242295e18d553d9fa9f44 Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Wed, 18 Sep 2024 14:50:10 +0300 Subject: [PATCH 07/14] UCP: Revert Fix ppln with cuda + minor fixes This reverts commit 267fdeea837f47dfab2196ce40a92af4d8f65b60. --- src/ucp/proto/proto_common.c | 23 +++++++++++------------ src/ucp/proto/proto_common.h | 12 +++++------- src/ucp/proto/proto_multi.c | 11 ++++------- src/ucp/proto/proto_multi.h | 1 - src/ucp/proto/proto_select.h | 2 ++ src/ucp/proto/proto_single.c | 3 +-- src/ucp/rndv/proto_rndv.c | 15 ++++++++------- src/ucp/rndv/rndv_get.c | 14 +++++--------- src/ucp/rndv/rndv_put.c | 13 ++++++------- src/ucp/rndv/rndv_rtr.c | 2 +- 10 files changed, 43 insertions(+), 53 deletions(-) diff --git a/src/ucp/proto/proto_common.c b/src/ucp/proto/proto_common.c index cd4771e8648..b93e01dc1e4 100644 --- a/src/ucp/proto/proto_common.c +++ b/src/ucp/proto/proto_common.c @@ -409,8 +409,7 @@ ucp_lane_index_t ucp_proto_common_find_lanes(const ucp_proto_init_params_t *params, uct_ep_operation_t memtype_op, unsigned flags, ptrdiff_t max_iov_offs, size_t min_iov, - ucp_lane_type_t lane_type, - ucs_memory_type_t mem_type, uint64_t tl_cap_flags, + ucp_lane_type_t lane_type, uint64_t tl_cap_flags, ucp_lane_index_t max_lanes, ucp_lane_map_t exclude_map, ucp_lane_index_t *lanes) { @@ -495,24 +494,25 @@ ucp_proto_common_find_lanes(const ucp_proto_init_params_t *params, } /* Check memory registration capabilities for zero-copy case */ - if (flags & (UCP_PROTO_COMMON_INIT_FLAG_SEND_ZCOPY | - UCP_PROTO_COMMON_INIT_FLAG_MTYPE_ZCOPY)) { + if (flags & UCP_PROTO_COMMON_INIT_FLAG_SEND_ZCOPY) { if (md_attr->flags & UCT_MD_FLAG_NEED_MEMH) { /* Memory domain must support registration on the relevant * memory type */ - if (!(context->reg_md_map[mem_type] & UCS_BIT(md_index))) { + if (!(context->reg_md_map[select_param->mem_type] & + UCS_BIT(md_index))) { ucs_trace("%s: md %s cannot register %s memory", lane_desc, context->tl_mds[md_index].rsc.md_name, - ucs_memory_type_names[mem_type]); + ucs_memory_type_names[select_param->mem_type]); continue; } - } else if (!(md_attr->access_mem_types & UCS_BIT(mem_type))) { + } else if (!(md_attr->access_mem_types & + UCS_BIT(select_param->mem_type))) { /* * Memory domain which does not require a registration for zero * copy operation must be able to access the relevant memory type */ ucs_trace("%s: no access to mem type %s", lane_desc, - ucs_memory_type_names[mem_type]); + ucs_memory_type_names[select_param->mem_type]); continue; } } @@ -598,9 +598,8 @@ ucp_proto_common_reg_md_map(const ucp_proto_common_init_params_t *params, ucp_lane_index_t ucp_proto_common_find_lanes_with_min_frag( const ucp_proto_common_init_params_t *params, ucp_lane_type_t lane_type, - ucs_memory_type_t mem_type, uint64_t tl_cap_flags, - ucp_lane_index_t max_lanes, ucp_lane_map_t exclude_map, - ucp_lane_index_t *lanes) + uint64_t tl_cap_flags, ucp_lane_index_t max_lanes, + ucp_lane_map_t exclude_map, ucp_lane_index_t *lanes) { ucp_lane_index_t lane_index, lane, num_lanes, num_valid_lanes; const uct_iface_attr_t *iface_attr; @@ -609,7 +608,7 @@ ucp_lane_index_t ucp_proto_common_find_lanes_with_min_frag( num_lanes = ucp_proto_common_find_lanes(¶ms->super, params->memtype_op, params->flags, params->max_iov_offs, params->min_iov, lane_type, - mem_type, tl_cap_flags, max_lanes, + tl_cap_flags, max_lanes, exclude_map, lanes); num_valid_lanes = 0; diff --git a/src/ucp/proto/proto_common.h b/src/ucp/proto/proto_common.h index 10aacc07a6d..6492904530b 100644 --- a/src/ucp/proto/proto_common.h +++ b/src/ucp/proto/proto_common.h @@ -61,8 +61,7 @@ typedef enum { /* Supports starting the request when its datatype iterator offset is > 0 */ UCP_PROTO_COMMON_INIT_FLAG_RESUME = UCS_BIT(10), - UCP_PROTO_COMMON_INIT_FLAG_KEEP_MD_MAP = UCS_BIT(11), - UCP_PROTO_COMMON_INIT_FLAG_MTYPE_ZCOPY = UCS_BIT(12) + UCP_PROTO_COMMON_KEEP_MD_MAP = UCS_BIT(11) } ucp_proto_common_init_flags_t; @@ -249,17 +248,16 @@ ucp_proto_common_get_lane_perf(const ucp_proto_common_init_params_t *params, /* @return number of lanes found */ ucp_lane_index_t ucp_proto_common_find_lanes_with_min_frag( const ucp_proto_common_init_params_t *params, ucp_lane_type_t lane_type, - ucs_memory_type_t mem_type, uint64_t tl_cap_flags, - ucp_lane_index_t max_lanes, ucp_lane_map_t exclude_map, - ucp_lane_index_t *lanes); + uint64_t tl_cap_flags, ucp_lane_index_t max_lanes, + ucp_lane_map_t exclude_map, ucp_lane_index_t *lanes); ucp_lane_index_t ucp_proto_common_find_lanes(const ucp_proto_init_params_t *params, uct_ep_operation_t memtype_op, unsigned flags, ptrdiff_t max_iov_offs, size_t min_iov, - ucp_lane_type_t lane_type, ucs_memory_type_t mem_type, - uint64_t tl_cap_flags, ucp_lane_index_t max_lanes, + ucp_lane_type_t lane_type, uint64_t tl_cap_flags, + ucp_lane_index_t max_lanes, ucp_lane_map_t exclude_map, ucp_lane_index_t *lanes); diff --git a/src/ucp/proto/proto_multi.c b/src/ucp/proto/proto_multi.c index 3bbdfd69982..0689866c2a3 100644 --- a/src/ucp/proto/proto_multi.c +++ b/src/ucp/proto/proto_multi.c @@ -22,7 +22,6 @@ ucs_status_t ucp_proto_multi_init(const ucp_proto_multi_init_params_t *params, ucp_proto_caps_t *caps, - ucs_memory_type_t mem_type, ucp_proto_multi_priv_t *mpriv) { ucp_context_h context = params->super.super.worker->context; @@ -56,8 +55,8 @@ ucs_status_t ucp_proto_multi_init(const ucp_proto_multi_init_params_t *params, /* Find first lane */ num_lanes = ucp_proto_common_find_lanes_with_min_frag( - ¶ms->super, params->first.lane_type, mem_type, - params->first.tl_cap_flags, 1, 0, lanes); + ¶ms->super, params->first.lane_type, params->first.tl_cap_flags, + 1, 0, lanes); if (num_lanes == 0) { ucs_trace("no lanes for %s", ucp_proto_id_field(params->super.super.proto_id, name)); @@ -66,7 +65,7 @@ ucs_status_t ucp_proto_multi_init(const ucp_proto_multi_init_params_t *params, /* Find rest of the lanes */ num_lanes += ucp_proto_common_find_lanes_with_min_frag( - ¶ms->super, params->middle.lane_type, mem_type, + ¶ms->super, params->middle.lane_type, params->middle.tl_cap_flags, UCP_PROTO_MAX_LANES - 1, UCS_BIT(lanes[0]), lanes + 1); @@ -265,9 +264,7 @@ void ucp_proto_multi_probe(const ucp_proto_multi_init_params_t *params) ucp_proto_caps_t caps; ucs_status_t status; - status = ucp_proto_multi_init(params, &caps, - params->super.super.select_param->mem_type, - &mpriv); + status = ucp_proto_multi_init(params, &caps, &mpriv); if (status != UCS_OK) { return; } diff --git a/src/ucp/proto/proto_multi.h b/src/ucp/proto/proto_multi.h index 828ceeac272..8e6dff043c7 100644 --- a/src/ucp/proto/proto_multi.h +++ b/src/ucp/proto/proto_multi.h @@ -160,7 +160,6 @@ typedef ucs_status_t (*ucp_proto_multi_lane_send_func_t)(ucp_request_t *req, ucs_status_t ucp_proto_multi_init(const ucp_proto_multi_init_params_t *params, ucp_proto_caps_t *caps, - ucs_memory_type_t mem_type, ucp_proto_multi_priv_t *mpriv); diff --git a/src/ucp/proto/proto_select.h b/src/ucp/proto/proto_select.h index 9441c3c0044..c3371d4a33f 100644 --- a/src/ucp/proto/proto_select.h +++ b/src/ucp/proto/proto_select.h @@ -82,6 +82,8 @@ struct ucp_proto_select_param { uint8_t sys_dev; /* Reply buffer system device */ } UCS_S_PACKED reply; + uint8_t mem_type_flags; + /* Align struct size to uint64_t */ uint8_t padding[2]; diff --git a/src/ucp/proto/proto_single.c b/src/ucp/proto/proto_single.c index 07cb74eb7ff..ddd754e86a2 100644 --- a/src/ucp/proto/proto_single.c +++ b/src/ucp/proto/proto_single.c @@ -30,8 +30,7 @@ ucs_status_t ucp_proto_single_init(const ucp_proto_single_init_params_t *params, ucs_status_t status; num_lanes = ucp_proto_common_find_lanes_with_min_frag( - ¶ms->super, params->lane_type, - params->super.super.select_param->mem_type, params->tl_cap_flags, 1, + ¶ms->super, params->lane_type, params->tl_cap_flags, 1, params->super.exclude_map, &lane); if (num_lanes == 0) { ucs_trace("no lanes for %s", diff --git a/src/ucp/rndv/proto_rndv.c b/src/ucp/rndv/proto_rndv.c index 524fedcc715..bdf3b37288c 100644 --- a/src/ucp/rndv/proto_rndv.c +++ b/src/ucp/rndv/proto_rndv.c @@ -269,7 +269,7 @@ ucp_proto_rndv_ctrl_init_priv(const ucp_proto_rndv_ctrl_init_params_t *params, /* Use only memory domains for which the unpacking of the remote key was * successful */ if ((init_params->rkey_config_key != NULL) && - !(params->super.flags & UCP_PROTO_COMMON_INIT_FLAG_KEEP_MD_MAP)) { + !(params->super.flags & UCP_PROTO_COMMON_KEEP_MD_MAP)) { rpriv->md_map &= ~init_params->rkey_config_key->unreachable_md_map; } @@ -519,10 +519,12 @@ ucp_proto_rndv_find_ctrl_lane(const ucp_proto_init_params_t *params) { ucp_lane_index_t lane, num_lanes; - num_lanes = ucp_proto_common_find_lanes( - params, UCT_EP_OP_LAST, UCP_PROTO_COMMON_INIT_FLAG_HDR_ONLY, - UCP_PROTO_COMMON_OFFSET_INVALID, 1, UCP_LANE_TYPE_AM, - UCS_MEMORY_TYPE_HOST, UCT_IFACE_FLAG_AM_BCOPY, 1, 0, &lane); + num_lanes = ucp_proto_common_find_lanes(params, UCT_EP_OP_LAST, + UCP_PROTO_COMMON_INIT_FLAG_HDR_ONLY, + UCP_PROTO_COMMON_OFFSET_INVALID, 1, + UCP_LANE_TYPE_AM, + UCT_IFACE_FLAG_AM_BCOPY, 1, 0, + &lane); if (num_lanes == 0) { ucs_debug("no active message lane for %s", ucp_proto_id_field(params->proto_id, name)); @@ -707,8 +709,7 @@ ucp_proto_rndv_bulk_init(const ucp_proto_multi_init_params_t *init_params, ucp_proto_caps_t multi_caps; ucs_status_t status; - status = ucp_proto_multi_init(init_params, &multi_caps, - rpriv->frag_mem_type, mpriv); + status = ucp_proto_multi_init(init_params, &multi_caps, mpriv); if (status != UCS_OK) { return status; } diff --git a/src/ucp/rndv/rndv_get.c b/src/ucp/rndv/rndv_get.c index 4f26e320772..fc850c30095 100644 --- a/src/ucp/rndv/rndv_get.c +++ b/src/ucp/rndv/rndv_get.c @@ -71,13 +71,13 @@ ucp_proto_rndv_get_common_probe(const ucp_proto_init_params_t *init_params, return; } - rpriv.frag_mem_type = frag_mem_type; status = ucp_proto_rndv_bulk_init(¶ms, UCP_PROTO_RNDV_GET_DESC, UCP_PROTO_RNDV_ATS_NAME, &rpriv, &caps); if (status != UCS_OK) { return; } + rpriv.frag_mem_type = frag_mem_type; priv_size = UCP_PROTO_MULTI_EXTENDED_PRIV_SIZE(&rpriv, mpriv); ucp_proto_common_add_proto(¶ms.super, &caps, &rpriv, priv_size); } @@ -127,7 +127,7 @@ ucp_proto_rndv_get_zcopy_probe(const ucp_proto_init_params_t *init_params) UCT_EP_OP_LAST, UCP_PROTO_COMMON_INIT_FLAG_SEND_ZCOPY | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, - 0, 0, init_params->select_param->mem_type); + 0, 0, UCS_MEMORY_TYPE_HOST); } static void @@ -287,12 +287,9 @@ static ucs_status_t ucp_proto_rndv_get_mtype_fetch_progress(uct_pending_req_t *uct_req) { ucp_request_t *req = ucs_container_of(uct_req, ucp_request_t, send.uct); - const ucp_proto_rndv_bulk_priv_t *rpriv; + const ucp_proto_rndv_bulk_priv_t *rpriv = req->send.proto_config->priv; ucs_status_t status; - /* coverity[tainted_data_downcast] */ - rpriv = req->send.proto_config->priv; - if (!(req->flags & UCP_REQUEST_FLAG_PROTO_INITIALIZED)) { status = ucp_proto_rndv_mtype_request_init(req, rpriv->frag_mem_type); if (status != UCS_OK) { @@ -325,13 +322,12 @@ ucp_proto_rndv_get_mtype_probe(const ucp_proto_init_params_t *init_params) status = ucp_proto_rndv_mtype_init(init_params, frag_mem_type, &mdesc_md_map, &frag_size); if (status != UCS_OK) { - continue; + return; } ucp_proto_rndv_get_common_probe(init_params, UCS_BIT(UCP_RNDV_MODE_GET_PIPELINE), - frag_size, UCT_EP_OP_PUT_ZCOPY, - UCP_PROTO_COMMON_INIT_FLAG_MTYPE_ZCOPY, + frag_size, UCT_EP_OP_PUT_ZCOPY, 0, mdesc_md_map, 1, frag_mem_type); } } diff --git a/src/ucp/rndv/rndv_put.c b/src/ucp/rndv/rndv_put.c index 639a77885e0..4fddf4a8216 100644 --- a/src/ucp/rndv/rndv_put.c +++ b/src/ucp/rndv/rndv_put.c @@ -278,7 +278,6 @@ ucp_proto_rndv_put_common_probe(const ucp_proto_init_params_t *init_params, return; } - rpriv.bulk.frag_mem_type = frag_mem_type; status = ucp_proto_rndv_bulk_init(¶ms, UCP_PROTO_RNDV_PUT_DESC, UCP_PROTO_RNDV_ATP_NAME, &rpriv.bulk, &caps); @@ -347,8 +346,9 @@ ucp_proto_rndv_put_common_probe(const ucp_proto_init_params_t *init_params, if (send_atp) { ucs_assert(rpriv.atp_map != 0); } - rpriv.atp_num_lanes = ucs_popcount(rpriv.atp_map); - rpriv.stat_counter = stat_counter; + rpriv.atp_num_lanes = ucs_popcount(rpriv.atp_map); + rpriv.stat_counter = stat_counter; + rpriv.bulk.frag_mem_type = frag_mem_type; priv_size = UCP_PROTO_MULTI_EXTENDED_PRIV_SIZE(&rpriv, bulk.mpriv); ucp_proto_common_add_proto(¶ms.super, &caps, &rpriv, priv_size); @@ -419,7 +419,7 @@ ucp_proto_rndv_put_zcopy_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_SEND_ZCOPY | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, 0, ucp_proto_rndv_put_zcopy_completion, 0, - UCP_WORKER_STAT_RNDV_PUT_ZCOPY, init_params->select_param->mem_type); + UCP_WORKER_STAT_RNDV_PUT_ZCOPY, UCS_MEMORY_TYPE_HOST); } static void @@ -592,9 +592,8 @@ ucp_proto_rndv_put_mtype_probe(const ucp_proto_init_params_t *init_params) ucp_proto_rndv_put_common_probe( init_params, UCS_BIT(UCP_RNDV_MODE_PUT_PIPELINE), frag_size, - UCT_EP_OP_GET_ZCOPY, UCP_PROTO_COMMON_INIT_FLAG_MTYPE_ZCOPY, - mdesc_md_map, comp_cb, 1, UCP_WORKER_STAT_RNDV_PUT_MTYPE_ZCOPY, - frag_mem_type); + UCT_EP_OP_GET_ZCOPY, 0, mdesc_md_map, comp_cb, 1, + UCP_WORKER_STAT_RNDV_PUT_MTYPE_ZCOPY, frag_mem_type); } static void diff --git a/src/ucp/rndv/rndv_rtr.c b/src/ucp/rndv/rndv_rtr.c index d6508a9a4cb..b64bac5f25d 100644 --- a/src/ucp/rndv/rndv_rtr.c +++ b/src/ucp/rndv/rndv_rtr.c @@ -389,7 +389,7 @@ ucp_proto_rndv_rtr_mtype_probe(const ucp_proto_init_params_t *init_params) .super.memtype_op = UCT_EP_OP_LAST, .super.flags = UCP_PROTO_COMMON_INIT_FLAG_RESPONSE | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING | - UCP_PROTO_COMMON_INIT_FLAG_KEEP_MD_MAP, + UCP_PROTO_COMMON_KEEP_MD_MAP, .super.exclude_map = 0, .remote_op_id = UCP_OP_ID_RNDV_SEND, .lane = ucp_proto_rndv_find_ctrl_lane(init_params), From 6f8d57a1b6b7182b7186ef9c1665f86133596841 Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Wed, 18 Sep 2024 15:55:24 +0300 Subject: [PATCH 08/14] UCP: Use reg_mem_type param instead of flag --- src/ucp/am/eager_multi.c | 2 ++ src/ucp/am/eager_single.c | 3 +++ src/ucp/proto/proto_common.c | 27 +++++++++++++-------------- src/ucp/proto/proto_common.h | 8 ++++++-- src/ucp/rma/amo_offload.c | 1 + src/ucp/rma/amo_sw.c | 1 + src/ucp/rma/get_am.c | 1 + src/ucp/rma/get_offload.c | 2 ++ src/ucp/rma/put_am.c | 1 + src/ucp/rma/put_offload.c | 3 +++ src/ucp/rndv/proto_rndv.c | 2 ++ src/ucp/rndv/rndv_am.c | 3 +++ src/ucp/rndv/rndv_get.c | 3 ++- src/ucp/rndv/rndv_put.c | 4 +++- src/ucp/rndv/rndv_rkey_ptr.c | 2 ++ src/ucp/rndv/rndv_rtr.c | 2 ++ src/ucp/stream/stream_multi.c | 2 ++ src/ucp/tag/eager_multi.c | 2 ++ src/ucp/tag/eager_single.c | 4 ++++ src/ucp/tag/offload/eager.c | 3 +++ src/ucp/tag/offload/rndv.c | 2 ++ 21 files changed, 60 insertions(+), 18 deletions(-) diff --git a/src/ucp/am/eager_multi.c b/src/ucp/am/eager_multi.c index ee40787a8ed..f2279313009 100644 --- a/src/ucp/am/eager_multi.c +++ b/src/ucp/am/eager_multi.c @@ -37,6 +37,7 @@ ucp_am_eager_multi_bcopy_proto_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING | UCP_PROTO_COMMON_INIT_FLAG_RESUME, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .max_lanes = context->config.ext.max_eager_lanes, .initial_reg_md_map = 0, .first.lane_type = UCP_LANE_TYPE_AM, @@ -197,6 +198,7 @@ ucp_am_eager_multi_zcopy_proto_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_CAP_SEG_SIZE | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, .super.exclude_map = 0, + .super.reg_mem_type = init_params->select_param->mem_type, .max_lanes = context->config.ext.max_eager_lanes, .initial_reg_md_map = 0, .opt_align_offs = UCP_PROTO_COMMON_OFFSET_INVALID, diff --git a/src/ucp/am/eager_single.c b/src/ucp/am/eager_single.c index 68bec0725ab..1eff688ade6 100644 --- a/src/ucp/am/eager_single.c +++ b/src/ucp/am/eager_single.c @@ -111,6 +111,7 @@ ucp_am_eager_short_probe_common(const ucp_proto_init_params_t *init_params, UCP_PROTO_COMMON_INIT_FLAG_CAP_SEG_SIZE | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .lane_type = UCP_LANE_TYPE_AM, .tl_cap_flags = UCT_IFACE_FLAG_AM_SHORT }; @@ -240,6 +241,7 @@ static void ucp_am_eager_single_bcopy_probe_common( UCP_PROTO_COMMON_INIT_FLAG_CAP_SEG_SIZE | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .lane_type = UCP_LANE_TYPE_AM, .tl_cap_flags = UCT_IFACE_FLAG_AM_BCOPY }; @@ -330,6 +332,7 @@ static void ucp_am_eager_single_zcopy_probe_common( UCP_PROTO_COMMON_INIT_FLAG_CAP_SEG_SIZE | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, .super.exclude_map = 0, + .super.reg_mem_type = init_params->select_param->mem_type, .lane_type = UCP_LANE_TYPE_AM, .tl_cap_flags = UCT_IFACE_FLAG_AM_ZCOPY }; diff --git a/src/ucp/proto/proto_common.c b/src/ucp/proto/proto_common.c index b93e01dc1e4..e98e006b66c 100644 --- a/src/ucp/proto/proto_common.c +++ b/src/ucp/proto/proto_common.c @@ -409,8 +409,9 @@ ucp_lane_index_t ucp_proto_common_find_lanes(const ucp_proto_init_params_t *params, uct_ep_operation_t memtype_op, unsigned flags, ptrdiff_t max_iov_offs, size_t min_iov, - ucp_lane_type_t lane_type, uint64_t tl_cap_flags, - ucp_lane_index_t max_lanes, + ucp_lane_type_t lane_type, + ucs_memory_type_t reg_mem_type, + uint64_t tl_cap_flags, ucp_lane_index_t max_lanes, ucp_lane_map_t exclude_map, ucp_lane_index_t *lanes) { UCS_STRING_BUFFER_ONSTACK(sel_param_strb, UCP_PROTO_SELECT_PARAM_STR_MAX); @@ -494,25 +495,23 @@ ucp_proto_common_find_lanes(const ucp_proto_init_params_t *params, } /* Check memory registration capabilities for zero-copy case */ - if (flags & UCP_PROTO_COMMON_INIT_FLAG_SEND_ZCOPY) { + if (reg_mem_type != UCS_MEMORY_TYPE_UNKNOWN) { if (md_attr->flags & UCT_MD_FLAG_NEED_MEMH) { /* Memory domain must support registration on the relevant * memory type */ - if (!(context->reg_md_map[select_param->mem_type] & - UCS_BIT(md_index))) { + if (!(context->reg_md_map[reg_mem_type] & UCS_BIT(md_index))) { ucs_trace("%s: md %s cannot register %s memory", lane_desc, context->tl_mds[md_index].rsc.md_name, - ucs_memory_type_names[select_param->mem_type]); + ucs_memory_type_names[reg_mem_type]); continue; } - } else if (!(md_attr->access_mem_types & - UCS_BIT(select_param->mem_type))) { + } else if (!(md_attr->access_mem_types & UCS_BIT(reg_mem_type))) { /* * Memory domain which does not require a registration for zero * copy operation must be able to access the relevant memory type */ ucs_trace("%s: no access to mem type %s", lane_desc, - ucs_memory_type_names[select_param->mem_type]); + ucs_memory_type_names[reg_mem_type]); continue; } } @@ -605,11 +604,11 @@ ucp_lane_index_t ucp_proto_common_find_lanes_with_min_frag( const uct_iface_attr_t *iface_attr; size_t tl_min_frag, tl_max_frag; - num_lanes = ucp_proto_common_find_lanes(¶ms->super, params->memtype_op, - params->flags, params->max_iov_offs, - params->min_iov, lane_type, - tl_cap_flags, max_lanes, - exclude_map, lanes); + num_lanes = ucp_proto_common_find_lanes( + ¶ms->super, params->memtype_op, params->flags, + params->max_iov_offs, params->min_iov, lane_type, + params->reg_mem_type, tl_cap_flags, max_lanes, exclude_map, + lanes); num_valid_lanes = 0; for (lane_index = 0; lane_index < num_lanes; ++lane_index) { diff --git a/src/ucp/proto/proto_common.h b/src/ucp/proto/proto_common.h index 6492904530b..5d0625fef71 100644 --- a/src/ucp/proto/proto_common.h +++ b/src/ucp/proto/proto_common.h @@ -121,6 +121,9 @@ typedef struct { /* Map of unsuitable lanes */ ucp_lane_map_t exclude_map; + + /* Memory type that the transport should be capable of registering. */ + ucs_memory_type_t reg_mem_type; } ucp_proto_common_init_params_t; @@ -256,8 +259,9 @@ ucp_lane_index_t ucp_proto_common_find_lanes(const ucp_proto_init_params_t *params, uct_ep_operation_t memtype_op, unsigned flags, ptrdiff_t max_iov_offs, size_t min_iov, - ucp_lane_type_t lane_type, uint64_t tl_cap_flags, - ucp_lane_index_t max_lanes, + ucp_lane_type_t lane_type, + ucs_memory_type_t reg_mem_type, + uint64_t tl_cap_flags, ucp_lane_index_t max_lanes, ucp_lane_map_t exclude_map, ucp_lane_index_t *lanes); diff --git a/src/ucp/rma/amo_offload.c b/src/ucp/rma/amo_offload.c index b35c36d8c6c..76a478b2a7c 100644 --- a/src/ucp/rma/amo_offload.c +++ b/src/ucp/rma/amo_offload.c @@ -171,6 +171,7 @@ static void ucp_proto_amo_probe(const ucp_proto_init_params_t *init_params, UCP_PROTO_COMMON_INIT_FLAG_RECV_ZCOPY | UCP_PROTO_COMMON_INIT_FLAG_SINGLE_FRAG, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .lane_type = UCP_LANE_TYPE_AMO, .tl_cap_flags = 0 }; diff --git a/src/ucp/rma/amo_sw.c b/src/ucp/rma/amo_sw.c index 21ca2da90a6..f8859502b2b 100644 --- a/src/ucp/rma/amo_sw.c +++ b/src/ucp/rma/amo_sw.c @@ -424,6 +424,7 @@ static void ucp_proto_amo_sw_probe(const ucp_proto_init_params_t *init_params, .super.flags = flags | UCP_PROTO_COMMON_INIT_FLAG_SINGLE_FRAG | UCP_PROTO_COMMON_INIT_FLAG_CAP_SEG_SIZE, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .lane_type = UCP_LANE_TYPE_AM, .tl_cap_flags = 0 }; diff --git a/src/ucp/rma/get_am.c b/src/ucp/rma/get_am.c index 03585204b5f..4d6a6960b9f 100644 --- a/src/ucp/rma/get_am.c +++ b/src/ucp/rma/get_am.c @@ -95,6 +95,7 @@ ucp_proto_get_am_bcopy_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_CAP_SEG_SIZE | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .lane_type = UCP_LANE_TYPE_AM, .tl_cap_flags = UCT_IFACE_FLAG_AM_BCOPY }; diff --git a/src/ucp/rma/get_offload.c b/src/ucp/rma/get_offload.c index 5f01c879087..2b824bd40e0 100644 --- a/src/ucp/rma/get_offload.c +++ b/src/ucp/rma/get_offload.c @@ -96,6 +96,7 @@ ucp_proto_get_offload_bcopy_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_REMOTE_ACCESS | UCP_PROTO_COMMON_INIT_FLAG_RESPONSE, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .max_lanes = UCP_PROTO_RMA_MAX_BCOPY_LANES, .initial_reg_md_map = 0, .first.tl_cap_flags = UCT_IFACE_FLAG_GET_BCOPY, @@ -202,6 +203,7 @@ ucp_proto_get_offload_zcopy_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_RESPONSE | UCP_PROTO_COMMON_INIT_FLAG_MIN_FRAG, .super.exclude_map = 0, + .super.reg_mem_type = init_params->select_param->mem_type, .max_lanes = context->config.ext.max_rma_lanes, .initial_reg_md_map = 0, .first.tl_cap_flags = UCT_IFACE_FLAG_GET_ZCOPY, diff --git a/src/ucp/rma/put_am.c b/src/ucp/rma/put_am.c index 1150978951d..6e73626298e 100644 --- a/src/ucp/rma/put_am.c +++ b/src/ucp/rma/put_am.c @@ -97,6 +97,7 @@ ucp_proto_put_am_bcopy_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING | UCP_PROTO_COMMON_INIT_FLAG_RESUME, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .max_lanes = 1, .initial_reg_md_map = 0, .first.tl_cap_flags = UCT_IFACE_FLAG_AM_BCOPY, diff --git a/src/ucp/rma/put_offload.c b/src/ucp/rma/put_offload.c index e108d8bbb27..63bf4a45c0e 100644 --- a/src/ucp/rma/put_offload.c +++ b/src/ucp/rma/put_offload.c @@ -70,6 +70,7 @@ ucp_proto_put_offload_short_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_SINGLE_FRAG | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .lane_type = UCP_LANE_TYPE_RMA, .tl_cap_flags = UCT_IFACE_FLAG_PUT_SHORT }; @@ -166,6 +167,7 @@ ucp_proto_put_offload_bcopy_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_REMOTE_ACCESS | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .max_lanes = UCP_PROTO_RMA_MAX_BCOPY_LANES, .initial_reg_md_map = 0, .first.tl_cap_flags = UCT_IFACE_FLAG_PUT_BCOPY, @@ -254,6 +256,7 @@ ucp_proto_put_offload_zcopy_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_REMOTE_ACCESS | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, .super.exclude_map = 0, + .super.reg_mem_type = init_params->select_param->mem_type, .max_lanes = context->config.ext.max_rma_lanes, .initial_reg_md_map = 0, .first.tl_cap_flags = UCT_IFACE_FLAG_PUT_ZCOPY, diff --git a/src/ucp/rndv/proto_rndv.c b/src/ucp/rndv/proto_rndv.c index bdf3b37288c..a186e42c85c 100644 --- a/src/ucp/rndv/proto_rndv.c +++ b/src/ucp/rndv/proto_rndv.c @@ -523,6 +523,7 @@ ucp_proto_rndv_find_ctrl_lane(const ucp_proto_init_params_t *params) UCP_PROTO_COMMON_INIT_FLAG_HDR_ONLY, UCP_PROTO_COMMON_OFFSET_INVALID, 1, UCP_LANE_TYPE_AM, + UCS_MEMORY_TYPE_UNKNOWN, UCT_IFACE_FLAG_AM_BCOPY, 1, 0, &lane); if (num_lanes == 0) { @@ -557,6 +558,7 @@ void ucp_proto_rndv_rts_probe(const ucp_proto_init_params_t *init_params) .super.flags = UCP_PROTO_COMMON_INIT_FLAG_RESPONSE | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .remote_op_id = UCP_OP_ID_RNDV_RECV, .lane = ucp_proto_rndv_find_ctrl_lane(init_params), .unpack_time = UCS_LINEAR_FUNC_ZERO, diff --git a/src/ucp/rndv/rndv_am.c b/src/ucp/rndv/rndv_am.c index 293a2e58f11..2f5ff27fe39 100644 --- a/src/ucp/rndv/rndv_am.c +++ b/src/ucp/rndv/rndv_am.c @@ -123,6 +123,7 @@ static void ucp_rndv_am_bcopy_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING | UCP_PROTO_COMMON_INIT_FLAG_RESUME, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .first.tl_cap_flags = UCT_IFACE_FLAG_AM_BCOPY, .middle.tl_cap_flags = UCT_IFACE_FLAG_AM_BCOPY }; @@ -199,6 +200,8 @@ static void ucp_rndv_am_zcopy_probe(const ucp_proto_init_params_t *init_params) .super.flags = UCP_PROTO_COMMON_INIT_FLAG_SEND_ZCOPY | UCP_PROTO_COMMON_INIT_FLAG_CAP_SEG_SIZE | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, + .super.exclude_map = 0, + .super.reg_mem_type = init_params->select_param->mem_type, .first.tl_cap_flags = UCT_IFACE_FLAG_AM_ZCOPY, .middle.tl_cap_flags = UCT_IFACE_FLAG_AM_ZCOPY }; diff --git a/src/ucp/rndv/rndv_get.c b/src/ucp/rndv/rndv_get.c index fc850c30095..d00b1a6f86f 100644 --- a/src/ucp/rndv/rndv_get.c +++ b/src/ucp/rndv/rndv_get.c @@ -51,6 +51,7 @@ ucp_proto_rndv_get_common_probe(const ucp_proto_init_params_t *init_params, UCP_PROTO_COMMON_INIT_FLAG_RESPONSE | UCP_PROTO_COMMON_INIT_FLAG_MIN_FRAG, .super.exclude_map = 0, + .super.reg_mem_type = frag_mem_type, .max_lanes = context->config.ext.max_rndv_lanes, .initial_reg_md_map = initial_reg_md_map, .first.tl_cap_flags = UCT_IFACE_FLAG_GET_ZCOPY, @@ -127,7 +128,7 @@ ucp_proto_rndv_get_zcopy_probe(const ucp_proto_init_params_t *init_params) UCT_EP_OP_LAST, UCP_PROTO_COMMON_INIT_FLAG_SEND_ZCOPY | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, - 0, 0, UCS_MEMORY_TYPE_HOST); + 0, 0, init_params->select_param->mem_type); } static void diff --git a/src/ucp/rndv/rndv_put.c b/src/ucp/rndv/rndv_put.c index 4fddf4a8216..182221921dd 100644 --- a/src/ucp/rndv/rndv_put.c +++ b/src/ucp/rndv/rndv_put.c @@ -252,6 +252,7 @@ ucp_proto_rndv_put_common_probe(const ucp_proto_init_params_t *init_params, UCP_PROTO_COMMON_INIT_FLAG_REMOTE_ACCESS | UCP_PROTO_COMMON_INIT_FLAG_MIN_FRAG, .super.exclude_map = 0, + .super.reg_mem_type = frag_mem_type, .max_lanes = context->config.ext.max_rndv_lanes, .initial_reg_md_map = initial_reg_md_map, .first.tl_cap_flags = UCT_IFACE_FLAG_PUT_ZCOPY, @@ -419,7 +420,8 @@ ucp_proto_rndv_put_zcopy_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_SEND_ZCOPY | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, 0, ucp_proto_rndv_put_zcopy_completion, 0, - UCP_WORKER_STAT_RNDV_PUT_ZCOPY, UCS_MEMORY_TYPE_HOST); + UCP_WORKER_STAT_RNDV_PUT_ZCOPY, + init_params->select_param->mem_type); } static void diff --git a/src/ucp/rndv/rndv_rkey_ptr.c b/src/ucp/rndv/rndv_rkey_ptr.c index 3c69cf9dad5..3169a1e2023 100644 --- a/src/ucp/rndv/rndv_rkey_ptr.c +++ b/src/ucp/rndv/rndv_rkey_ptr.c @@ -57,6 +57,7 @@ ucp_proto_rndv_rkey_ptr_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_REMOTE_ACCESS | UCP_PROTO_COMMON_INIT_FLAG_SINGLE_FRAG, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .lane_type = UCP_LANE_TYPE_RKEY_PTR, .tl_cap_flags = 0, }; @@ -225,6 +226,7 @@ ucp_proto_rndv_rkey_ptr_mtype_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_REMOTE_ACCESS, .super.exclude_map = (rkey_ptr_lane == UCP_NULL_LANE) ? 0 : UCS_BIT(rkey_ptr_lane), + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .lane_type = UCP_LANE_TYPE_LAST, .tl_cap_flags = 0 }; diff --git a/src/ucp/rndv/rndv_rtr.c b/src/ucp/rndv/rndv_rtr.c index b64bac5f25d..de6a1baaf59 100644 --- a/src/ucp/rndv/rndv_rtr.c +++ b/src/ucp/rndv/rndv_rtr.c @@ -177,6 +177,7 @@ static void ucp_proto_rndv_rtr_probe(const ucp_proto_init_params_t *init_params) .super.flags = UCP_PROTO_COMMON_INIT_FLAG_RESPONSE | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .remote_op_id = UCP_OP_ID_RNDV_SEND, .lane = ucp_proto_rndv_find_ctrl_lane(init_params), .unpack_time = UCS_LINEAR_FUNC_ZERO, @@ -391,6 +392,7 @@ ucp_proto_rndv_rtr_mtype_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING | UCP_PROTO_COMMON_KEEP_MD_MAP, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .remote_op_id = UCP_OP_ID_RNDV_SEND, .lane = ucp_proto_rndv_find_ctrl_lane(init_params), .perf_bias = 0.0, diff --git a/src/ucp/stream/stream_multi.c b/src/ucp/stream/stream_multi.c index 511ae9f5a9b..c61cb7a59b9 100644 --- a/src/ucp/stream/stream_multi.c +++ b/src/ucp/stream/stream_multi.c @@ -92,6 +92,7 @@ ucp_stream_multi_bcopy_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING | UCP_PROTO_COMMON_INIT_FLAG_RESUME, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .max_lanes = 1, .initial_reg_md_map = 0, .opt_align_offs = UCP_PROTO_COMMON_OFFSET_INVALID, @@ -165,6 +166,7 @@ ucp_stream_multi_zcopy_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_CAP_SEG_SIZE | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, .super.exclude_map = 0, + .super.reg_mem_type = init_params->select_param->mem_type, .max_lanes = 1, .initial_reg_md_map = 0, .opt_align_offs = UCP_PROTO_COMMON_OFFSET_INVALID, diff --git a/src/ucp/tag/eager_multi.c b/src/ucp/tag/eager_multi.c index 184526fc572..9ed442a359a 100644 --- a/src/ucp/tag/eager_multi.c +++ b/src/ucp/tag/eager_multi.c @@ -70,6 +70,7 @@ static void ucp_proto_eager_bcopy_multi_common_probe( UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING | UCP_PROTO_COMMON_INIT_FLAG_RESUME, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .opt_align_offs = UCP_PROTO_COMMON_OFFSET_INVALID, .first.tl_cap_flags = UCT_IFACE_FLAG_AM_BCOPY, .middle.tl_cap_flags = UCT_IFACE_FLAG_AM_BCOPY @@ -241,6 +242,7 @@ ucp_proto_eager_zcopy_multi_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_CAP_SEG_SIZE | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, .super.exclude_map = 0, + .super.reg_mem_type = init_params->select_param->mem_type, .opt_align_offs = UCP_PROTO_COMMON_OFFSET_INVALID, .first.tl_cap_flags = UCT_IFACE_FLAG_AM_ZCOPY, .middle.tl_cap_flags = UCT_IFACE_FLAG_AM_ZCOPY diff --git a/src/ucp/tag/eager_single.c b/src/ucp/tag/eager_single.c index 79d508a1c1f..2ff2d2c6480 100644 --- a/src/ucp/tag/eager_single.c +++ b/src/ucp/tag/eager_single.c @@ -67,6 +67,7 @@ ucp_proto_eager_short_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_CAP_SEG_SIZE | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .lane_type = UCP_LANE_TYPE_AM, .tl_cap_flags = UCT_IFACE_FLAG_AM_SHORT }; @@ -139,6 +140,8 @@ ucp_proto_eager_bcopy_single_probe(const ucp_proto_init_params_t *init_params) .super.flags = UCP_PROTO_COMMON_INIT_FLAG_SINGLE_FRAG | UCP_PROTO_COMMON_INIT_FLAG_CAP_SEG_SIZE | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, + .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .lane_type = UCP_LANE_TYPE_AM, .tl_cap_flags = UCT_IFACE_FLAG_AM_BCOPY }; @@ -186,6 +189,7 @@ ucp_proto_eager_zcopy_single_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_CAP_SEG_SIZE | UCP_PROTO_COMMON_INIT_FLAG_ERR_HANDLING, .super.exclude_map = 0, + .super.reg_mem_type = init_params->select_param->mem_type, .lane_type = UCP_LANE_TYPE_AM, .tl_cap_flags = UCT_IFACE_FLAG_AM_ZCOPY }; diff --git a/src/ucp/tag/offload/eager.c b/src/ucp/tag/offload/eager.c index 893b01738a2..55430bb3e54 100644 --- a/src/ucp/tag/offload/eager.c +++ b/src/ucp/tag/offload/eager.c @@ -65,6 +65,7 @@ static void ucp_proto_eager_tag_offload_short_probe( UCP_PROTO_COMMON_INIT_FLAG_RECV_ZCOPY | UCP_PROTO_COMMON_INIT_FLAG_CAP_SEG_SIZE, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .lane_type = UCP_LANE_TYPE_TAG, .tl_cap_flags = UCT_IFACE_FLAG_TAG_EAGER_SHORT }; @@ -139,6 +140,7 @@ static void ucp_proto_eager_tag_offload_bcopy_probe_common( UCP_PROTO_COMMON_INIT_FLAG_RECV_ZCOPY | UCP_PROTO_COMMON_INIT_FLAG_CAP_SEG_SIZE, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .lane_type = UCP_LANE_TYPE_TAG, .tl_cap_flags = UCT_IFACE_FLAG_TAG_EAGER_BCOPY }; @@ -249,6 +251,7 @@ static void ucp_proto_eager_tag_offload_zcopy_probe_common( UCP_PROTO_COMMON_INIT_FLAG_SINGLE_FRAG | UCP_PROTO_COMMON_INIT_FLAG_CAP_SEG_SIZE, .super.exclude_map = 0, + .super.reg_mem_type = init_params->select_param->mem_type, .lane_type = UCP_LANE_TYPE_TAG, .tl_cap_flags = UCT_IFACE_FLAG_TAG_EAGER_ZCOPY }; diff --git a/src/ucp/tag/offload/rndv.c b/src/ucp/tag/offload/rndv.c index fa9e60d9fa0..aa6d2a187b1 100644 --- a/src/ucp/tag/offload/rndv.c +++ b/src/ucp/tag/offload/rndv.c @@ -45,6 +45,7 @@ ucp_tag_rndv_offload_proto_probe(const ucp_proto_init_params_t *init_params) UCP_PROTO_COMMON_INIT_FLAG_RECV_ZCOPY | UCP_PROTO_COMMON_INIT_FLAG_SINGLE_FRAG, .super.exclude_map = 0, + .super.reg_mem_type = init_params->select_param->mem_type, .lane_type = UCP_LANE_TYPE_TAG, .tl_cap_flags = UCT_IFACE_FLAG_TAG_RNDV_ZCOPY }; @@ -179,6 +180,7 @@ ucp_tag_rndv_offload_sw_proto_probe(const ucp_proto_init_params_t *init_params) .super.memtype_op = UCT_EP_OP_LAST, .super.flags = UCP_PROTO_COMMON_INIT_FLAG_RESPONSE, .super.exclude_map = 0, + .super.reg_mem_type = UCS_MEMORY_TYPE_UNKNOWN, .remote_op_id = UCP_OP_ID_RNDV_RECV, .lane = init_params->ep_config_key->tag_lane, .unpack_time = UCS_LINEAR_FUNC_ZERO, From 7b78980a8c5aa08ab7b8958fd594d36bd71f52a2 Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Wed, 18 Sep 2024 16:01:59 +0300 Subject: [PATCH 09/14] UCP: minor fixes --- src/ucp/proto/proto_select.h | 2 -- src/ucp/rndv/rndv_get.c | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ucp/proto/proto_select.h b/src/ucp/proto/proto_select.h index c3371d4a33f..9441c3c0044 100644 --- a/src/ucp/proto/proto_select.h +++ b/src/ucp/proto/proto_select.h @@ -82,8 +82,6 @@ struct ucp_proto_select_param { uint8_t sys_dev; /* Reply buffer system device */ } UCS_S_PACKED reply; - uint8_t mem_type_flags; - /* Align struct size to uint64_t */ uint8_t padding[2]; diff --git a/src/ucp/rndv/rndv_get.c b/src/ucp/rndv/rndv_get.c index d00b1a6f86f..3185df17363 100644 --- a/src/ucp/rndv/rndv_get.c +++ b/src/ucp/rndv/rndv_get.c @@ -323,7 +323,7 @@ ucp_proto_rndv_get_mtype_probe(const ucp_proto_init_params_t *init_params) status = ucp_proto_rndv_mtype_init(init_params, frag_mem_type, &mdesc_md_map, &frag_size); if (status != UCS_OK) { - return; + continue; } ucp_proto_rndv_get_common_probe(init_params, From 08a8ba7ae9d5800c85488d59f6fadfbba4160203 Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Wed, 18 Sep 2024 19:26:07 +0300 Subject: [PATCH 10/14] UCP: Fix coverity errors --- src/ucp/rndv/proto_rndv.c | 2 ++ src/ucp/rndv/rndv.c | 9 ++++++++- src/ucp/rndv/rndv_get.c | 7 +++++-- src/ucp/rndv/rndv_mtype.inl | 10 +++++++--- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/ucp/rndv/proto_rndv.c b/src/ucp/rndv/proto_rndv.c index a186e42c85c..a88bb60b30d 100644 --- a/src/ucp/rndv/proto_rndv.c +++ b/src/ucp/rndv/proto_rndv.c @@ -201,6 +201,8 @@ ucp_proto_rndv_ctrl_perf(const ucp_proto_init_params_t *params, return UCS_OK; } + ucs_assert(lane < UCP_MAX_LANES); + perf_attr.field_mask = UCT_PERF_ATTR_FIELD_OPERATION | UCT_PERF_ATTR_FIELD_SEND_PRE_OVERHEAD | UCT_PERF_ATTR_FIELD_SEND_POST_OVERHEAD | diff --git a/src/ucp/rndv/rndv.c b/src/ucp/rndv/rndv.c index 90dba6f211e..b4306157d00 100644 --- a/src/ucp/rndv/rndv.c +++ b/src/ucp/rndv/rndv.c @@ -24,6 +24,8 @@ static UCS_F_ALWAYS_INLINE int ucp_rndv_frag_memtype(ucp_context_t *context) { + ucs_memory_type_t frag_mem_type; + if (context->config.ext.rndv_frag_mem_types == 0) { return UCS_MEMORY_TYPE_HOST; } @@ -32,7 +34,12 @@ ucp_rndv_frag_memtype(ucp_context_t *context) * first one from the map. Anyway for proto v1 UCX_RNDV_FRAG_MEM_TYPE is * supposed to be used. */ - return ucs_ffs64(context->config.ext.rndv_frag_mem_types); + frag_mem_type = ucs_ffs64(context->config.ext.rndv_frag_mem_types); + + ucs_assertv(frag_mem_type < UCS_MEMORY_TYPE_UNKNOWN, "frag_mem_type = %u", + frag_mem_type); + + return frag_mem_type; } static UCS_F_ALWAYS_INLINE int diff --git a/src/ucp/rndv/rndv_get.c b/src/ucp/rndv/rndv_get.c index 3185df17363..b338ccbdc89 100644 --- a/src/ucp/rndv/rndv_get.c +++ b/src/ucp/rndv/rndv_get.c @@ -79,7 +79,7 @@ ucp_proto_rndv_get_common_probe(const ucp_proto_init_params_t *init_params, } rpriv.frag_mem_type = frag_mem_type; - priv_size = UCP_PROTO_MULTI_EXTENDED_PRIV_SIZE(&rpriv, mpriv); + priv_size = UCP_PROTO_MULTI_EXTENDED_PRIV_SIZE(&rpriv, mpriv); ucp_proto_common_add_proto(¶ms.super, &caps, &rpriv, priv_size); } @@ -288,9 +288,12 @@ static ucs_status_t ucp_proto_rndv_get_mtype_fetch_progress(uct_pending_req_t *uct_req) { ucp_request_t *req = ucs_container_of(uct_req, ucp_request_t, send.uct); - const ucp_proto_rndv_bulk_priv_t *rpriv = req->send.proto_config->priv; + const ucp_proto_rndv_bulk_priv_t *rpriv; ucs_status_t status; + /* coverity[tainted_data_downcast] */ + rpriv = req->send.proto_config->priv; + if (!(req->flags & UCP_REQUEST_FLAG_PROTO_INITIALIZED)) { status = ucp_proto_rndv_mtype_request_init(req, rpriv->frag_mem_type); if (status != UCS_OK) { diff --git a/src/ucp/rndv/rndv_mtype.inl b/src/ucp/rndv/rndv_mtype.inl index 18911e50879..d04cd4447e3 100644 --- a/src/ucp/rndv/rndv_mtype.inl +++ b/src/ucp/rndv/rndv_mtype.inl @@ -176,13 +176,17 @@ ucp_proto_rndv_mtype_query_desc(const ucp_proto_query_params_t *params, UCS_STRING_BUFFER_FIXED(strb, attr->desc, sizeof(attr->desc)); ucp_context_h context = params->worker->context; ucs_memory_type_t mem_type = params->select_param->mem_type; - ucp_ep_h mtype_ep = ucp_proto_rndv_mtype_ep(params->worker, - frag_mem_type, - mem_type); + ucp_ep_h mtype_ep; ucp_lane_index_t lane; ucp_rsc_index_t rsc_index; const char *tl_name; + /* Make coverity happy */ + ucs_assertv(frag_mem_type < UCS_MEMORY_TYPE_UNKNOWN, "frag_mem_type = %u", + frag_mem_type); + + mtype_ep = ucp_proto_rndv_mtype_ep(params->worker, frag_mem_type, + mem_type); ucs_assert(mtype_ep != NULL); lane = ucp_ep_config(mtype_ep)->key.rma_bw_lanes[0]; From c1f718ce3767a6461ff9cebe0da04ceb9c25df21 Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Wed, 18 Sep 2024 22:06:29 +0300 Subject: [PATCH 11/14] UCP: Fix comment --- src/ucp/rndv/rndv_put.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ucp/rndv/rndv_put.c b/src/ucp/rndv/rndv_put.c index 182221921dd..b87756f0885 100644 --- a/src/ucp/rndv/rndv_put.c +++ b/src/ucp/rndv/rndv_put.c @@ -572,11 +572,12 @@ ucp_proto_rndv_put_mtype_probe(const ucp_proto_init_params_t *init_params) ucs_memory_type_t frag_mem_type; if (init_params->rkey_config_key == NULL) { - /* FIXME: maybe can initialize proto with all available types if no - * rkey in RTR. - */ frag_mem_type = UCS_MEMORY_TYPE_HOST; } else { + /* Can initialize only the same fragment type as received in RTR + * because pipeline protocols assume that both peers use the same + * fragment sizes (and they are different for different memory types by + * default). */ frag_mem_type = init_params->rkey_config_key->mem_type; } From 8ce64096ba68f32fb1d4d5795bcc3a7addf2f074 Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Thu, 19 Sep 2024 19:28:10 +0300 Subject: [PATCH 12/14] UCP: Fix send perf estimation --- src/ucp/proto/proto_init.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ucp/proto/proto_init.c b/src/ucp/proto/proto_init.c index c3da5194ec3..452fcea2347 100644 --- a/src/ucp/proto/proto_init.c +++ b/src/ucp/proto/proto_init.c @@ -497,8 +497,8 @@ ucp_proto_common_init_send_perf(const ucp_proto_common_init_params_t *params, ucs_assert(reg_md_map == 0); if ((params->flags & UCP_PROTO_COMMON_INIT_FLAG_REMOTE_ACCESS) && - (params->super.rkey_config_key != NULL)) { - remote_mem_type = params->super.rkey_config_key->mem_type; + (params->reg_mem_type != UCS_MEMORY_TYPE_UNKNOWN)) { + remote_mem_type = params->reg_mem_type; } else { remote_mem_type = UCS_MEMORY_TYPE_HOST; } From c041cdfbef5ffe41c08e81ab2ec443f1e12b6e86 Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Fri, 20 Sep 2024 09:44:25 +0300 Subject: [PATCH 13/14] UCP: Do not use cuda staging if can't alloc cuda mem --- src/ucp/core/ucp_mm.c | 19 ++++++++++--------- src/ucp/rndv/rndv_rtr.c | 19 +++++++++++-------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/ucp/core/ucp_mm.c b/src/ucp/core/ucp_mm.c index 1a47a21d078..4fd056eef1e 100644 --- a/src/ucp/core/ucp_mm.c +++ b/src/ucp/core/ucp_mm.c @@ -1700,27 +1700,28 @@ ucs_status_t ucp_mm_get_alloc_md_index(ucp_context_h context, ucp_md_index_t *md_idx, ucs_memory_type_t alloc_mem_type) { - ucs_status_t status; + ucs_status_t status = UCS_OK; uct_allocated_memory_t mem; if (!context->alloc_md[alloc_mem_type].initialized) { + context->alloc_md[alloc_mem_type].initialized = 1; + status = ucp_mem_do_alloc(context, NULL, 1, UCT_MD_MEM_ACCESS_RMA | UCT_MD_MEM_FLAG_HIDE_ERRORS, alloc_mem_type, "get_alloc_md_id", &mem); - if (status != UCS_OK) { - return status; + if (status == UCS_OK) { + context->alloc_md[alloc_mem_type].md_index = + ucp_mem_get_md_index(context, mem.md, mem.method); + uct_mem_free(&mem); + } else { + context->alloc_md[alloc_mem_type].md_index = UCP_NULL_RESOURCE; } - - context->alloc_md[alloc_mem_type].initialized = 1; - context->alloc_md[alloc_mem_type].md_index = - ucp_mem_get_md_index(context, mem.md, mem.method); - uct_mem_free(&mem); } *md_idx = context->alloc_md[alloc_mem_type].md_index; - return UCS_OK; + return status; } static ucs_status_t diff --git a/src/ucp/rndv/rndv_rtr.c b/src/ucp/rndv/rndv_rtr.c index de6a1baaf59..85ad4ddf631 100644 --- a/src/ucp/rndv/rndv_rtr.c +++ b/src/ucp/rndv/rndv_rtr.c @@ -418,21 +418,24 @@ ucp_proto_rndv_rtr_mtype_probe(const ucp_proto_init_params_t *init_params) continue; } + status = ucp_mm_get_alloc_md_index(context, &md_index, frag_mem_type); + if ((status == UCS_OK) && (md_index != UCP_NULL_RESOURCE)) { + params.md_map = UCS_BIT(md_index); + } else if (frag_mem_type != UCS_MEMORY_TYPE_HOST) { + /* To use non-host staging buffers it should be possible to + * allocate them with MD */ + continue; + } + params.mem_info.type = frag_mem_type; + params.md_map = 0; status = ucp_proto_init_buffer_copy_time( init_params->worker, "rtr/mtype unpack", frag_mem_type, init_params->select_param->mem_type, UCT_EP_OP_PUT_ZCOPY, ¶ms.unpack_time, ¶ms.unpack_perf_node); if (status != UCS_OK) { - return; - } - - status = ucp_mm_get_alloc_md_index(context, &md_index, frag_mem_type); - if ((status != UCS_OK) || (md_index == UCP_NULL_RESOURCE)) { - params.md_map = 0; - } else { - params.md_map = UCS_BIT(md_index); + continue; } rpriv.super.pack_cb = ucp_proto_rndv_rtr_mtype_pack; From bbc40bef18f9d6bffdb48b27de47ddd5645cb87d Mon Sep 17 00:00:00 2001 From: Mikhail Brinskii Date: Sun, 22 Sep 2024 12:27:58 +0300 Subject: [PATCH 14/14] UCP: Do not use cuda staging ppl if dev context is not set --- src/ucp/core/ucp_mm.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/ucp/core/ucp_mm.c b/src/ucp/core/ucp_mm.c index 4fd056eef1e..3b8f77e0f5c 100644 --- a/src/ucp/core/ucp_mm.c +++ b/src/ucp/core/ucp_mm.c @@ -1696,6 +1696,32 @@ void ucp_mem_rcache_cleanup(ucp_context_h context) } } +static int ucp_mm_alloc_memtype_device_ready(ucp_context_h context, + ucs_memory_type_t mem_type) +{ + const ucp_tl_resource_desc_t *rsc; + const ucp_tl_md_t *md; + ucp_rsc_index_t rsc_index; + + for (rsc_index = 0; rsc_index < context->num_tls; ++rsc_index) { + rsc = &context->tl_rscs[rsc_index]; + md = &context->tl_mds[rsc->md_index]; + + if (!(md->attr.alloc_mem_types & UCS_BIT(mem_type))) { + continue; + } + + if (rsc->tl_rsc.sys_device != UCS_SYS_DEVICE_ID_UNKNOWN) { + return 1; + } + + ucs_debug("no device found for md %s to alloc %s memory", + md->rsc.md_name, ucs_memory_type_names[mem_type]); + } + + return 0; +} + ucs_status_t ucp_mm_get_alloc_md_index(ucp_context_h context, ucp_md_index_t *md_idx, ucs_memory_type_t alloc_mem_type) @@ -1706,6 +1732,13 @@ ucp_mm_get_alloc_md_index(ucp_context_h context, ucp_md_index_t *md_idx, if (!context->alloc_md[alloc_mem_type].initialized) { context->alloc_md[alloc_mem_type].initialized = 1; + if ((alloc_mem_type != UCS_MEMORY_TYPE_HOST) && + !ucp_mm_alloc_memtype_device_ready(context, alloc_mem_type)) { + context->alloc_md[alloc_mem_type].md_index = UCP_NULL_RESOURCE; + status = UCS_ERR_NO_DEVICE; + goto out; + } + status = ucp_mem_do_alloc(context, NULL, 1, UCT_MD_MEM_ACCESS_RMA | UCT_MD_MEM_FLAG_HIDE_ERRORS, @@ -1720,6 +1753,7 @@ ucp_mm_get_alloc_md_index(ucp_context_h context, ucp_md_index_t *md_idx, } } +out: *md_idx = context->alloc_md[alloc_mem_type].md_index; return status; }