UCP RMA: Fix put-based plugin #167
Conversation
src/ucx_rma_plugin.c
Outdated
| NCCL_PARAM(UCXAckDelay, "UCXPUT_ACK_DELAY", 1); | ||
| NCCL_PARAM(UCXAckSkip, "UCXPUT_ACK_SKIP", 0); |
There was a problem hiding this comment.
why not
| NCCL_PARAM(UCXAckDelay, "UCXPUT_ACK_DELAY", 1); | |
| NCCL_PARAM(UCXAckSkip, "UCXPUT_ACK_SKIP", 0); | |
| NCCL_PARAM(UCXAckDelay, "UCX_PUT_ACK_DELAY", 1); | |
| NCCL_PARAM(UCXAckSkip, "UCX_PUT_ACK_SKIP", 0); |
| char inflight; /* Chunk still being sent */ | ||
| char reqs; /* Count request alive */ | ||
| int sizes[NCCL_UCP_MAX_RECV]; | ||
| unsigned short id; /* Id of the origin RTR again */ |
There was a problem hiding this comment.
in general, to check if data has fully arrived (although in this case the struct occupies only 1 cacheline)
| } | ||
| static ncclResult_t nccl_ucx_rma_connect(int dev, void *listen_handle, | ||
| void **send_comm, | ||
| ncclNetDeviceHandle_t **sendDevComm) { |
There was a problem hiding this comment.
why not ncclNetDeviceHandle_v7_t?
There was a problem hiding this comment.
keep main function with latest struct definition (also matches implementation in the ib_plugin.c file).
anyways we have in net_device.h:
27 typedef ncclNetDeviceHandle_v7_t ncclNetDeviceHandle_v8_t;
28 typedef ncclNetDeviceHandle_v8_t ncclNetDeviceHandle_t;
src/ucx_rma_plugin.c
Outdated
| w->address = malloc(attr.address_length); | ||
| if (w->address == NULL) { | ||
| ucp_worker_release_address(w->ucp_worker, attr.address); | ||
| return ncclSystemError; |
| UCXCHECK(ucp_worker_query(w->ucp_worker, &attr)); | ||
|
|
||
| if (attr.thread_mode != UCS_THREAD_MODE_MULTI) { | ||
| INFO(NCCL_NET, "Thread mode multi is not supported"); |
src/ucx_rma_plugin.c
Outdated
| req->comm = comm; | ||
| req->type = NCCL_UCP_TYPE_ISEND; | ||
| req->rtr_id = rtr->id_start; | ||
| req->inflight = (UCS_PTR_STATUS(status_ptr) == UCS_INPROGRESS); |
There was a problem hiding this comment.
| req->inflight = (UCS_PTR_STATUS(status_ptr) == UCS_INPROGRESS); | |
| req->inflight = (UCS_PTR_IS_PTR(status_ptr)); |
| if (UCS_PTR_IS_ERR(req->st)) { | ||
| WARN("NET/UCX_RMA: am_send_nb failed"); | ||
| return ncclInternalError; | ||
| assert(tag != INT_MAX); |
There was a problem hiding this comment.
we use this value to mark entries as consumed later
| comm->rem_fifo.tail++; | ||
| out: | ||
| if ((*request == NULL) && (comm->total == 0)) { | ||
| ucp_worker_progress(comm->worker->ucp_worker); |
There was a problem hiding this comment.
depending on the transport, it could be that we need to progress to receive the first remote puts.
(could there also be timeouts to service?).
src/ucx_rma_plugin.c
Outdated
| assert((UCS_PTR_STATUS(status_ptr) == UCS_INPROGRESS) || | ||
| (UCS_PTR_STATUS(status_ptr) == UCS_OK)); |
There was a problem hiding this comment.
it is already checked 2 lines above
src/ucx_rma_plugin.c
Outdated
| status = nccl_ucp_shared_put(comm, atp, sizeof(*atp), remote, | ||
| &req->inflight); | ||
| req->inflight += (status == UCS_INPROGRESS); | ||
| rtr->avail -= (status == UCS_INPROGRESS) || (status == UCS_OK); |
There was a problem hiding this comment.
| rtr->avail -= (status == UCS_INPROGRESS) || (status == UCS_OK); | |
| rtr->avail -= !UCS_STATUS_IS_ERR(status); |
Fix UCP RMA plugin.