Skip to content

Commit f475cc4

Browse files
jiaxiyanjiaxiyan
committed
prov/efa: Add unit tests for efa_rma
Signed-off-by: Jessie Yang <[email protected]>
1 parent 2d4ac0e commit f475cc4

7 files changed

+342
-1
lines changed

‎prov/efa/Makefile.include

+2-1
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,8 @@ nodist_prov_efa_test_efa_unit_test_SOURCES = \
149149
prov/efa/test/efa_unit_test_mr.c \
150150
prov/efa/test/efa_unit_test_rdm_peer.c \
151151
prov/efa/test/efa_unit_test_pke.c \
152-
prov/efa/test/efa_unit_test_msg.c
152+
prov/efa/test/efa_unit_test_msg.c \
153+
prov/efa/test/efa_unit_test_rma.c
153154

154155

155156
efa_CPPFLAGS += -I$(top_srcdir)/include -I$(top_srcdir)/prov/efa/test $(cmocka_CPPFLAGS)

‎prov/efa/test/efa_unit_test_common.c

+16
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,22 @@ void efa_unit_test_construct_tmsg(struct fi_msg_tagged *tmsg, struct iovec *iov,
3535
tmsg->ignore = ignore;
3636
}
3737

38+
void efa_unit_test_construct_msg_rma(struct fi_msg_rma *msg, struct iovec *iov,
39+
void **desc, size_t iov_count,
40+
fi_addr_t addr, struct fi_rma_iov *rma_iov,
41+
size_t rma_iov_count, void *context,
42+
uint64_t data)
43+
{
44+
msg->msg_iov = iov;
45+
msg->iov_count = iov_count;
46+
msg->addr = addr;
47+
msg->context = context;
48+
msg->data = data;
49+
msg->desc = desc;
50+
msg->rma_iov = rma_iov;
51+
msg->rma_iov_count = rma_iov_count;
52+
}
53+
3854
struct fi_info *efa_unit_test_alloc_hints(enum fi_ep_type ep_type)
3955
{
4056
struct fi_info *hints;

‎prov/efa/test/efa_unit_test_mocks.c

+15
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,21 @@ int efa_mock_ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr,
219219
return mock();
220220
}
221221

222+
void efa_mock_ibv_wr_rdma_read_save_wr(struct ibv_qp_ex *qp, uint32_t rkey,
223+
uint64_t remote_addr)
224+
{
225+
g_ibv_submitted_wr_id_vec[g_ibv_submitted_wr_id_cnt] = (void *)qp->wr_id;
226+
g_ibv_submitted_wr_id_cnt++;
227+
}
228+
229+
void efa_mock_ibv_wr_rdma_write_imm_save_wr(struct ibv_qp_ex *qp, uint32_t rkey,
230+
uint64_t remote_addr,
231+
__be32 imm_data)
232+
{
233+
g_ibv_submitted_wr_id_vec[g_ibv_submitted_wr_id_cnt] = (void *) qp->wr_id;
234+
g_ibv_submitted_wr_id_cnt++;
235+
}
236+
222237
struct efa_unit_test_mocks g_efa_unit_test_mocks = {
223238
.local_host_id = 0,
224239
.peer_host_id = 0,

‎prov/efa/test/efa_unit_test_mocks.h

+7
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,13 @@ bool efa_mock_efa_device_support_unsolicited_write_recv(void);
9898
int efa_mock_ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr,
9999
struct ibv_recv_wr **bad_wr);
100100

101+
void efa_mock_ibv_wr_rdma_read_save_wr(struct ibv_qp_ex *qp, uint32_t rkey,
102+
uint64_t remote_addr);
103+
104+
void efa_mock_ibv_wr_rdma_write_imm_save_wr(struct ibv_qp_ex *qp, uint32_t rkey,
105+
uint64_t remote_addr,
106+
__be32 imm_data);
107+
101108
struct efa_unit_test_mocks
102109
{
103110
uint64_t local_host_id;

‎prov/efa/test/efa_unit_test_rma.c

+278
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,278 @@
1+
/* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only */
2+
/* SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All
3+
* rights reserved. */
4+
5+
#include "efa_unit_tests.h"
6+
#include "ofi_util.h"
7+
8+
extern struct fi_ops_rma efa_rma_ops;
9+
10+
static void test_efa_rma_prep(struct efa_resource *resource,
11+
fi_addr_t *addr)
12+
{
13+
struct ibv_qp_ex *ibv_qpx;
14+
struct efa_ep_addr raw_addr;
15+
struct efa_base_ep *base_ep;
16+
size_t raw_addr_len = sizeof(raw_addr);
17+
int ret;
18+
19+
efa_unit_test_resource_construct(resource, FI_EP_RDM);
20+
resource->ep->msg = &efa_msg_ops;
21+
22+
base_ep = container_of(resource->ep, struct efa_base_ep, util_ep.ep_fid);
23+
ibv_qpx = base_ep->qp->ibv_qp_ex;
24+
25+
ret = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
26+
assert_int_equal(ret, 0);
27+
raw_addr.qpn = 1;
28+
raw_addr.qkey = 0x1234;
29+
ret = fi_av_insert(resource->av, &raw_addr, 1, addr, 0 /* flags */,
30+
NULL /* context */);
31+
assert_int_equal(ret, 1);
32+
33+
ibv_qpx->wr_start = &efa_mock_ibv_wr_start_no_op;
34+
/* this mock will save the send work request (wr) in a global list */
35+
ibv_qpx->wr_rdma_read = &efa_mock_ibv_wr_rdma_read_save_wr;
36+
ibv_qpx->wr_rdma_write = &efa_mock_ibv_wr_rdma_write_save_wr;
37+
ibv_qpx->wr_rdma_write_imm = &efa_mock_ibv_wr_rdma_write_imm_save_wr;
38+
ibv_qpx->wr_set_inline_data_list = &efa_mock_ibv_wr_set_inline_data_list_no_op;
39+
ibv_qpx->wr_set_sge_list = &efa_mock_ibv_wr_set_sge_list_no_op;
40+
ibv_qpx->wr_set_ud_addr = &efa_mock_ibv_wr_set_ud_addr_no_op;
41+
ibv_qpx->wr_complete = &efa_mock_ibv_wr_complete_no_op;
42+
}
43+
44+
void test_efa_rma_read(struct efa_resource **state)
45+
{
46+
struct efa_resource *resource = *state;
47+
struct efa_unit_test_buff local_buff;
48+
fi_addr_t src_addr;
49+
void *desc;
50+
int ret;
51+
uint64_t remote_addr = 0x87654321;
52+
uint64_t remote_key = 123456;
53+
54+
test_efa_rma_prep(resource, &src_addr);
55+
efa_unit_test_buff_construct(&local_buff, resource, 4096 /* buff_size */);
56+
57+
desc = fi_mr_desc(local_buff.mr);
58+
59+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
60+
ret = fi_read(resource->ep, local_buff.buff, local_buff.size, desc,
61+
src_addr, remote_addr, remote_key, NULL /* context */);
62+
assert_int_equal(ret, 0);
63+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
64+
65+
efa_unit_test_buff_destruct(&local_buff);
66+
}
67+
68+
void test_efa_rma_readv(struct efa_resource **state)
69+
{
70+
struct efa_resource *resource = *state;
71+
struct efa_unit_test_buff local_buff;
72+
struct iovec iov;
73+
fi_addr_t src_addr;
74+
void *desc;
75+
int ret;
76+
uint64_t remote_addr = 0x87654321;
77+
uint64_t remote_key = 123456;
78+
79+
test_efa_rma_prep(resource, &src_addr);
80+
efa_unit_test_buff_construct(&local_buff, resource, 4096 /* buff_size */);
81+
82+
iov.iov_base = local_buff.buff;
83+
iov.iov_len = local_buff.size;
84+
desc = fi_mr_desc(local_buff.mr);
85+
86+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
87+
ret = fi_readv(resource->ep, &iov, &desc, 1, src_addr, remote_addr,
88+
remote_key, NULL /* context */);
89+
assert_int_equal(ret, 0);
90+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
91+
92+
efa_unit_test_buff_destruct(&local_buff);
93+
}
94+
95+
96+
void test_efa_rma_readmsg(struct efa_resource **state)
97+
{
98+
struct efa_resource *resource = *state;
99+
struct efa_unit_test_buff local_buff;
100+
struct iovec iov;
101+
struct fi_msg msg = {0};
102+
struct fi_rma_iov rma_iov;
103+
fi_addr_t src_addr;
104+
void *desc;
105+
int ret;
106+
107+
test_efa_rma_prep(resource, &src_addr);
108+
efa_unit_test_buff_construct(&local_buff, resource, 4096 /* buff_size */);
109+
110+
iov.iov_base = local_buff.buff;
111+
iov.iov_len = local_buff.size;
112+
rma_iov.len = local_buff.size;
113+
rma_iov.addr = 0x87654321;
114+
rma_iov.key = 123456;
115+
efa_unit_test_construct_msg_rma(&msg, &iov, &desc, 1, src_addr, &rma_iov,
116+
1, NULL, 0);
117+
118+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
119+
ret = fi_readmsg(resource->ep, &msg, 0);
120+
assert_int_equal(ret, 0);
121+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
122+
123+
efa_unit_test_buff_destruct(&local_buff);
124+
}
125+
126+
127+
void test_efa_rma_write(struct efa_resource **state)
128+
{
129+
struct efa_resource *resource = *state;
130+
struct efa_unit_test_buff local_buff;
131+
fi_addr_t dest_addr;
132+
void *desc;
133+
int ret;
134+
uint64_t remote_addr = 0x87654321;
135+
uint64_t remote_key = 123456;
136+
137+
test_efa_rma_prep(resource, &dest_addr);
138+
efa_unit_test_buff_construct(&local_buff, resource, 4096 /* buff_size */);
139+
140+
desc = fi_mr_desc(local_buff.mr);
141+
142+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
143+
ret = fi_write(resource->ep, local_buff.buff, local_buff.size, desc,
144+
dest_addr, remote_addr, remote_key, NULL /* context */);
145+
assert_int_equal(ret, 0);
146+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
147+
148+
efa_unit_test_buff_destruct(&local_buff);
149+
}
150+
151+
void test_efa_rma_writev(struct efa_resource **state)
152+
{
153+
struct efa_resource *resource = *state;
154+
struct efa_unit_test_buff local_buff;
155+
struct iovec iov;
156+
fi_addr_t dest_addr;
157+
void *desc;
158+
int ret;
159+
uint64_t remote_addr = 0x87654321;
160+
uint64_t remote_key = 123456;
161+
162+
test_efa_rma_prep(resource, &dest_addr);
163+
efa_unit_test_buff_construct(&local_buff, resource, 4096 /* buff_size */);
164+
165+
iov.iov_base = local_buff.buff;
166+
iov.iov_len = local_buff.size;
167+
desc = fi_mr_desc(local_buff.mr);
168+
169+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
170+
ret = fi_writev(resource->ep, &iov, &desc, 1, dest_addr, remote_addr,
171+
remote_key, NULL /* context */);
172+
assert_int_equal(ret, 0);
173+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
174+
175+
efa_unit_test_buff_destruct(&local_buff);
176+
}
177+
178+
void test_efa_rma_writemsg(struct efa_resource **state)
179+
{
180+
struct efa_resource *resource = *state;
181+
struct efa_unit_test_buff local_buff;
182+
struct iovec iov;
183+
struct fi_msg msg = {0};
184+
struct fi_rma_iov rma_iov;
185+
fi_addr_t src_addr;
186+
void *desc;
187+
int ret;
188+
189+
test_efa_rma_prep(resource, &src_addr);
190+
efa_unit_test_buff_construct(&local_buff, resource, 4096 /* buff_size */);
191+
192+
iov.iov_base = local_buff.buff;
193+
iov.iov_len = local_buff.size;
194+
rma_iov.len = local_buff.size;
195+
rma_iov.addr = 0x87654321;
196+
rma_iov.key = 123456;
197+
efa_unit_test_construct_msg_rma(&msg, &iov, &desc, 1, src_addr, rma_iov,
198+
1, NULL, 0);
199+
200+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
201+
ret = fi_writemsg(resource->ep, &msg, 0);
202+
assert_int_equal(ret, 0);
203+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
204+
205+
efa_unit_test_buff_destruct(&local_buff);
206+
}
207+
208+
void test_efa_rma_writedata(struct efa_resource **state)
209+
{
210+
struct efa_resource *resource = *state;
211+
struct efa_unit_test_buff local_buff;
212+
fi_addr_t dest_addr;
213+
void *desc;
214+
int ret;
215+
uint64_t remote_addr = 0x87654321;
216+
uint64_t remote_key = 123456;
217+
218+
test_efa_rma_prep(resource, &dest_addr);
219+
efa_unit_test_buff_construct(&local_buff, resource, 4096 /* buff_size */);
220+
221+
desc = fi_mr_desc(local_buff.mr);
222+
223+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
224+
ret = fi_write(resource->ep, local_buff.buff, local_buff.size, desc, 0,
225+
dest_addr, remote_addr, remote_key, NULL /* context */);
226+
assert_int_equal(ret, 0);
227+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
228+
229+
efa_unit_test_buff_destruct(&local_buff);
230+
}
231+
232+
void test_efa_rma_inject_write(struct efa_resource **state)
233+
{
234+
struct efa_resource *resource = *state;
235+
struct efa_unit_test_buff local_buff;
236+
fi_addr_t dest_addr;
237+
void *desc;
238+
int ret;
239+
uint64_t remote_addr = 0x87654321;
240+
uint64_t remote_key = 123456;
241+
242+
test_efa_rma_prep(resource, &dest_addr);
243+
efa_unit_test_buff_construct(&local_buff, resource, 32 /* buff_size */);
244+
245+
desc = fi_mr_desc(local_buff.mr);
246+
247+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
248+
ret = fi_inject(resource->ep, local_buff.buff, local_buff.size,
249+
dest_addr, remote_addr, remote_key);
250+
assert_int_equal(ret, 0);
251+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
252+
253+
efa_unit_test_buff_destruct(&local_buff);
254+
}
255+
256+
void test_efa_rma_inject_writedata(struct efa_resource **state)
257+
{
258+
struct efa_resource *resource = *state;
259+
struct efa_unit_test_buff local_buff;
260+
fi_addr_t dest_addr;
261+
void *desc;
262+
int ret;
263+
uint64_t remote_addr = 0x87654321;
264+
uint64_t remote_key = 123456;
265+
266+
test_efa_rma_prep(resource, &dest_addr);
267+
efa_unit_test_buff_construct(&local_buff, resource, 32 /* buff_size */);
268+
269+
desc = fi_mr_desc(local_buff.mr);
270+
271+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
272+
ret = fi_injectdata(resource->ep, local_buff.buff, local_buff.size, 0,
273+
dest_addr, remote_addr, remote_key);
274+
assert_int_equal(ret, 0);
275+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
276+
277+
efa_unit_test_buff_destruct(&local_buff);
278+
}

‎prov/efa/test/efa_unit_tests.c

+9
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,15 @@ int main(void)
217217
cmocka_unit_test_setup_teardown(test_efa_msg_fi_senddata, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
218218
cmocka_unit_test_setup_teardown(test_efa_msg_fi_inject, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
219219
cmocka_unit_test_setup_teardown(test_efa_msg_fi_injectdata, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
220+
cmocka_unit_test_setup_teardown(test_efa_rma_read, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
221+
cmocka_unit_test_setup_teardown(test_efa_rma_readv, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
222+
cmocka_unit_test_setup_teardown(test_efa_rma_readmsg, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
223+
cmocka_unit_test_setup_teardown(test_efa_rma_write, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
224+
cmocka_unit_test_setup_teardown(test_efa_rma_writev, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
225+
cmocka_unit_test_setup_teardown(test_efa_rma_writemsg, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
226+
cmocka_unit_test_setup_teardown(test_efa_rma_writedata, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
227+
cmocka_unit_test_setup_teardown(test_efa_rma_inject_write, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
228+
cmocka_unit_test_setup_teardown(test_efa_rma_inject_writedata, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
220229
};
221230

222231
cmocka_set_message_output(CM_OUTPUT_XML);

‎prov/efa/test/efa_unit_tests.h

+15
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,12 @@ void efa_unit_test_construct_tmsg(struct fi_msg_tagged *tmsg, struct iovec *iov,
6060
void **desc, uint64_t tag,
6161
uint64_t ignore);
6262

63+
void efa_unit_test_construct_msg_rma(struct fi_msg_rma *msg, struct iovec *iov,
64+
void **desc, size_t iov_count,
65+
fi_addr_t addr, struct fi_rma_iov *rma_iov,
66+
size_t rma_iov_count, void *context,
67+
uint64_t data);
68+
6369
void new_temp_file(char *template, size_t len);
6470

6571
struct efa_unit_test_buff {
@@ -231,6 +237,15 @@ void test_efa_msg_fi_sendmsg();
231237
void test_efa_msg_fi_senddata();
232238
void test_efa_msg_fi_inject();
233239
void test_efa_msg_fi_injectdata();
240+
void test_efa_rma_read();
241+
void test_efa_rma_readv();
242+
void test_efa_rma_readmsg();
243+
void test_efa_rma_write();
244+
void test_efa_rma_writev();
245+
void test_efa_rma_writemsg();
246+
void test_efa_rma_writedata();
247+
void test_efa_rma_inject_write();
248+
void test_efa_rma_inject_writedata();
234249

235250
static inline
236251
int efa_unit_test_get_dlist_length(struct dlist_entry *head)

0 commit comments

Comments
 (0)