Skip to content

Commit b724db8

Browse files
jiaxiyanjiaxiyan
committed
prov/efa: Add unit tests for efa_msg
Signed-off-by: Jessie Yang <[email protected]>
1 parent 8f56b12 commit b724db8

6 files changed

+154
-1
lines changed

‎prov/efa/Makefile.include

+2-1
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,8 @@ prov_efa_test_efa_unit_test_LDFLAGS = $(cmocka_rpath) $(efa_LDFLAGS) $(cmocka_LD
162162
-Wl,--wrap=ofi_cudaMalloc \
163163
-Wl,--wrap=ofi_copy_from_hmem_iov \
164164
-Wl,--wrap=efa_rdm_pke_read \
165-
-Wl,--wrap=efa_device_support_unsolicited_write_recv
165+
-Wl,--wrap=efa_device_support_unsolicited_write_recv \
166+
-Wl,--wrap=ibv_post_recv
166167

167168
if HAVE_EFADV_CQ_EX
168169
prov_efa_test_efa_unit_test_LDFLAGS += -Wl,--wrap=efadv_create_cq

‎prov/efa/test/efa_unit_test_mocks.c

+20
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@ void efa_mock_ibv_wr_send_verify_handshake_pkt_local_host_id_and_save_wr(struct
8888
return efa_mock_ibv_wr_send_save_wr(qp);
8989
}
9090

91+
void efa_mock_ibv_wr_send_imm_save_wr(struct ibv_qp_ex *qp, __be32 imm_data)
92+
{
93+
g_ibv_submitted_wr_id_vec[g_ibv_submitted_wr_id_cnt] = (void *)qp->wr_id;
94+
g_ibv_submitted_wr_id_cnt++;
95+
}
96+
9197
void efa_mock_ibv_wr_set_inline_data_list_no_op(struct ibv_qp_ex *qp,
9298
size_t num_buf,
9399
const struct ibv_data_buf *buf_list)
@@ -207,6 +213,13 @@ bool efa_mock_efa_device_support_unsolicited_write_recv()
207213
return mock();
208214
}
209215

216+
int efa_mock_ibv_post_recv_return_mock(struct ibv_qp *qp,
217+
struct ibv_recv_wr *wr,
218+
struct ibv_recv_wr **bad_wr)
219+
{
220+
return mock();
221+
}
222+
210223
struct efa_unit_test_mocks g_efa_unit_test_mocks = {
211224
.local_host_id = 0,
212225
.peer_host_id = 0,
@@ -225,6 +238,7 @@ struct efa_unit_test_mocks g_efa_unit_test_mocks = {
225238
.efa_rdm_pke_read = __real_efa_rdm_pke_read,
226239
.efa_device_support_unsolicited_write_recv = __real_efa_device_support_unsolicited_write_recv,
227240
.ibv_is_fork_initialized = __real_ibv_is_fork_initialized,
241+
.ibv_post_recv = __real_ibv_post_recv,
228242
#if HAVE_EFADV_QUERY_MR
229243
.efadv_query_mr = __real_efadv_query_mr,
230244
#endif
@@ -373,6 +387,12 @@ enum ibv_fork_status efa_mock_ibv_is_fork_initialized_return_mock(void)
373387
return mock();
374388
}
375389

390+
int __wrap_ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr,
391+
struct ibv_recv_wr **bad_wr)
392+
{
393+
return g_efa_unit_test_mocks.ibv_post_recv(qp, wr, bad_wr);
394+
}
395+
376396
#if HAVE_EFADV_QUERY_MR
377397
int __wrap_efadv_query_mr(struct ibv_mr *ibv_mr, struct efadv_mr_attr *attr, uint32_t inlen)
378398
{

‎prov/efa/test/efa_unit_test_mocks.h

+12
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,12 @@ uint32_t efa_mock_ibv_read_wc_flags_return_mock(struct ibv_cq_ex *current);
7474

7575
bool efa_mock_efadv_wc_is_unsolicited(struct efadv_cq *efadv_cq);
7676

77+
void efa_mock_ibv_wr_send_imm_save_wr(struct ibv_qp_ex *qp, __be32 imm_data);
78+
79+
int efa_mock_ibv_post_recv_return_mock(struct ibv_qp *qp,
80+
struct ibv_recv_wr *wr,
81+
struct ibv_recv_wr **bad_wr);
82+
7783
ssize_t __real_ofi_copy_from_hmem_iov(void *dest, size_t size,
7884
enum fi_hmem_iface hmem_iface, uint64_t device,
7985
const struct iovec *hmem_iov,
@@ -89,6 +95,9 @@ int __real_efa_rdm_pke_read(struct efa_rdm_ope *ope);
8995

9096
bool __real_efa_device_support_unsolicited_write_recv();
9197

98+
int __real_ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr,
99+
struct ibv_recv_wr **bad_wr);
100+
92101
int efa_mock_efa_rdm_pke_read_return_mock(struct efa_rdm_ope *ope);
93102

94103
bool efa_mock_efa_device_support_unsolicited_write_recv(void);
@@ -128,6 +137,9 @@ struct efa_unit_test_mocks
128137

129138
enum ibv_fork_status (*ibv_is_fork_initialized)(void);
130139

140+
int (*ibv_post_recv)(struct ibv_qp *qp, struct ibv_recv_wr *wr,
141+
struct ibv_recv_wr **bad_wr);
142+
131143
#if HAVE_EFADV_QUERY_MR
132144
int (*efadv_query_mr)(struct ibv_mr *ibv_mr, struct efadv_mr_attr *attr, uint32_t inlen);
133145
#endif

‎prov/efa/test/efa_unit_test_msg.c

+115
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only */
2+
/* SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All rights reserved. */
3+
4+
#include "efa_unit_tests.h"
5+
#include "ofi_util.h"
6+
#include "efa_rdm_ep.h"
7+
8+
9+
void test_efa_msg_recv(struct efa_resource **state)
10+
{
11+
struct efa_resource *resource = *state;
12+
struct iovec iov;
13+
struct efa_unit_test_buff send_buff;
14+
struct efa_ep_addr raw_addr;
15+
size_t raw_addr_len = sizeof(raw_addr);
16+
fi_addr_t addr;
17+
int ret;
18+
struct fi_msg msg = {0};
19+
struct ibv_qp_ex *ibv_qpx;
20+
21+
efa_unit_test_resource_construct(resource, FI_EP_DGRAM);
22+
efa_unit_test_buff_construct(&send_buff, resource, 4096 /* buff_size */);
23+
ibv_qpx = efa_rdm_ep->base_ep.qp->ibv_qp_ex;
24+
25+
ret = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
26+
assert_int_equal(ret, 0);
27+
raw_addr.qpn = 1;
28+
raw_addr.qkey = 0x1234;
29+
ret = fi_av_insert(resource->av, &raw_addr, 1, &addr, 0 /* flags */, NULL /* context */);
30+
assert_int_equal(ret, 1);
31+
32+
iov.iov_base = send_buff.buff;
33+
iov.iov_len = send_buff.size;
34+
35+
efa_unit_test_construct_msg(&msg, &iov, 1, addr, NULL, 0, &fi_mr_desc(send_buff.mr));
36+
37+
g_efa_unit_test_mocks.ibv_post_recv = &efa_mock_ibv_post_recv_return_mock;
38+
will_return(efa_mock_ibv_post_recv_return_mock, 0);
39+
40+
ret = fi_recv(resource->ep, send_buff.buff, send_buff.size, fi_mr_desc(send_buff.mr), addr, NULL /* context */);
41+
assert_int_equal(ret, 0);
42+
43+
// will_return(efa_mock_ibv_post_recv_return_mock, ENOMEM);
44+
// ret = fi_recv(resource->ep, send_buff.buff, send_buff.size, fi_mr_desc(send_buff.mr), addr, NULL /* context */);
45+
// assert_int_equal(ret, -FI_EAGAIN);
46+
47+
ret = fi_recvv(resource->ep, &iov, &fi_mr_desc(send_buff.mr), 1, addr, NULL /* context */);
48+
assert_int_equal(ret, 0);
49+
50+
ret = fi_recvmsg(resource->ep, &msg, 0);
51+
assert_int_equal(ret, 0);
52+
53+
efa_unit_test_buff_destruct(&send_buff);
54+
}
55+
56+
57+
void test_efa_msg_send(struct efa_resource **state)
58+
{
59+
struct efa_resource *resource = *state;
60+
struct iovec iov;
61+
struct efa_unit_test_buff send_buff;
62+
struct efa_ep_addr raw_addr;
63+
size_t raw_addr_len = sizeof(raw_addr);
64+
fi_addr_t addr;
65+
int ret;
66+
struct fi_msg msg = {0};
67+
struct ibv_qp_ex *ibv_qpx;
68+
69+
efa_unit_test_resource_construct(resource, FI_EP_DGRAM);
70+
efa_unit_test_buff_construct(&send_buff, resource, 4096 /* buff_size */);
71+
ibv_qpx = efa_rdm_ep->base_ep.qp->ibv_qp_ex;
72+
73+
ret = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
74+
assert_int_equal(ret, 0);
75+
raw_addr.qpn = 1;
76+
raw_addr.qkey = 0x1234;
77+
ret = fi_av_insert(resource->av, &raw_addr, 1, &addr, 0 /* flags */, NULL /* context */);
78+
assert_int_equal(ret, 1);
79+
80+
iov.iov_base = send_buff.buff;
81+
iov.iov_len = send_buff.size;
82+
83+
efa_unit_test_construct_msg(&msg, &iov, 1, addr, NULL, 0, &fi_mr_desc(send_buff.mr));
84+
85+
ibv_qpx->wr_start = &efa_mock_ibv_wr_start_no_op;
86+
/* this mock will save the send work request (wr) in a global list */
87+
ibv_qpx->wr_send = &efa_mock_ibv_wr_send_save_wr;
88+
ibv_qpx->wr_send_imm = &efa_mock_ibv_wr_send_imm_save_wr;
89+
ibv_qpx->wr_set_sge_list = &efa_mock_ibv_wr_set_sge_list_no_op;
90+
ibv_qpx->wr_set_ud_addr = &efa_mock_ibv_wr_set_ud_addr_no_op;
91+
ibv_qpx->wr_complete = &efa_mock_ibv_wr_complete_no_op;
92+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 0);
93+
94+
err = fi_send(resource->ep, send_buff.buff, send_buff.size, fi_mr_desc(send_buff.mr), addr, NULL /* context */);
95+
assert_int_equal(err, 0);
96+
/* fi_send() called efa_mock_ibv_wr_send_save_wr(), which saved one send_wr in g_ibv_submitted_wr_id_vec */
97+
assert_int_equal(g_ibv_submitted_wr_id_cnt, 1);
98+
99+
ret = fi_sendv(resource->ep, &iov, &fi_mr_desc(send_buff.mr), 1, addr, NULL);
100+
assert_int_equal(ret, 0);
101+
102+
ret = fi_senddata(resource->ep, send_buff.buff, send_buff.size, fi_mr_desc(send_buff.mr), 0, addr, NULL);
103+
assert_int_equal(ret, 0);
104+
105+
ret = fi_sendmsg(resource->ep, &msg, 0);
106+
assert_int_equal(ret, 0);
107+
108+
ret = fi_inject(resource->ep, send_buff.buff, send_buff.size, addr);
109+
assert_int_equal(ret, 0);
110+
111+
ret = fi_injectdata(resource->ep, send_buff.buff, send_buff.size, addr);
112+
assert_int_equal(ret, 0);
113+
114+
efa_unit_test_buff_destruct(&send_buff);
115+
}

‎prov/efa/test/efa_unit_tests.c

+3
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ static int efa_unit_test_mocks_teardown(void **state)
6363
.efa_rdm_pke_read = __real_efa_rdm_pke_read,
6464
.efa_device_support_unsolicited_write_recv = __real_efa_device_support_unsolicited_write_recv,
6565
.ibv_is_fork_initialized = __real_ibv_is_fork_initialized,
66+
.ibv_post_recv = __real_ibv_post_recv,
6667
};
6768

6869
/* Reset environment */
@@ -206,6 +207,8 @@ int main(void)
206207
cmocka_unit_test_setup_teardown(test_efa_rdm_peer_keep_pke_in_overflow_list, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
207208
cmocka_unit_test_setup_teardown(test_efa_rdm_peer_append_overflow_pke_to_recvwin, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
208209
cmocka_unit_test_setup_teardown(test_efa_rdm_pke_handle_longcts_rtm_send_completion, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
210+
cmocka_unit_test_setup_teardown(test_efa_msg_recv, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
211+
cmocka_unit_test_setup_teardown(test_efa_msg_send, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
209212
};
210213

211214
cmocka_set_message_output(CM_OUTPUT_XML);

‎prov/efa/test/efa_unit_tests.h

+2
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,8 @@ void test_efa_rdm_peer_move_overflow_pke_to_recvwin();
220220
void test_efa_rdm_peer_keep_pke_in_overflow_list();
221221
void test_efa_rdm_peer_append_overflow_pke_to_recvwin();
222222
void test_efa_rdm_pke_handle_longcts_rtm_send_completion();
223+
void test_efa_msg_recv();
224+
void test_efa_msg_send();
223225

224226
static inline
225227
int efa_unit_test_get_dlist_length(struct dlist_entry *head)

0 commit comments

Comments
 (0)